Fix duplicate downloads (#450)

We want downloads to not overwrite files in the filestore, otherwise we
risk overwriting a copy that is already linked elsewhere and thus
creating a redundant copy from the old link. Instead, fail to write, but
don't complain. This insures we use the filestore copy that already
exists.

Author: jessebrennan

hca/dss/__init__.py

@@ -21,7 +21,7 @@
 from atomicwrites import atomic_write
 from requests.exceptions import ChunkedEncodingError, ConnectionError, ReadTimeout
 
-from hca.dss.util import iter_paths, object_name_builder, hardlink
+from hca.dss.util import iter_paths, object_name_builder, hardlink, atomic_overwrite
 from glob import escape as glob_escape
 from hca.util import tsv
 from ..util import SwaggerClient, DEFAULT_THREAD_COUNT
@@ -528,7 +528,7 @@ def _download_bundle_manifest(self, manifest_bytes, bundle_dir, dss_file):
             logger.info("Skipping download of '%s' because it already exists at '%s'.", dss_file.name, dest_path)
         else:
             self._make_dirs_if_necessary(dest_path)
-            with atomic_write(dest_path, mode="wb", overwrite=True) as fh:
+            with atomic_overwrite(dest_path, mode="wb") as fh:
                 fh.write(manifest_bytes)
         file_path = os.path.join(bundle_dir, dss_file.name)
         self._make_dirs_if_necessary(file_path)
@@ -592,14 +592,14 @@ def _download_file(self, dss_file, dest_path):
         ranged get doesn't yield the correct header, then we start over.
         """
         self._make_dirs_if_necessary(dest_path)
-        with atomic_write(dest_path, mode="wb", overwrite=True) as fh:
+        with atomic_overwrite(dest_path, mode="wb") as fh:
             if dss_file.size == 0:
                 return
 
             download_hash = self._do_download_file(dss_file, fh)
 
             if download_hash.lower() != dss_file.sha256.lower():
-                # No need to delete what's been written. atomic_write ensures we're cleaned up
+                # No need to delete what's been written. atomic_overwrite ensures we're cleaned up
                 logger.error("%s", "File {}: GET FAILED. Checksum mismatch.".format(dss_file.uuid))
                 raise ValueError("Expected sha256 {} Received sha256 {}".format(
                     dss_file.sha256.lower(), download_hash.lower()))

hca/dss/util/__init__.py

@@ -3,6 +3,9 @@
 import os
 import shutil
 from builtins import FileExistsError
+
+import atomicwrites
+
 from ...util.compat import scandir
 
 log = logging.getLogger(__name__)
@@ -61,3 +64,22 @@ def hardlink(source, link_name):
             shutil.copyfile(source, link_name)
         else:
             raise
+
+
+class atomic_overwrite:
+    """Atomically write, but don't complain if file already exists"""
+
+    def __init__(self, *args, **kwargs):
+        self.writer = atomicwrites.atomic_write(*args, overwrite=False, **kwargs)
+
+    def __enter__(self):
+        return self.writer.__enter__()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        # During atomicwrites.atomic_write's commit() method which is called
+        # while exiting, a FileExistsError may be raised. It's fine to catch
+        # this since atomicwrites still cleans up after itself.
+        try:
+            return self.writer.__exit__(exc_type, exc_val, exc_tb)
+        except FileExistsError:
+            pass