Making file cleanup + added test

Signed-off-by: Cédric Foellmi <cedric@onekiloparsec.dev>

Author: onekiloparsec

arcsecond/cloud/uploader/allskycameraimages/uploader.py

@@ -2,7 +2,6 @@
 
 from arcsecond.cloud.uploader.errors import UploadRemoteFileMetadataError
 from arcsecond.cloud.uploader.uploader import BaseFileUploader
-from arcsecond.cloud.uploader.utils import AutoCleanupFile
 from .context import AllSkyCameraImageUploadContext
 
 
@@ -16,9 +15,10 @@ def _prepare_upload(self):
 
     def _get_upload_data_fields(self):
         filename = os.path.basename(self._file_path)
-        auto_cleanup_file = AutoCleanupFile(self._file_path)
+        self._file = open(self._file_path, "rb")
+        self._cleanup_resources.append(self._file)
         return {
-            "file": (filename, auto_cleanup_file, "application/octet-stream"),
+            "file": (filename, self._file, "application/octet-stream"),
             "camera": self._context.camera_uuid,
         }
 

arcsecond/cloud/uploader/datafiles/uploader.py

@@ -2,7 +2,6 @@
 
 from arcsecond.cloud.uploader.errors import UploadRemoteFileMetadataError
 from arcsecond.cloud.uploader.uploader import BaseFileUploader
-from arcsecond.cloud.uploader.utils import get_upload_progress_printer
 from .context import DatasetUploadContext
 from .errors import UploadRemoteDatasetPreparationError
 
@@ -38,9 +37,10 @@ def _prepare_upload(self):
 
     def _get_upload_data_fields(self):
         filename = os.path.basename(self._file_path)
-        # auto_cleanup_file = AutoCleanupFile(self._file_path)
+        self._file = open(self._file_path, "rb")
+        self._cleanup_resources.append(self._file)
         return {
-            "file": (filename, open(self._file_path, "rb"), "application/octet-stream"),
+            "file": (filename, self._file, "application/octet-stream"),
             "dataset": self._context.dataset_uuid,
         }
 

arcsecond/cloud/uploader/uploader.py

@@ -40,6 +40,7 @@ def __init__(
         self._file_size = os.path.getsize(file_path)
 
         self._uploaded_file = None
+        self._cleanup_resources = []
 
     @property
     def log_prefix(self):
@@ -71,6 +72,8 @@ def _get_upload_data(self):
 
         fields = self._get_upload_data_fields()
         assert len(fields) > 0
+        assert len(self._cleanup_resources) > 0
+
         e = MultipartEncoder(fields=fields)
 
         # Create progress monitor if display_progress is True
@@ -104,7 +107,7 @@ def _perform_upload(self):
             return
 
         if "already exists in dataset" in str(
-            error
+                error
         ):  # VERY WEAK!!! But solution with HTTP 409 isn't nice either.
             self._status = [Status.SKIPPED, Substatus.ALREADY_SYNCED, None]
         else:
@@ -119,6 +122,15 @@ def _update_metadata(self, **kwargs):
         """Update metadata after upload - to be implemented by subclasses"""
         raise NotImplementedError()
 
+    def _cleanup(self):
+        for resource in self._cleanup_resources:
+            try:
+                if hasattr(resource, 'close') and not getattr(resource, 'closed', False):
+                    resource.close()
+            except Exception as e:
+                self._logger.error(f"{self.log_prefix} {str(e)}.")
+        self._cleanup_resources = []
+
     def upload_file(self, **kwargs):
         """Generic upload method that orchestrates the upload process"""
         if self._context.is_validated is False:
@@ -141,6 +153,8 @@ def upload_file(self, **kwargs):
             # Just try again
             time.sleep(1)
             self._perform_upload()
+        finally:
+            self._cleanup()
 
         # Update metadata if upload successful
         if self._status[0] == Status.SKIPPED:

arcsecond/cloud/uploader/utils.py

@@ -1,5 +1,4 @@
 import math
-from pathlib import Path
 
 
 def is_file_hidden(path):
@@ -43,28 +42,3 @@ def percent_printer(monitor):
         print(f"[{hashes}{spaces}] {(fraction * 100):.1f}%", end="\r")
 
     return percent_printer
-
-
-class AutoCleanupFile:
-    def __init__(self, file_path):
-        self.file_path = file_path
-        self.file_obj = open(file_path, "rb")
-        self._size = Path(self.file_path).stat().st_size
-
-    def read(self, size=-1):
-        return self.file_obj.read(size)
-
-    def seek(self, offset, whence=0):
-        return self.file_obj.seek(offset, whence)
-
-    def tell(self):
-        return self.file_obj.tell()
-
-    def close(self):
-        self.file_obj.close()
-
-    def __len__(self):
-        return self._size
-
-    def __del__(self):
-        self.close()

tests/cloud/uploader/datafiles/test_uploader_full_process.py

@@ -7,14 +7,15 @@
 import responses
 
 from api.constants import ARCSECOND_API_URL_DEV
-from arcsecond import ArcsecondConfig, DatasetUploadContext, DatasetFileUploader
+from arcsecond import ArcsecondConfig, DatasetUploadContext, DatasetFileUploader, AllSkyCameraImageUploadContext, \
+    AllSkyCameraImageFileUploader
 from arcsecond.cloud.uploader.constants import Substatus, Status
 from arcsecond.options import State
-from tests.utils import prepare_successful_login, prepare_upload_files
+from tests.utils import prepare_successful_login, prepare_upload_files, prepare_upload_allskyimage
 
 
 @responses.activate
-def test_full_upload_process():
+def test_full_upload_process_datafiles():
     dataset_uuid = str(uuid.uuid4())
     telescope_uuid = str(uuid.uuid4())
     org_subdomain = 'test-portal'
@@ -76,3 +77,65 @@ def test_full_upload_process():
             assert status.value == Status.OK.value
             assert substatus.value == Substatus.DONE.value
             assert error is None
+
+
+@responses.activate
+def test_full_upload_process_allskyimages():
+    camera_uuid = str(uuid.uuid4())
+    org_subdomain = 'test-portal'
+
+    prepare_successful_login(org_subdomain)
+    prepare_upload_allskyimage(camera_uuid, org_subdomain)
+
+    # file upload
+    image_id = random.randint(1, 1000)
+    responses.post(
+        "/".join([ARCSECOND_API_URL_DEV, org_subdomain, 'allskycameraimages']) + "/",
+        status=201,
+        json={"status": "success", "id": image_id}
+    )
+    # update metadata
+    responses.patch(
+        "/".join([ARCSECOND_API_URL_DEV, org_subdomain, 'allskycameraimages', str(image_id)]) + "/",
+        status=200,
+        json={"id": image_id}
+    )
+
+    state = State(is_using_cli=False, verbose=False, api_name='cloud')
+    config = {
+        'cloud': {
+            'username': 'dummy',
+            'upload_key': '1234',
+            'api_server': ARCSECOND_API_URL_DEV
+        }
+    }
+    config = ArcsecondConfig(state, config)  # it will read your config file.
+
+    context = AllSkyCameraImageUploadContext(
+        config,
+        input_camera_uuid=camera_uuid,
+        org_subdomain=org_subdomain
+    )
+
+    context.validate()  # important step to perform before uploading.
+    fixtures_dir = Path(__file__).parent.parent.parent.parent / "fixtures"
+    fixture_files = list(fixtures_dir.glob('*.fits'))
+
+    for fixture_file in fixture_files:
+        # Create a temporary directory and copy the fixture file there
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_path = Path(temp_dir) / fixture_file.name
+            shutil.copy(fixture_file, temp_path)
+
+            # Use the actual file for uploading
+            uploader = AllSkyCameraImageFileUploader(
+                context,
+                str(temp_dir),
+                str(temp_path),
+                display_progress=False
+            )
+
+            status, substatus, error = uploader.upload_file()
+            assert status.value == Status.OK.value
+            assert substatus.value == Substatus.DONE.value
+            assert error is None

tests/utils.py

@@ -66,6 +66,19 @@ def prepare_upload_files(dataset_uuid, telescope_uuid, org_subdomain=''):
             json={"subdomain": org_subdomain, "name": "dummy org"},
         )
 
+def prepare_upload_allskyimage(camera_uuid, org_subdomain=''):
+    responses.get(
+        "/".join([part for part in [ARCSECOND_API_URL_DEV, org_subdomain, 'allskycameras', camera_uuid] if part]) + "/",
+        status=201,
+        json={"status": "success", "uuid": camera_uuid}
+    )
+    if org_subdomain:
+        responses.get(
+            "/".join([ARCSECOND_API_URL_DEV, 'organisations', org_subdomain]) + "/",
+            status=200,
+            json={"subdomain": org_subdomain, "name": "dummy org"},
+        )
+
 
 def save_test_credentials(username, memberships=None):
     config = ArcsecondConfig(State(api_name="test"))