improve: do not crash on failed downloads

Author: FelixAbrahamsson

README.md

@@ -12,7 +12,7 @@ Download image files
 
 ```python
 from PIL import Image
-from fast_s3 import Fetcher
+from fast_s3 import Fetcher, Status
 
 
 large_list_of_image_paths = [...]
@@ -30,7 +30,8 @@ fetcher = Fetcher(
 )
 
 for file in fetcher:
-    Image.open(file.buffer).save(file.path)
+    if file.status != Status.error:
+        Image.open(file.buffer).save(file.path)
 
 fetcher.close()
 ```

fast_s3/__init__.py

@@ -1,2 +1,3 @@
+from .file import File, Status
 from .fetcher import Fetcher
 from .uploader import Uploader

fast_s3/fetcher.py

@@ -2,7 +2,9 @@
 from pathlib import Path
 from typing import List, Union
 
-from .file import File
+from botocore.exceptions import ClientError
+
+from .file import File, Status
 from .transfer_manager import transfer_manager
 
 
@@ -42,21 +44,24 @@ def __iter__(self):
 
         if self.ordered:
             for _ in range(len(self)):
-                file = self.files.pop(0)
-                file.future.result()
-                yield file
-                self.queue_download_()
+                yield self.process_index(0)
         else:
             for _ in range(len(self)):
                 for index, file in enumerate(self.files):
                     if file.future.done():
                         break
                 else:
                     index = 0
-                file = self.files.pop(index)
-                file.future.result()
-                yield file
-                self.queue_download_()
+                yield self.process_index(index)
+
+    def process_index(self, index):
+        file = self.files.pop(index)
+        self.queue_download_()
+        try:
+            file.future.result()
+            return file.with_status(Status.done)
+        except ClientError:
+            return file.with_status(Status.error)
 
     def queue_download_(self):
         if self.current_path_index < len(self):

fast_s3/file.py

@@ -1,15 +1,28 @@
 import io
+from enum import Enum
 from pathlib import Path
 from typing import Union
 
 from pydantic import BaseModel
 from s3transfer.futures import TransferFuture
 
 
+class Status(str, Enum):
+    pending = "pending"
+    done = "done"
+    error = "error"
+
+
 class File(BaseModel):
     buffer: io.BytesIO
     future: TransferFuture
     path: Union[str, Path]
+    status: Status = Status.pending
 
     class Config:
         arbitrary_types_allowed = True
+
+    def with_status(self, status: Status):
+        attributes = self.dict()
+        attributes.update(status=status)
+        return File(**attributes)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "fast_s3"
-version = "0.0.0"
+version = "0.0.2"
 description = "Download images from s3 fast"
 authors = ["NextML AB"]
 readme = "README.md"