linting and overwrite refactoring, started segmentation implementation

Author: Jonas Widmer

.gitignore

@@ -165,6 +165,7 @@ cython_debug/
 .DS_Store
 .AppleDouble
 .LSOverride
+qodana.yaml
 
 # Icon must end with two \r
 Icon

flow_preprocessor/exceptions/exceptions.py

@@ -1,10 +1,19 @@
 class ImageFetchException(Exception):
+    """
+    Exception raised when an image cannot be fetched
+    """
     pass
 
 
 class ImageProcessException(Exception):
+    """
+    Exception raised when an image cannot be processed
+    """
     pass
 
 
 class ParseTextLinesException(Exception):
+    """
+    Exception raised when an error occurs while parsing
+    """
     pass

flow_preprocessor/preprocessing_logic/fetch_images.py

@@ -1,3 +1,7 @@
+"""
+Defining the ImageDownloader class to manage the image fetching.
+"""
+
 # ===============================================================================
 # IMPORT STATEMENTS
 # ===============================================================================
@@ -42,11 +46,16 @@ def _request_image_via_url(self, url: str, filename: str) -> None:
         :param url: the image url
         :param filename: the image filename
         """
-        response = requests.get(url)
-        response.raise_for_status()
-        with open(filename, 'wb') as file:
-            file.write(response.content)
-        logger.info(f'{self.__class__.__name__} - File downloaded: {filename}')
+        try:
+            response = requests.get(url, timeout=20)
+            response.raise_for_status()
+            with open(filename, 'wb') as file:
+                file.write(response.content)
+            logger.info('%s - File downloaded: %s', self.__class__.__name__, filename)
+        except requests.exceptions.Timeout:
+            logger.info('%s - Image download timed out', self.__class__.__name__)
+        except requests.exceptions.RequestException as e:
+            logger.info('%s - Image download failed: %s', self.__class__.__name__, e)
 
     def fetch_image(self, page: Page, img_output: str) -> None:
         """
@@ -68,26 +77,32 @@ def fetch_image(self, page: Page, img_output: str) -> None:
         except requests.exceptions.RequestException as e:
             self.failed_downloads.append(image_filename)
             logger.error(
-                f'{self.__class__.__name__} - Failed to download file {image_filename}',
+                '%s - Failed to download file %s',
+                self.__class__.__name__,
+                image_filename,
                 exc_info=True
             )
-            raise ImageFetchException('Failed to download file %s.', e)
+            raise ImageFetchException(f'Failed to download file {e}.') from e
         except (et.XMLSyntaxError, et.ParseError, IndexError, TypeError, ValueError) as e:
             if image_filename is not None:
                 self.failed_processing.append(image_filename)
             logger.error(
-                f'{self.__class__.__name__} - Error parsing file {page}',
+                '%s - Error parsing file %s',
+                self.__class__.__name__,
+                page,
                 exc_info=True
             )
-            raise ImageProcessException('Error parsing file %s: %s', e)
+            raise ImageProcessException(f'Error parsing file {e}') from e
         except Exception as e:
             if image_filename is not None:
                 self.failed_processing.append(image_filename)
             logger.error(
-                f'{self.__class__.__name__} - An unexpected error occurred for file {page}',
+                '%s - An unexpected error occurred for file %s',
+                self.__class__.__name__,
+                page,
                 exc_info=True
             )
-            raise Exception('An unexpected error occurred for file %s: %s', e)
+            raise RuntimeError(f'An unexpected error occurred for file {image_filename}: {e}') from e
 
     def get_failed_downloads(self) -> List[str]:
         """

flow_preprocessor/preprocessing_logic/models.py

@@ -1,17 +1,26 @@
+"""
+Models to use for the preprocessing package - mainly the status model
+"""
 from datetime import datetime
-
-from pydantic import BaseModel, Field
 from typing import Optional, List
 import enum
 
+from pydantic import BaseModel, Field
+
 
 class StateEnum(enum.Enum):
+    """
+    List of states of the process
+    """
     IN_PROGRESS = "in_progress"
     FAILED = "failed"
     DONE = "done"
 
 
 class PreprocessState(BaseModel):
+    """
+    The state of a preprocessing job
+    """
     process_id: str = Field(alias="process_id",
                             description="The uniqueid of the preprocess status.",
                             title="ID")
@@ -38,21 +47,6 @@ class PreprocessState(BaseModel):
                                alias="stop_on_fail",
                                description="Whether to stop processing on failure.",
                                title="Stop-On-Fail")
-    directory: str = Field(default="tmp",
-                           alias="directory",
-                           description="Directory to save the files temporarily to.",
-                           title="Directory",
-                           examples=["tmp"])
-    in_path: str = Field(default="fetched",
-                         alias="in_path",
-                         description="Path to save the fetched files.",
-                         title="In-Path",
-                         examples=["fetched"])
-    out_path: str = Field(default="preprocessed",
-                          alias="out_path",
-                          description="Path to save the preprocessed files.",
-                          title="Out-Path",
-                          examples=["preprocessed"])
     progress: int = Field(alias="progress",
                           description="The progress of the preprocess status.",
                           title="Progress",
@@ -89,7 +83,16 @@ class PreprocessState(BaseModel):
                                                       description="The names of the files that failed downloading.",
                                                       title="Filenames-Failed-Download",
                                                       default=[])
+    line_images: Optional[List] = Field(alias="line_images",
+                                        description="The names of the lines images processed.",
+                                        title="Filenames-Line-Images-Processed",
+                                        default=[])
     runtime: Optional[int] = Field(alias="runtime",
                                    description="The runtime of the preprocess status.",
                                    title="Runtime",
                                    default=0)
+    segment: Optional[bool] = Field(alias="segment",
+                                    description="Whether the images have to be segmented before processing.",
+                                    title="Segment",
+                                    default=False
+                                    )