fix delayed caching; further improved typing annotations

Author: gr-markin

src/ts_deepscan/__init__.py

@@ -188,6 +188,8 @@ def onProgress(finished: int, total: int):
         'finished': _scanner.finishedTasks
     }
 
+    result = {relpath:{res.category: res.data for res in res_list} for relpath, res_list in result.items()}
+
     _scan = Scan(result=result,
                  no_result=no_result,
                  stats=stats,  # prepare_stats(result, no_result, stats)

src/ts_deepscan/scanner/ParallelScanner.py

@@ -100,9 +100,7 @@ def _do_scan(self, files: t.List[FileScanInput]) -> ScanResults:
             self._notifyCompletion(relpath, result, errors)
 
             if result:
-                results.update({
-                    relpath: result
-                })
+                results[relpath] = result
 
             tasks_done += 1
 

src/ts_deepscan/scanner/PoolExecutorScanner.py

@@ -22,7 +22,7 @@ def _do_scan(self, files: t.List[FileScanInput]) -> ScanResults:
         results = {}
         tasks = []
 
-        def task_completed(_task: futures.Future[t.Tuple[str, FileScanResult, list]]):
+        def task_completed(_task: futures.Future[FileScanResult]):
             if _task.cancelled():
                 self.finishedTasks += 1
                 self._progress()
@@ -32,9 +32,7 @@ def task_completed(_task: futures.Future[t.Tuple[str, FileScanResult, list]]):
             self._notifyCompletion(relpath, result, errors)
 
             if result:
-                results.update({
-                    relpath: result
-                })
+                results[relpath] = result                
 
         executor = futures.ProcessPoolExecutor(max_workers=self._num_jobs if self._num_jobs > 0 else None)
 

src/ts_deepscan/scanner/PoolScanner.py

@@ -9,6 +9,7 @@
 from queue import Queue
 
 from . import FileScanInput, ScanResults
+from ..analyser import AnalysisResult
 
 from .pool import Pool
 from .Scanner import Scanner
@@ -56,13 +57,12 @@ def process_results():
                     error_msgs.append(f'An error occured while scanning {relpath} using \'{cat}\' analyser')
 
                 self._notifyCompletion(relpath, result, error_msgs)
-
+    
                 if result:
-                    results.update({
-                        relpath: result
-                    })
+                    results[relpath] = result
+
 
-        def report_results(relpath: str, result: dict, errors: t.Dict[str, str]):
+        def report_results(relpath: str, result: t.List[AnalysisResult], errors: t.Dict[str, str]):
             results_queue.put((relpath, result, errors))
 
         pool = futures.ThreadPoolExecutor()
@@ -87,11 +87,11 @@ def report_results(relpath: str, result: dict, errors: t.Dict[str, str]):
 
     def _get_scan_file_fn(self) -> t.Callable[[Path,
                                                t.Optional[Path],
-                                               t.Callable[[str, dict, t.Dict[str, str]], None]], None]:
+                                               t.Callable[[str, t.List[AnalysisResult], t.Dict[str, str]], None]], None]:
 
         def _scan_file(path: Path,
                        root: t.Optional[Path],
-                       report_results: t.Callable[[str, dict, t.Dict[str, str]], None]):
+                       report_results: t.Callable[[str, t.List[AnalysisResult], t.Dict[str, str]], None]):
 
             PoolScanner._scan_file_parallel(path,
                                             root,
@@ -107,18 +107,16 @@ def _scan_file_parallel(path: Path,
                             root: t.Optional[Path],
                             analysers: t.List[FileAnalyser],
                             timeout: int,
-                            report_results: t.Callable[[str, dict, t.Dict[str, str]], None],
+                            report_results: t.Callable[[str, t.List[AnalysisResult], t.Dict[str, str]], None],
                             pool: Pool):
-        result = {}
+        result = []
         errors = {}
 
         relpath = str(path.relative_to(root) if root else path)
 
-        def task_completed(_cat):
-            def _callback(_res):
-                if _res:
-                    result[_res.category] = _res.data
-            return _callback
+        def task_completed(_res):
+            if _res:
+                result.append(_res)
 
         def task_failed(_cat):
             def _callback(_err):
@@ -127,7 +125,7 @@ def _callback(_err):
 
         tasks = [pool.apply_async(
             _apply_analysis, (analyser, path, root),
-            callback=task_completed(analyser.category),
+            callback=task_completed,
             error_callback=task_failed(analyser.category))
                 for analyser in analysers if analyser.accepts(path)]
 

src/ts_deepscan/scanner/Scanner.py

@@ -16,9 +16,11 @@
 from gitignore_parser import parse_gitignore
 
 from . import FileScanInput, FileScanResult, ScanResults
-from ..analyser import FileAnalyser
+
+from ..analyser import FileAnalyser, AnalysisResult
 from .postprocessing import PostProcessor
 
+
 def _register_unpack_formats() -> t.Set[str]:
     """
     Registers unpack formats and returns all supported archive extensions.
@@ -79,7 +81,7 @@ def __init__(self,
         # Callback accepting a relative path
         self.onPathIgnored: t.Optional[t.Callable[[str], None]] = None
         # Callback accepting relative path and results
-        self.onFileScanCompleted: t.Optional[t.Callable[[str, dict, list], None]] = None
+        self.onFileScanCompleted: t.Optional[t.Callable[[str, t.List[AnalysisResult], t.List[str]], None]] = None
         # Callback accepting number of finished and total tasks
         self.onProgress: t.Optional[t.Callable[[int, int], None]] = None
 
@@ -90,22 +92,22 @@ def __init__(self,
         self._cleanup: t.List[Path] = []
 
     @staticmethod
-    def _scan_file(path: Path, analysers: t.List[FileAnalyser], root: t.Optional[Path]) -> t.Tuple[str, FileScanResult, t.List[str]]:
-        result = {}
+    def _scan_file(path: Path, analysers: t.List[FileAnalyser], root: t.Optional[Path]) -> FileScanResult:
+        results = []
         errors = []
 
         relpath = str(path.relative_to(root) if root else path)
 
         for analyser in analysers:
             try:
                 if analyser.accepts(path) and (res := analyser(path, root=root)):
-                    result[res.category] = res.data
+                    results.append(res)
             except: # noqa
                 msg = f'An error occured while scanning {relpath} using \'{analyser.category}\' analyser'
                 util.error(msg)
                 errors.append(msg)
 
-        return relpath, result, errors
+        return relpath, results, errors
 
     @property
     def options(self) -> dict:
@@ -201,11 +203,9 @@ def _do_scan(self, files: t.List[FileScanInput]) -> ScanResults:
             relpath, result, errors = self.__class__._scan_file(path, self.analysers, root)
 
             self._notifyCompletion(relpath, result, errors)
-
+            
             if result:
-                results.update({
-                    relpath: result
-                })
+                results[relpath] = result
 
         return results
 
@@ -222,7 +222,7 @@ def _progress(self):
         if self.onProgress:
             self.onProgress(self.finishedTasks, self.totalTasks)
 
-    def _notifyCompletion(self, relpath: str, result: dict, errors: list):
+    def _notifyCompletion(self, relpath: str, result: t.List[AnalysisResult], errors: t.List[str]):
         if self.onFileScanCompleted:
             self.onFileScanCompleted(relpath, result, errors)
 

src/ts_deepscan/scanner/__init__.py

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import typing as t
-import dataclasses_json
 import osadl_matrix
 
 from pathlib import Path
@@ -13,18 +12,19 @@
 from dataclasses import dataclass, field
 from dataclasses_json import dataclass_json
 
+from ..analyser import AnalysisResult
 
 # Alias for a file scan input represented as:
 #  ( absolute_file_path, optional_root_path )
 FileScanInput = t.Tuple[Path, t.Optional[Path]]
 
 # Alias for a file scan result represented as:
-#  { analyser_category: analyser_result_data }
-FileScanResult = t.Dict[str, t.Any]
+#  ( relative_file_path, [ analyser_results ], [ error_messages ] )
+FileScanResult = t.Tuple[str, t.List[AnalysisResult], t.List[str]]
 
 # Alisas for scanner results represented as:
-#  { relative_file_path: { analyser_category: analyser_result_data } }
-ScanResults = t.Dict[str, FileScanResult]
+#  { relative_file_path: [ analyser_results ] }
+ScanResults = t.Dict[str, t.List[AnalysisResult]]
 
 
 @dataclass_json

src/ts_deepscan/scanner/postprocessing/__init__.py

@@ -28,8 +28,6 @@ def apply(self, results: ScanResults) -> ScanResults:
 
 
 class CachingPostProcessor(PostProcessor):
-    from ...caching import ResultsCache, CacheableResult
-
     def __init__(self, cache: ResultsCache, processor: t.Optional[PostProcessor] = None):
         self._cache = cache
         self._processor = processor        
@@ -38,10 +36,9 @@ def apply(self, results: ScanResults) -> ScanResults:
         if self._processor:
             results = self._processor.apply(results)
 
-        for file_results in results.values():
-            for cat, res in file_results.items():
+        for file_results in results.values():            
+            for res in file_results:
                 if isinstance(res, CacheableResult):
                     self._cache.store(res)
-                    file_results[cat] = res.data
 
         return results