refactor: move some utils to core like validation or rate limit

corentin-hrflow · corentin-hrflow · commit 9df862aa337d · 2024-03-08T17:34:52.000+01:00
diff --git a/hrflow/auth/__init__.py b/hrflow/auth/__init__.py
@@ -1,6 +1,6 @@
 import typing as t
 
-from ..utils import validate_key, validate_response
+from ..core.validation import validate_key, validate_response
 
 API_SECRET_REGEX = r"^ask[rw]?_[0-9a-f]{32}$"
 
diff --git a/hrflow/board/__init__.py b/hrflow/board/__init__.py
@@ -1,4 +1,4 @@
-from ..utils import (
+from ..core.validation import (
     ORDER_BY_VALUES,
     validate_key,
     validate_limit,
diff --git a/hrflow/core/__init__.py b/hrflow/core/__init__.py
@@ -0,0 +1,37 @@
+import os
+
+from .validation import (
+    is_valid_extension,
+    is_valid_filename,
+    validate_key,
+    validate_reference,
+)
+
+
+def format_item_payload(item, provider_key, key, reference=None, email=None):
+    provider = "source_key" if item == "profile" else "board_key"
+
+    payload = {provider: validate_key("provider", provider_key)}
+    if key:
+        payload["key"] = validate_key("item", key)
+    if reference:
+        payload["reference"] = validate_reference(reference)
+    if email:
+        payload["profile_email"] = email
+
+    return payload
+
+
+def get_files_from_dir(dir_path, is_recurcive):
+    file_res = []
+    files_path = os.listdir(dir_path)
+
+    for file_path in files_path:
+        true_path = os.path.join(dir_path, file_path)
+        if os.path.isdir(true_path) and is_recurcive:
+            if is_valid_filename(true_path):
+                file_res += get_files_from_dir(true_path, is_recurcive)
+            continue
+        if is_valid_extension(true_path):
+            file_res.append(true_path)
+    return file_res
diff --git a/hrflow/core/rate_limit.py b/hrflow/core/rate_limit.py
diff --git a/hrflow/core/validation.py b/hrflow/core/validation.py
@@ -32,8 +32,6 @@
 ]
 INVALID_FILENAME = [".", ".."]
 
-ITEM_TYPE = ["profile", "job"]
-
 
 def validate_boolean(name, value):
     """
diff --git a/hrflow/job/asking.py b/hrflow/job/asking.py
@@ -1,6 +1,11 @@
 import typing as t
 
-from ..utils import KEY_REGEX, validate_key, validate_reference, validate_response
+from ..core.validation import (
+    KEY_REGEX,
+    validate_key,
+    validate_reference,
+    validate_response,
+)
 
 
 class JobAsking:
diff --git a/hrflow/job/embedding.py b/hrflow/job/embedding.py
@@ -1,4 +1,5 @@
-from ..utils import format_item_payload, validate_response
+from ..core.validation import validate_response
+from ..core import format_item_payload
 
 
 class JobEmbedding:
diff --git a/hrflow/job/scoring.py b/hrflow/job/scoring.py
@@ -1,6 +1,6 @@
 import json
 
-from ..utils import (
+from ..core.validation import (
     ORDER_BY_VALUES,
     SORT_BY_VALUES,
     STAGE_VALUES,
diff --git a/hrflow/job/searching.py b/hrflow/job/searching.py
@@ -1,6 +1,6 @@
 import json
 
-from ..utils import (
+from ..core.validation import (
     ORDER_BY_VALUES,
     SORT_BY_VALUES,
     STAGE_VALUES,
diff --git a/hrflow/job/storing.py b/hrflow/job/storing.py
@@ -1,9 +1,8 @@
 import json
 
-from ..utils import (
+from ..core.validation import (
     ORDER_BY_VALUES,
     SORT_BY_VALUES,
-    format_item_payload,
     validate_boolean,
     validate_key,
     validate_limit,
@@ -13,6 +12,7 @@
     validate_response,
     validate_value,
 )
+from ..core import format_item_payload
 
 
 class JobStoring:
diff --git a/hrflow/profile/asking.py b/hrflow/profile/asking.py
@@ -1,6 +1,11 @@
 import typing as t
 
-from ..utils import KEY_REGEX, validate_key, validate_reference, validate_response
+from ..core.validation import (
+    KEY_REGEX,
+    validate_key,
+    validate_reference,
+    validate_response,
+)
 
 
 class ProfileAsking:
diff --git a/hrflow/profile/attachment.py b/hrflow/profile/attachment.py
@@ -1,4 +1,5 @@
-from ..utils import format_item_payload, validate_response
+from ..core.validation import validate_response
+from ..core import format_item_payload
 
 
 class ProfileAttachments:
diff --git a/hrflow/profile/embedding.py b/hrflow/profile/embedding.py
@@ -1,6 +1,7 @@
 import json
 
-from ..utils import format_item_payload, validate_response
+from ..core.validation import validate_response
+from ..core import format_item_payload
 
 
 class ProfileEmbedding:
diff --git a/hrflow/profile/parsing.py b/hrflow/profile/parsing.py
@@ -2,16 +2,16 @@
 import os
 import shutil
 import uuid
+
 from tqdm import tqdm
 
-from ..utils import (
-    format_item_payload,
-    get_files_from_dir,
-    rate_limiter,
+from ..core.validation import (
     validate_key,
     validate_reference,
     validate_response,
 )
+from ..core import format_item_payload, get_files_from_dir
+from ..core.rate_limit import rate_limiter
 
 
 class ProfileParsing:
@@ -88,12 +88,12 @@ def add_file(
             "sync_parsing_indexing": sync_parsing_indexing,
             "webhook_parsing_sending": webhook_parsing_sending,
         }
-        
+
         if profile_file_name is None:
             file_payload = {"file": profile_file}
         else:
             file_payload = {"file": (profile_file_name, profile_file)}
-        
+
         response = self.client.post(
             "profile/parsing/file", data=payload, files=file_payload
         )
diff --git a/hrflow/profile/revealing.py b/hrflow/profile/revealing.py
@@ -1,4 +1,5 @@
-from ..utils import format_item_payload, validate_response
+from ..core.validation import validate_response
+from ..core import format_item_payload
 
 
 class ProfileRevealing:
diff --git a/hrflow/profile/scoring.py b/hrflow/profile/scoring.py
@@ -1,6 +1,6 @@
 import json
 
-from ..utils import (
+from ..core.validation import (
     ORDER_BY_VALUES,
     SORT_BY_VALUES,
     STAGE_VALUES,
diff --git a/hrflow/profile/searching.py b/hrflow/profile/searching.py
@@ -1,6 +1,6 @@
 import json
 
-from ..utils import (
+from ..core.validation import (
     ORDER_BY_VALUES,
     SORT_BY_VALUES,
     STAGE_VALUES,
diff --git a/hrflow/profile/storing.py b/hrflow/profile/storing.py
@@ -1,9 +1,8 @@
 import json
 
-from ..utils import (
+from ..core.validation import (
     ORDER_BY_VALUES,
     SORT_BY_VALUES,
-    format_item_payload,
     validate_boolean,
     validate_key,
     validate_limit,
@@ -13,6 +12,7 @@
     validate_response,
     validate_value,
 )
+from ..core import format_item_payload
 
 
 class ProfileStoring:
diff --git a/hrflow/profile/unfolding.py b/hrflow/profile/unfolding.py
@@ -1,6 +1,11 @@
 import typing as t
 
-from ..utils import KEY_REGEX, validate_key, validate_reference, validate_response
+from ..core.validation import (
+    KEY_REGEX,
+    validate_key,
+    validate_reference,
+    validate_response,
+)
 
 
 class ProfileUnfolding:
diff --git a/hrflow/rating/__init__.py b/hrflow/rating/__init__.py
@@ -1,4 +1,4 @@
-from ..utils import validate_response
+from ..core.validation import validate_response
 
 
 class Rating:
diff --git a/hrflow/source/__init__.py b/hrflow/source/__init__.py
@@ -1,4 +1,4 @@
-from ..utils import (
+from ..core.validation import (
     ORDER_BY_VALUES,
     validate_key,
     validate_limit,
diff --git a/hrflow/text/embedding.py b/hrflow/text/embedding.py
@@ -1,4 +1,4 @@
-from ..utils import validate_response
+from ..core.validation import validate_response
 
 
 class TextEmbedding:
diff --git a/hrflow/text/imaging.py b/hrflow/text/imaging.py
@@ -1,4 +1,4 @@
-from ..utils import validate_response
+from ..core.validation import validate_response
 
 
 class TextImaging:
diff --git a/hrflow/text/linking.py b/hrflow/text/linking.py
@@ -1,4 +1,4 @@
-from ..utils import validate_response
+from ..core.validation import validate_response
 
 
 class TextLinking:
diff --git a/hrflow/text/ocr.py b/hrflow/text/ocr.py
@@ -1,4 +1,4 @@
-from ..utils import validate_response
+from ..core.validation import validate_response
 
 
 class TextOCR:
diff --git a/hrflow/text/parsing.py b/hrflow/text/parsing.py
@@ -1,6 +1,6 @@
 import typing as t
 
-from ..utils import validate_response
+from ..core.validation import validate_response
 
 
 class TextParsing:
diff --git a/hrflow/text/tagging.py b/hrflow/text/tagging.py
@@ -1,6 +1,6 @@
 import typing as t
 
-from ..utils import validate_response
+from ..core.validation import validate_response
 
 
 class TextTagging:
diff --git a/hrflow/tracking/__init__.py b/hrflow/tracking/__init__.py
@@ -1,4 +1,4 @@
-from ..utils import validate_response
+from ..core.validation import validate_response
 
 
 class Tracking:
diff --git a/hrflow/utils/__init__.py b/hrflow/utils/__init__.py
@@ -1,54 +1,3 @@
 # noqa: F401
-import os
-
 from .evaluation import generate_parsing_evaluation_report, parsing_evaluator
-from .limit_rate import rate_limiter
 from .storing import get_all_jobs, get_all_profiles
-from .validation import (
-    INVALID_FILENAME,
-    ITEM_TYPE,
-    KEY_REGEX,
-    ORDER_BY_VALUES,
-    SORT_BY_VALUES,
-    STAGE_VALUES,
-    VALID_EXTENSIONS,
-    is_valid_extension,
-    is_valid_filename,
-    validate_boolean,
-    validate_key,
-    validate_limit,
-    validate_page,
-    validate_provider_keys,
-    validate_reference,
-    validate_response,
-    validate_value,
-)
-
-
-def format_item_payload(item, provider_key, key, reference=None, email=None):
-    provider = "source_key" if item == "profile" else "board_key"
-
-    payload = {provider: validate_key("provider", provider_key)}
-    if key:
-        payload["key"] = validate_key("item", key)
-    if reference:
-        payload["reference"] = validate_reference(reference)
-    if email:
-        payload["profile_email"] = email
-
-    return payload
-
-
-def get_files_from_dir(dir_path, is_recurcive):
-    file_res = []
-    files_path = os.listdir(dir_path)
-
-    for file_path in files_path:
-        true_path = os.path.join(dir_path, file_path)
-        if os.path.isdir(true_path) and is_recurcive:
-            if is_valid_filename(true_path):
-                file_res += get_files_from_dir(true_path, is_recurcive)
-            continue
-        if is_valid_extension(true_path):
-            file_res.append(true_path)
-    return file_res
diff --git a/tests/utils/schemas.py b/tests/utils/schemas.py
@@ -12,7 +12,7 @@
 )
 from pytest import fail
 
-from hrflow.utils import KEY_REGEX
+from hrflow.core.validation import KEY_REGEX
 
 from .enums import PERMISSION
 

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-from ..utils import (`
	`1`	`+from ..core.validation import (`
`2`	`2`	`ORDER_BY_VALUES,`
`3`	`3`	`validate_key,`
`4`	`4`	`validate_limit,`
Original file line number	Diff line number	Diff line change
`@@ -32,8 +32,6 @@`
`32`	`32`	`]`
`33`	`33`	`INVALID_FILENAME = [".", ".."]`
`34`	`34`
`35`		`-ITEM_TYPE = ["profile", "job"]`
`36`		`-`
`37`	`35`
`38`	`36`	`def validate_boolean(name, value):`
`39`	`37`	`"""`