Merge pull request #17 from Riminder/feature/add-parsing-eval-and-rate-limit

Feat : add parsing eval and rate limit

Author: corentin-hrflow

.gitignore

@@ -14,6 +14,7 @@ test/*
 test_assets/*
 tests/assets
 .htpasswd
+test.ipynb
 
 docker/dependencies/libs/*
 

Documentation.md

@@ -605,7 +605,7 @@ Here is an example on how to handle webhooks
 * Here an example on how to get help:
 
  ```python
-    >>> from hrflow.hrflow.profile.parsing import ProfileParsing
+    >>> from hrflow.profile.parsing import ProfileParsing
     >>> help(ProfileParsing.get)
 
     #Help on function get in module hrflow.profile.parsing:

Makefile

@@ -1,7 +1,7 @@
 # Define variables
 ARGS := 
 
-clean:
+clean: clean_cache
 	rm -rf build dist *.egg-info
 
 clean_cache:

README.md

@@ -65,12 +65,11 @@ with open("path_to_file.pdf", "rb") as f:
 
 #Parse it using this method without reference:
 response = client.profile.parsing.add_file(
-            source_key="INSERT_THE_TARGET_SOURCE_KEY",
-            profile_file=file,
-            sync_parsing=1, # This is to invoke real time parsing
-            tags=[{"name": "application_reference", "value": "TS_X12345"}], # Attach an application tag to the profile to be parsed
-        )
-
+    source_key="INSERT_THE_TARGET_SOURCE_KEY",
+    profile_file=file,
+    sync_parsing=1, # This is to invoke real time parsing
+    tags=[{"name": "application_reference", "value": "TS_X12345"}], # Attach an application tag to the profile to be parsed
+)
 ```
 
 
@@ -79,4 +78,4 @@ response = client.profile.parsing.add_file(
 - [HrFlow.ai Academy](https://www.youtube.com/@hrflow.aiacademy9534) on Youtube for videos on how to get started with HrFlow.ai
 - [Updates page](https://updates.hrflow.ai/) to keep you informed about our product releases
 - [Documentation](https://developers.hrflow.ai/reference/authentication) to provide information on HrFlow.ai features
-- [Our Roadmap](https://roadmap.hrflow.ai/) to show upcoming features or request new ones
+- [Our Roadmap](https://roadmap.hrflow.ai/) to show upcoming features or request new ones

hrflow/__init__.py

@@ -7,4 +7,4 @@
     __url__,
     __version__,
 )
-from .hrflow.hrflow import Hrflow
+from .hrflow import Hrflow

hrflow/hrflow/auth/__init__.py hrflow/auth/__init__.pyhrflow/hrflow/auth/__init__.py renamed to hrflow/auth/__init__.py

@@ -1,6 +1,7 @@
 import typing as t
 
-from ..utils import validate_key, validate_response
+from ..core.rate_limit import rate_limiter
+from ..core.validation import validate_key, validate_response
 
 API_SECRET_REGEX = r"^ask[rw]?_[0-9a-f]{32}$"
 
@@ -9,6 +10,7 @@ class Auth:
     def __init__(self, api):
         self.client = api
 
+    @rate_limiter
     def get(self) -> t.Dict[str, t.Any]:
         """
         Try your API Keys. This endpoint allows you to learn how to add the right

hrflow/hrflow/board/__init__.py hrflow/board/__init__.pyhrflow/hrflow/board/__init__.py renamed to hrflow/board/__init__.py

@@ -1,4 +1,5 @@
-from ..utils import (
+from ..core.rate_limit import rate_limiter
+from ..core.validation import (
     ORDER_BY_VALUES,
     validate_key,
     validate_limit,
@@ -12,6 +13,7 @@ class Board(object):
     def __init__(self, client):
         self.client = client
 
+    @rate_limiter
     def list(self, name=None, page=1, limit=30, sort_by="date", order_by="desc"):
         """
         Search boards for given filters.
@@ -42,6 +44,7 @@ def list(self, name=None, page=1, limit=30, sort_by="date", order_by="desc"):
         response = self.client.get("boards", query_params)
         return validate_response(response)
 
+    @rate_limiter
     def get(self, key=None):
         """
         Get source given a board key.

hrflow/core/__init__.py

@@ -0,0 +1,37 @@
+import os
+
+from .validation import (
+    is_valid_extension,
+    is_valid_filename,
+    validate_key,
+    validate_reference,
+)
+
+
+def format_item_payload(item, provider_key, key, reference=None, email=None):
+    provider = "source_key" if item == "profile" else "board_key"
+
+    payload = {provider: validate_key("provider", provider_key)}
+    if key:
+        payload["key"] = validate_key("item", key)
+    if reference:
+        payload["reference"] = validate_reference(reference)
+    if email:
+        payload["profile_email"] = email
+
+    return payload
+
+
+def get_files_from_dir(dir_path, is_recurcive):
+    file_res = []
+    files_path = os.listdir(dir_path)
+
+    for file_path in files_path:
+        true_path = os.path.join(dir_path, file_path)
+        if os.path.isdir(true_path) and is_recurcive:
+            if is_valid_filename(true_path):
+                file_res += get_files_from_dir(true_path, is_recurcive)
+            continue
+        if is_valid_extension(true_path):
+            file_res.append(true_path)
+    return file_res

hrflow/core/rate_limit.py

@@ -0,0 +1,59 @@
+from functools import wraps
+from time import sleep, time
+
+DEFAULT_MAX_REQUESTS_PER_MINUTE = None
+DEFAULT_MIN_SLEEP_PER_REQUEST = 0
+SECONDS_IN_MINUTE = 60
+
+
+def rate_limiter(func):
+    """
+    Decorator that applies rate limiting to a function.
+
+    Parameters in the decorated function:
+        max_requests_per_minute: <int> The maximum number of requests that can be made
+            in a minute. If None, there is no limit.
+        min_sleep_per_request: <float> The minimum time to wait between requests.
+
+    Usage:
+    >>> @rate_limiter()
+    ... def my_function(param1, param2):
+    ...    pass
+    ... my_function(1, 2, max_requests_per_minute=10, min_sleep_per_request=0.1)
+    ... # The function will be called at most 10 times per minute
+    ... # with at least 0.1 seconds between each call
+    """
+    requests_per_minute = 0
+    last_reset_time = time()
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        max_requests_per_minute = kwargs.pop(
+            "max_requests_per_minute", DEFAULT_MAX_REQUESTS_PER_MINUTE
+        )
+        min_sleep_per_request = kwargs.pop(
+            "min_sleep_per_request", DEFAULT_MIN_SLEEP_PER_REQUEST
+        )
+        nonlocal requests_per_minute, last_reset_time
+
+        current_time = time()
+        elapsed_time = current_time - last_reset_time
+
+        if elapsed_time < SECONDS_IN_MINUTE:
+            requests_per_minute += 1
+            if (
+                max_requests_per_minute is not None
+                and requests_per_minute > max_requests_per_minute
+            ):
+
+                sleep(SECONDS_IN_MINUTE - elapsed_time)
+                requests_per_minute = 0
+                last_reset_time = time()
+        else:
+            requests_per_minute = 0
+            last_reset_time = current_time
+
+        sleep(min_sleep_per_request)
+        return func(*args, **kwargs)
+
+    return wrapper

hrflow/hrflow/utils/__init__.py hrflow/core/validation.pyhrflow/hrflow/utils/__init__.py renamed to hrflow/core/validation.py

@@ -1,6 +1,5 @@
 import os
 import re
-import typing as t
 
 KEY_REGEX = r"^[0-9a-f]{40}$"
 STAGE_VALUES = [None, "new", "yes", "later", "no"]
@@ -33,22 +32,6 @@
 ]
 INVALID_FILENAME = [".", ".."]
 
-ITEM_TYPE = ["profile", "job"]
-
-
-def format_item_payload(item, provider_key, key, reference=None, email=None):
-    provider = "source_key" if item == "profile" else "board_key"
-
-    payload = {provider: validate_key("provider", provider_key)}
-    if key:
-        payload["key"] = validate_key("item", key)
-    if reference:
-        payload["reference"] = validate_reference(reference)
-    if email:
-        payload["profile_email"] = email
-
-    return payload
-
 
 def validate_boolean(name, value):
     """
@@ -126,21 +109,6 @@ def is_valid_filename(file_path):
     return name not in INVALID_FILENAME
 
 
-def get_files_from_dir(dir_path, is_recurcive):
-    file_res = []
-    files_path = os.listdir(dir_path)
-
-    for file_path in files_path:
-        true_path = os.path.join(dir_path, file_path)
-        if os.path.isdir(true_path) and is_recurcive:
-            if is_valid_filename(true_path):
-                file_res += get_files_from_dir(true_path, is_recurcive)
-            continue
-        if is_valid_extension(true_path):
-            file_res.append(true_path)
-    return file_res
-
-
 def validate_response(response):
     if response.headers["Content-Type"] != "application/json":
         return {