Introduce fetch translations timeout setting

Author: foteinigk

README.md

@@ -43,6 +43,40 @@ To learn more about using Transifex Python toolkit check:
 * For a general overview visit [Transifex Native overview](https://developers.transifex.com/docs/native?utm_campaign=tx-native&utm_source=github&utm_medium=link)
 * For some common questions & answers check our [Transifex Native community](https://community.transifex.com/c/transifex-native/17)
 
+# Django settings reference
+
+The Transifex Native Django sdk is controlled via a set of configuration options defined in Django settings:
+
+- TRANSIFEX_TOKEN: API token that connects your application to a Transifex project.
+Must be set for both pushing source strings and fetching translations.
+
+- TRANSIFEX_SECRET: Secret used together with the token for authenticated operations against CDS (e.g. pushing source content, invalidating cache).
+
+- TRANSIFEX_CDS_HOST: Override the default CDS host (https://cds.svc.transifex.net).
+
+- TRANSIFEX_FILTER_STATUS: Optional CDS filter[status] parameter used when fetching translations (e.g. "reviewed", "proofread"). If not set, CDS returns all available statuses.
+
+- TRANSIFEX_FILTER_TAGS: Optional CDS filter[tags] parameter used when fetching translations. Use this to limit fetched content to specific tags.
+
+- TRANSIFEX_MISSING_POLICY: Custom “missing translation” policy class.
+Defaults to the built-in SourceStringPolicy when not provided.
+
+- TRANSIFEX_ERROR_POLICY: Custom error handling policy class.
+Defaults to SourceErrorPolicy.
+
+- TRANSIFEX_CACHE
+Custom cache implementation. Defaults to an in-memory cache (MemoryCache) if not provided. Can be used to integrate with a shared cache (e.g. Redis, memcached).
+
+- SKIP_TRANSLATIONS_SYNC: If True, disables automatic translation sync (OTA) for this environment.
+
+- TRANSIFEX_SYNC_INTERVAL: Interval in seconds for the background sync daemon that fetches translations. Default: 30 * 60 (30 minutes). Set to 0 to disable periodic sync and only fetch on startup.
+
+- TRANSIFEX_FETCH_ALL_LANGUAGES: When True, fetch translations for all languages configured in CDS. When False (default), only fetch translations for languages listed in Django’s LANGUAGES setting.
+
+- TRANSIFEX_FETCH_TEMEOUT:
+Maximum time in seconds to wait when fetching translations or locales from CDS.
+0 (default) = no global timeout.
+
 # License
 
 Licensed under Apache License 2.0, see `LICENSE` file.

tests/native/core/test_cds.py

@@ -2,11 +2,10 @@
 
 import pytest
 import responses
-from mock import patch
+from mock import patch, MagicMock
 from transifex.native.cds import CDSHandler
 from transifex.native.parsing import SourceString
 
-
 class TestCDSHandler(object):
 
     def _lang_lists_equal(self, list_1, list_2):
@@ -635,3 +634,39 @@ def test_invalidate(self, patched_logger):
             for x in ('Unprocessable Entity', 'None')
         ]
         assert patched_logger.error.call_args[0][0] in messages
+
+
+    @patch('transifex.native.cds.requests.get')
+    @patch('transifex.native.cds.time.time')
+    @patch('transifex.native.cds.time.sleep')
+    def test_retry_get_request_times_out_on_202(self, mock_sleep, mock_time, mock_get):
+        """
+        If CDS keeps returning 202 and fetch_timeout is set, retry_get_request
+        should stop retrying after the timeout and return the last response.
+        """
+        cds_handler = CDSHandler(
+            ['el', 'en'],
+            'some_token',
+            fetch_timeout=1,  # 1 second total timeout
+        )
+
+        # Always return a 202 response
+        response_202 = MagicMock()
+        response_202.status_code = 202
+        mock_get.return_value = response_202
+
+        # Simulate time:
+        # - first call to time.time() -> start_ts = 0
+        # - after first loop iteration -> 0.5 (still under timeout)
+        # - after second loop iteration -> 1.1 (exceeds timeout, should exit)
+        mock_time.side_effect = [0, 0.5, 1.1]
+        mock_sleep.return_value = None
+
+        result = cds_handler.retry_get_request('https://some.host/languages')
+
+        # We should have called GET twice (two 202s) and then exited due to timeout
+        assert mock_get.call_count == 2
+        assert result is response_202
+
+        # We should have slept twice for the two 202 responses
+        assert mock_sleep.call_count == 2

transifex/native/__init__.py

@@ -8,6 +8,7 @@ def init(
     fetch_all_langs=False,
     filter_tags=None,
     filter_status=None,
+    fetch_timeout=0,
 ):
     """Initialize the framework.
 
@@ -25,6 +26,8 @@ def init(
     :param bool fetch_all_langs: force pull all remote languages
     :param str filter_tags: fetch only content with tags
     :param str filter_status: fetch only content with specific translation status
+    :param int fetch_timeout: maximum time in seconds to wait when fetching
+    translations or locales from CDS
     """
     if not tx.initialized:
         tx.init(
@@ -38,6 +41,7 @@ def init(
             fetch_all_langs=fetch_all_langs,
             filter_tags=filter_tags,
             filter_status=filter_status,
+            fetch_timeout=fetch_timeout,
         )
 
 

transifex/native/cds.py

@@ -57,7 +57,7 @@ class CDSHandler(object):
 
     def __init__(self, configured_languages, token, secret=None,
                  host=TRANSIFEX_CDS_HOST, fetch_all_langs=False,
-                 filter_tags=None, filter_status=None):
+                 filter_tags=None, filter_status=None, fetch_timeout=0):
         """Constructor.
 
         :param list configured_languages: a list of language codes for the
@@ -73,6 +73,7 @@ def __init__(self, configured_languages, token, secret=None,
         self.secret = secret
         self.host = host or TRANSIFEX_CDS_HOST
         self.etags = EtagStore()
+        self.fetch_timeout = fetch_timeout
 
     def fetch_languages(self):
         """Fetch the languages defined in the CDS for the specific project.
@@ -364,16 +365,33 @@ def _get_headers(self, use_secret=False, etag=None):
 
     def retry_get_request(self, *args, **kwargs):
         """ Resilient function for GET requests """
-        retries, last_response_status = 0, 202
-        while (last_response_status == 202 or
-                500 <= last_response_status < 600 and
-                retries < MAX_RETRIES):
-
-            if 500 <= last_response_status < 600:
-                retries += 1
-                time.sleep(retries * RETRY_DELAY_SEC)
+        retries_5xx = 0
+        last_response_status = 202
+        start_ts = time.time()
+        max_total_seconds = self.fetch_timeout
 
+        while True:
             response = requests.get(*args, **kwargs)
             last_response_status = response.status_code
 
-        return response
+            # Success or non-retryable status -> return immediately
+            if last_response_status < 500 and last_response_status != 202:
+                return response
+
+            # 202 handling
+            if last_response_status == 202:
+                time.sleep(RETRY_DELAY_SEC)
+
+            # 5xx handling
+            elif 500 <= last_response_status < 600:
+                retries_5xx += 1
+                if retries_5xx > MAX_RETRIES:
+                    return response
+                time.sleep(retries_5xx * RETRY_DELAY_SEC)
+
+            # Timeout handling
+            if (
+                max_total_seconds > 0 and
+                (time.time() - start_ts) >= max_total_seconds
+            ):
+                return response

transifex/native/core.py

@@ -42,6 +42,7 @@ def init(
         missing_policy=None, error_policy=None, cache=None,
         fetch_all_langs=False, filter_tags=None,
         filter_status=None,
+        fetch_timeout=0,
     ):
         """Create an instance of the core framework class.
 
@@ -62,6 +63,8 @@ def init(
         :param bool fetch_all_langs: force pull all remote languages
         :param str filter_tags: fetch only content with tags
         :param str filter_status: fetch only content with specific translation status
+        :param int fetch_timeout: maximum time in seconds to wait when fetching
+            translations or locales from CDS
         """
         self._languages = languages
         self._cache = cache or MemoryCache()
@@ -72,6 +75,7 @@ def init(
             fetch_all_langs=fetch_all_langs,
             filter_tags=filter_tags,
             filter_status=filter_status,
+            fetch_timeout=fetch_timeout,
         )
         self.initialized = True
 

transifex/native/django/apps.py

@@ -94,6 +94,7 @@ def ready(self):
             fetch_all_langs=native_settings.TRANSIFEX_FETCH_ALL_LANGUAGES,
             filter_tags=native_settings.TRANSIFEX_FILTER_TAGS,
             filter_status=native_settings.TRANSIFEX_FILTER_STATUS,
+            fetch_timeout=native_settings.TRANSIFEX_FETCH_TIMEOUT,
         )
 
         if fetch_translations:

transifex/native/django/settings.py

@@ -16,3 +16,7 @@
 TRANSIFEX_FETCH_ALL_LANGUAGES = getattr(settings,
                                         'TRANSIFEX_FETCH_ALL_LANGUAGES',
                                         False)
+TRANSIFEX_FETCH_TIMEOUT = getattr(settings,
+                                  'TRANSIFEX_FETCH_TIMEOUT',
+                                  0)
+