Merge pull request #36 from transifex/retry_fetch

Retry fetch languages/translations on 202

Author: Konstantinos Bairaktaris

tests/native/core/test_cds.py

@@ -11,9 +11,8 @@ class TestCDSHandler(object):
 
     def _lang_lists_equal(self, list_1, list_2):
         try:
-            sorted_1, sorted_2 = [
-                sorted(l, key=itemgetter('code')) for l in (list_1, list_2)
-            ]
+            sorted_1, sorted_2 = [sorted(language, key=itemgetter('code'))
+                                  for language in (list_1, list_2)]
             pairs = zip(sorted_1, sorted_2)
             difference = any(x != y for x, y in pairs)
         except Exception:
@@ -100,8 +99,8 @@ def test_fetch_languages(self, patched_logger):
 
         assert cds_handler.fetch_languages() == []
         patched_logger.error.assert_called_with(
-            'Error retrieving languages from CDS: UnknownError '
-            '(`400 Client Error: Bad Request for url: https://some.host/languages`)'
+            'Error retrieving languages from CDS: UnknownError (`400 Client '
+            'Error: Bad Request for url: https://some.host/languages`)'
         )
         responses.reset()
 
@@ -125,8 +124,8 @@ def test_fetch_languages(self, patched_logger):
 
         assert cds_handler.fetch_languages() == []
         patched_logger.error.assert_called_with(
-            'Error retrieving languages from CDS: UnknownError '
-            '(`403 Client Error: Forbidden for url: https://some.host/languages`)'
+            'Error retrieving languages from CDS: UnknownError (`403 Client '
+            'Error: Forbidden for url: https://some.host/languages`)'
         )
         responses.reset()
 
@@ -333,9 +332,8 @@ def test_fetch_translations_etags_management(self, patched_logger):
 
         responses.add(
             responses.GET, cds_host + '/content/en',
-            json={
-                # whatever, we don't care about the content of json repsone atm.
-            },
+            # whatever, we don't care about the content of json repsone atm.
+            json={},
             status=304
         )
 
@@ -451,3 +449,42 @@ def test_get_headers(self):
             'Accept-Encoding': 'gzip',
             'If-None-Match': 'something'
         }
+
+    @responses.activate
+    def test_retry_fetch_languages(self):
+        cds_host = 'https://some.host'
+        cds_handler = CDSHandler(
+            ['el', 'en'],
+            'some_token',
+            host=cds_host,
+        )
+        responses.add(responses.GET, cds_host + '/languages', status=202)
+        responses.add(responses.GET, cds_host + '/languages', status=202)
+        responses.add(responses.GET, cds_host + '/languages',
+                      json={'data': [{'code': "el"},
+                                     {'code': "en"}],
+                            'meta': {'some_key': "some_value"}},
+                      status=200)
+        languages_response = cds_handler.fetch_languages()
+        assert self._lang_lists_equal(
+            languages_response,
+            [{'code': 'el'}, {'code': 'en'}]
+        )
+
+    @responses.activate
+    def test_retry_fetch_translations(self):
+        cds_host = 'https://some.host'
+        cds_handler = CDSHandler(
+            ['el', 'en'],
+            'some_token',
+            host=cds_host,
+        )
+        responses.add(responses.GET, cds_host + '/content/el', status=202)
+        responses.add(responses.GET, cds_host + '/content/el', status=202)
+        responses.add(responses.GET,
+                      cds_host + '/content/el',
+                      json={'data': {'source': {'string': "translation"}}},
+                      status=200)
+        translations = cds_handler.fetch_translations('el')
+        assert (translations ==
+                {'el': (True, {'source': {'string': "translation"}})})

transifex/native/cds.py

@@ -47,7 +47,8 @@ def get(self, key):
 class CDSHandler(object):
     """Handles communication with the Content Delivery Service."""
 
-    def __init__(self, configured_languages, token, secret=None, host=TRANSIFEX_CDS_HOST):
+    def __init__(self, configured_languages, token, secret=None,
+                 host=TRANSIFEX_CDS_HOST):
         """Constructor.
 
         :param list configured_languages: a list of language codes for the
@@ -72,10 +73,13 @@ def fetch_languages(self):
         languages = []
 
         try:
-            response = requests.get(
-                self.host + cds_url,
-                headers=self._get_headers(),
-            )
+            last_response_status = 202
+            while last_response_status == 202:
+                response = requests.get(
+                    self.host + cds_url,
+                    headers=self._get_headers(),
+                )
+                last_response_status = response.status_code
 
             if not response.ok:
                 logger.error(
@@ -89,18 +93,16 @@ def fetch_languages(self):
             languages = json_content['data']
 
         except (KeyError, ValueError):
-            # Compatibility with python2.7 where `JSONDecodeError` doesn't exist
+            # Compatibility with python2.7 where `JSONDecodeError` doesn't
+            # exist
             logger.error(
                 'Error retrieving languages from CDS: Malformed response')
         except requests.ConnectionError:
             logger.error(
                 'Error retrieving languages from CDS: ConnectionError')
         except Exception as e:
-            logger.error(
-                'Error retrieving languages from CDS: UnknownError (`{}`)'.format(
-                    str(e)
-                )
-            )
+            logger.error('Error retrieving languages from CDS: UnknownError '
+                         '(`{}`)'.format(str(e)))
 
         return languages
 
@@ -128,12 +130,16 @@ def fetch_translations(self, language_code=None):
                 set(self.configured_language_codes):
 
             try:
-                response = requests.get(
-                    self.host + cds_url.format(language_code=language_code),
-                    headers=self._get_headers(
-                        etag=self.etags.get(language_code)
+                last_response_status = 202
+                while last_response_status == 202:
+                    response = requests.get(
+                        (self.host +
+                         cds_url.format(language_code=language_code)),
+                        headers=self._get_headers(
+                            etag=self.etags.get(language_code)
+                        )
                     )
-                )
+                    last_response_status = response.status_code
 
                 if not response.ok:
                     logger.error(
@@ -155,19 +161,19 @@ def fetch_translations(self, language_code=None):
                     )
 
             except (KeyError, ValueError):
-                # Compatibility with python2.7 where `JSONDecodeError` doesn't exist
-                logger.error(
-                    'Error retrieving translations from CDS: Malformed response')  # pragma no cover
+                # Compatibility with python2.7 where `JSONDecodeError` doesn't
+                # exist
+                logger.error('Error retrieving translations from CDS: '
+                             'Malformed response')  # pragma no cover
                 translations[language_code] = (False, {})  # pragma no cover
             except requests.ConnectionError:
                 logger.error(
                     'Error retrieving translations from CDS: ConnectionError')
                 translations[language_code] = (False, {})
             except Exception as e:
                 logger.error(
-                    'Error retrieving translations from CDS: UnknownError (`{}`)'.format(
-                        str(e)
-                    )
+                    'Error retrieving translations from CDS: UnknownError '
+                    '(`{}`)'.format(str(e))
                 )  # pragma no cover
                 translations[language_code] = (False, {})
 
@@ -176,19 +182,17 @@ def fetch_translations(self, language_code=None):
     def push_source_strings(self, strings, purge=False):
         """Push source strings to CDS.
 
-        :param list(SourceString) strings: a list of `SourceString` objects holding
-            source strings
-        :param bool purge: True deletes destination source content not included in
-                           pushed content.
-                           False appends the pushed content to destination source
-                           content.
+        :param list(SourceString) strings: a list of `SourceString` objects
+            holding source strings
+        :param bool purge: True deletes destination source content not included
+            in pushed content. False appends the pushed content to destination
+            source content.
         :return: the HTTP response object
         :rtype: requests.Response
         """
         if not self.secret:
-            raise Exception(
-                'You need to use `TRANSIFEX_SECRET` when pushing source content'
-            )
+            raise Exception('You need to use `TRANSIFEX_SECRET` when pushing '
+                            'source content')
 
         cds_url = TRANSIFEX_CDS_URLS['PUSH_SOURCE_STRINGS']
 
@@ -208,11 +212,8 @@ def push_source_strings(self, strings, purge=False):
             logger.error(
                 'Error pushing source strings to CDS: ConnectionError')
         except Exception as e:
-            logger.error(
-                'Error pushing source strings to CDS: UnknownError (`{}`)'.format(
-                    str(e)
-                )
-            )
+            logger.error('Error pushing source strings to CDS: UnknownError '
+                         '(`{}`)'.format(str(e)))
 
         return response