fixed regex bug for gitlab url. added logging and unit tests for http request module

Author: katzy687

package/cloudshell/cm/ansible/domain/Helpers/gitlab_api_url_validator.py

@@ -11,13 +11,13 @@ def is_gitlab_rest_url(url):
     EXAMPLE = "http://<SERVER_IP>/api/4/projects/<PROJECT_ID>/repository/files/<PROJECT_PATH>/raw?ref=<GIT_BRANCH>"
 
     # DETERMINE INTENT TO USE GITLAB
-    initial_pattern_check = "api/v\d/projects/\d/repository/files"
+    initial_pattern_check = "api/v\d/projects/\d+/repository/files"
     matching = re.search(initial_pattern_check, url)
     if not matching:
         return False
 
     # VALIDATE ENTIRE GITLAB URL STRING
-    gitlab_api_pattern = "https?://.+/api/v\d/projects/\d/repository/files/.+/raw\?ref=.+"
+    gitlab_api_pattern = "https?://.+/api/v\d/projects/\d+/repository/files/.+/raw\?ref=.+"
     matching = re.match(gitlab_api_pattern, url)
     if not matching:
         raise Exception("Gitlab Rest API URL failed validation. Should be of form: {}".format(EXAMPLE))
@@ -26,6 +26,6 @@ def is_gitlab_rest_url(url):
 
 
 if __name__ == "__main__":
-    input_url = "http://192.168.85.62/api/v4/projects/4/repository/files/hello_world.sh/raw?ref=master"
+    input_url = "http://192.168.85.62/api/v4/projects/41/repository/files/hello_world.sh/raw?ref=master"
     is_gitlab = is_gitlab_rest_url(input_url)
     print(is_gitlab)

package/cloudshell/cm/ansible/domain/ansible_command_executor.py

@@ -20,7 +20,7 @@ def execute_playbook(self, playbook_file, inventory_file, args, output_writer, l
         """
         :type playbook_file: str
         :type inventory_file: str
-        :type args: list[str]
+        :type args: str
         :type logger: Logger
         :type output_writer: OutputWriter
         :type cancel_sampler: CancellationSampler

package/cloudshell/cm/ansible/domain/http_request_service.py

@@ -5,7 +5,7 @@
 
 
 class HttpRequestService(object):
-    def get_response(self, url, auth, logger=None):
+    def get_response(self, url, auth, logger):
         """
         :param str url:
         :param HttpAuth auth:
@@ -14,37 +14,48 @@ def get_response(self, url, auth, logger=None):
         """
         is_gitlab_url = is_gitlab_rest_url(url)
         if is_gitlab_url:
+            logger.info("=== GITLAB Rest API request ===".format(url))
             response = self._get_gitlab_response(url, auth, logger)
             self._validate_response_status_code(response)
             self._invalidate_gitlab_login_page(response)
         else:
+            # if auth:
+            #     if not auth.username:
+            #         raise Exception("Auth download missing 'User' attribute value.")
+
             auth = (auth.username, auth.password) if auth else None
-            response = requests.get(url, auth=auth, stream=True, verify=False)
+            if auth:
+                logger.info("Auth download flow.")
+                response = requests.get(url, auth=auth, stream=True, verify=False)
+            else:
+                logger.info("No-auth download flow.")
+                response = requests.get(url, stream=True, verify=False)
             self._validate_response_status_code(response)
             self._invalidate_html(response.content)
+
+            logger.info("Playbook download response: {}".format(response.status_code))
         return response
 
     @staticmethod
-    def _get_gitlab_response(url, auth, logger=None):
+    def _get_gitlab_response(url, auth, logger):
         """
         :param url:
         :param auth:
         :param logging.Logger logger:
         :return:
         """
-        # GITLAB REST API CALL - ADDING TOKEN HEADER
-        if logger:
-            logger.info("downloading script via Gitlab Rest API call: {}".format(url))
         if auth:
+            logger.info("Gitlab download from private repo with token...")
             headers = {"PRIVATE-TOKEN": auth.password}
             return requests.get(url, stream=True, verify=False, headers=headers)
         else:
+            logger.info("Gitlab no auth download...")
             return requests.get(url, stream=True, verify=False)
 
     @staticmethod
     def _validate_response_status_code(response):
         if not response.ok:
-            raise Exception('Failed to download script file: ' + str(response.status_code) + ' ' + response.reason +
+            raise Exception('Failed to download script file: ' + str(response.status_code) + ' - ' + response.reason +
                             '. Please make sure the URL is valid, and any required credentials are correct.')
 
     @staticmethod
@@ -57,20 +68,8 @@ def _invalidate_html(self, content):
 
     def _invalidate_gitlab_login_page(self, response):
         """
-
         :param Response response: requests response object
         :return:
         """
         if self._is_content_html(response.content) and "users/sign_in" in response.url:
             raise Exception('Authentication failed. Reached Gitlab Login. Gitlab Access Token required.')
-
-
-if __name__ == "__main__":
-    TEST_URL = "http://192.168.85.62/api/v4/projects/2/repository/files/my_playbook.yml/raw?ref=master"
-    TEST_GITLAB_TOKEN = "UAAEwzj7qSZn3oHRjzbZ"
-    # TEST_GITLAB_TOKEN = "UAAEwzj7qSZn3oHRjzbZBROKEN"
-    auth = HttpAuth(username="", password=TEST_GITLAB_TOKEN)
-    service = HttpRequestService()
-    response = service.get_response(TEST_URL, auth)
-    print(response.ok)
-    pass

package/cloudshell/cm/ansible/domain/playbook_downloader.py

@@ -49,16 +49,16 @@ def _download(self, url, auth, logger, cancel_sampler):
         :rtype [str,int]
         :return The downloaded file name
         """
-        logger.info('Downloading file from \'%s\' ...'%url)
+        logger.info('Downloading file from \'%s\' ...' % url)
         response = self.http_request_service.get_response(url, auth, logger)
         file_name = self.filename_extractor.get_filename(response)
 
-        with self.file_system.create_file(file_name) as file:
+        with self.file_system.create_file(file_name) as f:
             for chunk in response.iter_content(PlaybookDownloader.CHUNK_SIZE):
                 if chunk:
-                    file.write(chunk)
+                    f.write(chunk)
                 cancel_sampler.throw_if_canceled()
-            file_size = file.tell()
+            file_size = f.tell()
 
         logger.info('Done (file: %s, size: %s bytes)).' % (file_name, file_size))
         return file_name, file_size
@@ -70,19 +70,21 @@ def _unzip(self, file_name, logger):
         :return: Playbook file name
         :rtype str
         """
-        logger.info('Zip file was found, extracting file: %s ...' % (file_name))
+        logger.info('Zip file was found, extracting file: %s ...' % file_name)
         zip_files = self.zip_service.extract_all(file_name)
-        logger.info('Done (extracted %s files).'%len(zip_files))
+        logger.info('Done (extracted %s files).' % len(zip_files))
         logger.info('Files: ' + os.linesep + (os.linesep+'\t').join(zip_files))
 
-        yaml_files = [file_name for file_name in self.file_system.get_entries(self.file_system.get_working_dir()) if file_name.endswith(".yaml") or file_name.endswith(".yml")]
+        yaml_files = [file_name for file_name in self.file_system.get_entries(self.file_system.get_working_dir())
+                      if file_name.endswith(".yaml") or file_name.endswith(".yml")]
         playbook_name = None
         if len(yaml_files) > 1:
-            playbook_name = next((file_name for file_name in yaml_files if file_name == "site.yaml" or file_name == "site.yml"), None)
+            playbook_name = next((file_name for file_name in yaml_files
+                                  if file_name == "site.yaml" or file_name == "site.yml"), None)
         if len(yaml_files) == 1:
             playbook_name = yaml_files[0]
         if not playbook_name:
             raise Exception("Playbook file name was not found in zip file")
-        logger.info('Found playbook: \'%s\' in zip file' % (playbook_name))
+        logger.info('Found playbook: \'%s\' in zip file' % playbook_name)
 
         return playbook_name

package/tests/mocks/file_system_service_mock.py

@@ -60,6 +60,7 @@ def get_entries(self, path):
             file_entries.append(file.path)
         return self.folders + file_entries
 
+
 class FileMock(object):
     def __init__(self, path, full_path):
         self.path = path

package/tests/test_http_request_service.py

@@ -0,0 +1,60 @@
+from unittest import TestCase
+from cloudshell.cm.ansible.domain.http_request_service import HttpRequestService
+from models import HttpAuth
+from mock import Mock
+
+# ADD CREDENTIALS HERE FOR TEST THEN REMOVE
+
+GITHUB_USER_NAME = ""
+GITHUB_PASSWORD = ""
+GITHUB_PUBLIC_RAW_URL = ""
+GITHUB_PRIVATE_REPO_RAW_URL = ""
+
+
+GITLAB_PERSONAL_ACCESS_TOKEN = "UAAEwzj7qSZn3oHRjzbZ"
+GITLAB_PUBLIC_REPO_API_URL = "http://192.168.85.62/api/v4/projects/3/repository/files/my_playbook.yml/raw?ref=master"
+GITLAB_PRIVATE_REPO_API_URL = "http://192.168.85.62/api/v4/projects/2/repository/files/my_playbook.yml/raw?ref=master"
+GITLAB_PUBLIC_REPO_RAW_URL = "http://192.168.85.62/root/public_test/-/raw/master/my_playbook.yml"
+
+
+class TestHttpService(TestCase):
+    def setUp(self):
+        self.http_service = HttpRequestService()
+        self.logger = Mock()
+
+    def test_gitlab_public_repo_raw_url(self):
+        response = self.http_service.get_response(GITLAB_PUBLIC_REPO_RAW_URL, None, self.logger)
+        self.assertTrue(response.ok)
+        print(response.status_code)
+
+    def test_gitlab_private_repo_url_with_token(self):
+        auth = HttpAuth(None, GITLAB_PERSONAL_ACCESS_TOKEN)
+        response = self.http_service.get_response(GITLAB_PRIVATE_REPO_API_URL, auth, self.logger)
+        self.assertTrue(response.ok)
+        print(response.status_code)
+
+    def test_gitlab_private_repo_url_multi_digit_project_id(self):
+        auth = HttpAuth(None, GITLAB_PERSONAL_ACCESS_TOKEN)
+        response = self.http_service.get_response(GITLAB_PRIVATE_REPO_API_URL, auth, self.logger)
+        self.assertTrue(response.ok)
+        print(response.status_code)
+
+    def test_gitlab_private_repo_url_with_token_fails(self):
+        auth = HttpAuth(None, None)
+        with self.assertRaises(Exception):
+            self.http_service.get_response(GITLAB_PRIVATE_REPO_API_URL, auth, self.logger)
+
+    def test_gitlab_public_repo_api_url_with_token(self):
+        auth = HttpAuth(None, GITLAB_PERSONAL_ACCESS_TOKEN)
+        response = self.http_service.get_response(GITLAB_PRIVATE_REPO_API_URL, auth, self.logger)
+        self.assertTrue(response.ok)
+        print(response.status_code)
+
+    def test_gitlab_public_repo_raw_url_with_token(self):
+        auth = HttpAuth(None, GITLAB_PERSONAL_ACCESS_TOKEN)
+        response = self.http_service.get_response(GITLAB_PUBLIC_REPO_RAW_URL, auth, self.logger)
+        self.assertTrue(response.ok)
+        print(response.status_code)
+
+
+

package/utility/test_gitlab_token.py

@@ -0,0 +1,14 @@
+import requests
+
+TEST_URL = "http://192.168.85.62/api/v4/projects/2/repository/files/my_playbook.yml/raw?ref=master"
+TEST_GITLAB_TOKEN = "UAAEwzj7qSZn3oHRjzbZ"
+
+
+headers = {"PRIVATE-TOKEN": TEST_GITLAB_TOKEN}
+response = requests.get(TEST_URL, stream=True, verify=False, headers=headers)
+
+if not response.ok:
+    raise Exception("Failed request. Error code: {}. Reason: {}".format(response.status_code, response.reason))
+
+print("=== Response Success ===")
+print(response.text)

package/version.txt

@@ -1 +1 @@
-1.5.4
+1.5.5