Merge pull request #1 from QualiSystemsLab/gitlab-support

added gitlab support and param over-rides. mostly copied from custom …

Author: katzy687

README.md

@@ -2,5 +2,33 @@
 [![Code Climate](https://codeclimate.com/github/QualiSystems/Ansible-Shell/badges/gpa.svg)](https://codeclimate.com/github/QualiSystems/Ansible-Shell)
 [![Dependency Status](https://dependencyci.com/github/QualiSystems/Ansible-Shell/badge)](https://dependencyci.com/github/QualiSystems/Ansible-Shell)
 
-# Ansible-Shell
-A CloudShell 'Shell' that allows integration with Ansible.
+# Ansible-Shell-Extended
+A CloudShell 'Shell' that allows integration with Ansible. This is an extended repo of the official configuration management package. 
+Custom changes to driver and package have been added.
+
+## Custom Param Overrides
+The following configuration management parameters can be over-ridden by adding the following:
+- REPO_URL
+- REPO_USER
+- REPO_PASSWORD (will be a plain text parameter)
+- CONNECTION_METHOD
+
+## Gitlab Support
+- Gitlab links are supported, but for Private Repos require the URL to be in format of their REST api
+- http://<SERVER_IP>/api/<API_VERSION>/projects/<PROJECT_ID>/repository/files/<PROJECT_PATH>/raw?ref=<GIT_BRANCH>
+- example - http://10.160.7.7/api/v4/projects/4/repository/files/hello_world.sh/raw?ref=master
+- The password field needs to be populated with gitlab access token, which will be sent along with request as header
+- Access Token only needed for private repos, password field can be left blank for public repos
+- The "User" field can be left blank for gitlab auth. Only access token needed
+- Public Repos appear to work fine with both "raw" url link as well as the API formatted URL with no token
+- Gitlab docs - https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html
+
+## To Install
+- Download python package from releases and place in local pypi server on Quali Server
+    - Path: C:\Program Files (x86)\QualiSystems\CloudShell\Server\Config\Pypi Server Repository
+- Delete venv (if it exists) to force creation of new venv with updated package
+    - Path: C:\ProgramData\QualiSystems\venv\Ansible_Driver_{{driver_uid}}
+
+## Changelog
+- 25/06/2020 - Extending package to disable SSL Verification
+- 25/12/2020 - Added Gitlab Support & Parameter Over-rides

package/cloudshell/cm/ansible/ansible_shell.py

@@ -6,7 +6,7 @@
 from cloudshell.cm.ansible.domain.exceptions import AnsibleException
 from cloudshell.cm.ansible.domain.ansible_command_executor import AnsibleCommandExecutor, ReservationOutputWriter
 from cloudshell.cm.ansible.domain.ansible_config_file import AnsibleConfigFile
-from cloudshell.cm.ansible.domain.ansible_configuration import AnsibleConfigurationParser
+from cloudshell.cm.ansible.domain.ansible_configuration import AnsibleConfigurationParser, AnsibleConfiguration
 from cloudshell.cm.ansible.domain.file_system_service import FileSystemService
 from cloudshell.cm.ansible.domain.filename_extractor import FilenameExtractor
 from cloudshell.cm.ansible.domain.host_vars_file import HostVarsFile
@@ -118,7 +118,8 @@ def _download_playbook(self, ansi_conf, cancellation_sampler, logger):
         :rtype str
         """
         repo = ansi_conf.playbook_repo
-        auth = HttpAuth(repo.username, repo.password) if repo.username else None
+        # we need password field to be passed for gitlab auth tokens (which require token and not user)
+        auth = HttpAuth(repo.username, repo.password) if repo.password else None
         playbook_name = self.downloader.get(ansi_conf.playbook_repo.url, auth, logger, cancellation_sampler)
         return playbook_name
 

package/cloudshell/cm/ansible/domain/ansible_configuration.py

@@ -1,8 +1,14 @@
 import json
-
 from cloudshell.api.cloudshell_api import CloudShellAPISession
 
 
+# OPTIONAL SCRIPT PARAMETERS, IF PRESENT WILL OVERRIDE THE DEFAULT READ-ONLY VALUES
+REPO_URL_PARAM = "REPO_URL"
+REPO_USER_PARAM = "REPO_USER"
+REPO_PASSWORD_PARAM = "REPO_PASSWORD"
+CONNECTION_METHOD_PARAM = "CONNECTION_METHOD"
+
+
 class AnsibleConfiguration(object):
     def __init__(self, playbook_repo=None, hosts_conf=None, additional_cmd_args=None, timeout_minutes = None):
         """
@@ -16,6 +22,9 @@ def __init__(self, playbook_repo=None, hosts_conf=None, additional_cmd_args=None
         self.hosts_conf = hosts_conf or []
         self.additional_cmd_args = additional_cmd_args
 
+    def get_pretty_json(self):
+        return json.dumps(self, default=lambda o: getattr(o, '__dict__', str(o)), indent=4)
+
 
 class PlaybookRepository(object):
     def __init__(self):
@@ -36,6 +45,29 @@ def __init__(self):
         self.parameters = {}
 
 
+def over_ride_defaults(ansi_conf, params_dict):
+    """
+    go over custom params and over-ride values
+    :param AnsibleConfiguration ansi_conf:
+    :param dict params_dict:
+    :return same config:
+    :rtype AnsibleConfiguration
+    """
+    if params_dict.get(REPO_URL_PARAM):
+        ansi_conf.playbook_repo.url = params_dict[REPO_URL_PARAM]
+
+    if params_dict.get(REPO_USER_PARAM):
+        ansi_conf.playbook_repo.username = params_dict[REPO_USER_PARAM]
+
+    if params_dict.get(REPO_PASSWORD_PARAM):
+        ansi_conf.playbook_repo.password = params_dict[REPO_PASSWORD_PARAM]
+
+    if params_dict.get(CONNECTION_METHOD_PARAM):
+        ansi_conf.hosts_conf[0].connection_method = params_dict[CONNECTION_METHOD_PARAM].lower()
+
+    return ansi_conf
+
+
 class AnsibleConfigurationParser(object):
 
     def __init__(self, api):
@@ -72,7 +104,9 @@ def json_to_object(self, json_str):
             host_conf.access_key = self._get_access_key(json_host)
             host_conf.groups = json_host.get('groups')
             if json_host.get('parameters'):
-                host_conf.parameters = dict((i['name'], i['value']) for i in json_host['parameters'])
+                all_params_dict = dict((i['name'], i['value']) for i in json_host['parameters'])
+                host_conf.parameters = all_params_dict
+                ansi_conf = over_ride_defaults(ansi_conf, all_params_dict)
             ansi_conf.hosts_conf.append(host_conf)
 
         return ansi_conf
@@ -94,7 +128,7 @@ def _get_access_key(self, json_host):
     @staticmethod
     def _validate(json_obj):
         """
-        :type json_obj: json
+        :type json_obj: dict
         :rtype bool
         """
         basic_msg = 'Failed to parse ansible configuration input json: '

package/cloudshell/cm/ansible/domain/filename_extractor.py

@@ -8,24 +8,43 @@ class FilenameExtractor(object):
     def __init__(self):
         self._filename_pattern = "(?P<filename>\s*[\w,\s-]+\.(yaml|yml|zip)\s*)"
         self.filename_patterns = {
-                "content-disposition": "\s*((?i)inline|attachment|extension-token)\s*;\s*filename=" + self._filename_pattern,
-                "x-artifactory-filename": self._filename_pattern
+            "content-disposition": "\s*((?i)inline|attachment|extension-token)\s*;\s*filename=" +
+                                   self._filename_pattern,
+            "x-artifactory-filename": self._filename_pattern,
+            "X-Gitlab-File-Name": self._filename_pattern
         }
 
-    def get_filename(self,response):
-        file_name = None;
+    def get_filename(self, response):
+        file_name = None
         for header_value, pattern in self.filename_patterns.iteritems():
-            matching = re.match(pattern, response.headers.get(header_value,""))
+            matching = re.match(pattern, response.headers.get(header_value, ""))
             if matching:
                 file_name = matching.group('filename')
-                break
-        #fallback, couldn't find file name from header, get it from url
+                if file_name:
+                    return file_name.strip()
+
+        # Fallback, couldn't find file name from header, get it from url
+
+        # === raw url search ===
+        # ex. - https://raw.githubusercontent.com/QualiSystemsLab/master/my_playbook.yml
         if not file_name:
             file_name_from_url = urllib.unquote(response.url[response.url.rfind('/') + 1:])
             matching = re.match(self._filename_pattern, file_name_from_url)
             if matching:
                 file_name = matching.group('filename')
-        if not file_name:
-            raise AnsibleException("playbook file of supported types: '.yml', '.yaml', '.zip' was not found")
-        return file_name.strip()
+                if file_name:
+                    return file_name.strip()
+
+        # === Gitlab REST url search ===
+        # ex. - 'http://192.168.85.62/api/v4/projects/2/repository/files/my_playbook.yml/raw?ref=master'
+        url_split = response.url.split("/")
+        if url_split >= 2:
+            file_name_from_url = url_split[-2]
+            matching = re.match(self._filename_pattern, file_name_from_url)
+            if matching:
+                file_name = matching.group('filename')
+                if file_name:
+                    return file_name.strip()
+
+        raise AnsibleException("playbook file of supported types: '.yml', '.yaml', '.zip' was not found")
 

package/cloudshell/cm/ansible/domain/gitlab_api_url_validator.py

@@ -0,0 +1,31 @@
+import re
+
+
+def is_gitlab_rest_url(url):
+    """"
+    should be of the following form:
+    http://192.168.85.62/api/{api_version}/projects/{project_id}/repository/files/{file_path}/raw?ref={git branch}
+    ex input - http://192.168.85.62/api/v4/projects/4/repository/files/hello_world.sh/raw?ref=master
+    :param str url: the user input url
+    """
+    EXAMPLE = "http://<SERVER_IP>/api/4/projects/<PROJECT_ID>/repository/files/<PROJECT_PATH>/raw?ref=<GIT_BRANCH>"
+
+    # DETERMINE INTENT TO USE GITLAB
+    initial_pattern_check = "api/v\d/projects/\d/repository/files"
+    matching = re.search(initial_pattern_check, url)
+    if not matching:
+        return False
+
+    # VALIDATE ENTIRE GITLAB URL STRING
+    gitlab_api_pattern = "https?://.+/api/v\d/projects/\d/repository/files/.+/raw\?ref=.+"
+    matching = re.match(gitlab_api_pattern, url)
+    if not matching:
+        raise Exception("Gitlab Rest API URL failed validation. Should be of form: {}".format(EXAMPLE))
+
+    return True
+
+
+if __name__ == "__main__":
+    input_url = "http://192.168.85.62/api/v4/projects/4/repository/files/hello_world.sh/raw?ref=master"
+    is_gitlab = is_gitlab_rest_url(input_url)
+    print(is_gitlab)

package/cloudshell/cm/ansible/domain/http_request_service.py

@@ -1,7 +1,66 @@
 import requests
+from requests import Response
+from cloudshell.cm.ansible.domain.gitlab_api_url_validator import is_gitlab_rest_url
 
 
 class HttpRequestService(object):
-    def get_response(self, url, auth):
-        # return requests.get(url, auth=(auth.username, auth.password) if auth else None, stream=True)
-        return requests.get(url, auth=(auth.username, auth.password) if auth else None, stream=True, verify=False)
+    def get_response(self, url, auth, logger=None):
+        """
+        :param url:
+        :param auth:
+        :param logging.Logger logger:
+        :return:
+        """
+        is_gitlab_url = is_gitlab_rest_url(url)
+        if is_gitlab_url:
+            response = self._get_gitlab_response(url, auth, logger)
+            self._validate_response_status_code(response)
+            self._invalidate_gitlab_login_page(response)
+        else:
+            auth = (auth.username, auth.password) if auth else None
+            response = requests.get(url, auth=auth, stream=True, verify=False)
+            self._validate_response_status_code(response)
+            self._invalidate_html(response.content)
+        return response
+
+    @staticmethod
+    def _get_gitlab_response(url, auth, logger=None):
+        """
+        :param url:
+        :param auth:
+        :param logging.Logger logger:
+        :return:
+        """
+        # GITLAB REST API CALL - ADDING TOKEN HEADER
+        if logger:
+            logger.info("downloading script via Gitlab Rest API call: {}".format(url))
+        if auth:
+            headers = {"PRIVATE-TOKEN": auth.password}
+            return requests.get(url, stream=True, verify=False, headers=headers)
+        else:
+            return requests.get(url, stream=True, verify=False)
+
+    @staticmethod
+    def _validate_response_status_code(response):
+        if 200 > response.status_code > 300:
+            raise Exception('Failed to download script file: ' + str(response.status_code) + ' ' + response.reason +
+                            '. Please make sure the URL is valid, and the credentials are correct and necessary.')
+
+    @staticmethod
+    def _is_content_html(content):
+        return content.lstrip('\n\r').lower().startswith('<!doctype html>')
+
+    def _invalidate_html(self, content):
+        if self._is_content_html(content):
+            raise Exception('Failed to download script file: url points to an html file')
+
+    def _invalidate_gitlab_login_page(self, response):
+        """
+
+        :param Response response: requests response object
+        :return:
+        """
+        if self._is_content_html(response.content) and "users/sign_in" in response.url:
+            raise Exception('Authentication failed. Reached Gitlab Login. Gitlab Access Token required.')
+
+

package/cloudshell/cm/ansible/domain/playbook_downloader.py

@@ -1,6 +1,7 @@
 import os
 
 from cloudshell.cm.ansible.domain.cancellation_sampler import CancellationSampler
+from cloudshell.cm.ansible.domain.http_request_service import HttpRequestService
 from file_system_service import FileSystemService
 from logging import Logger
 
@@ -17,6 +18,9 @@ class PlaybookDownloader(object):
     def __init__(self, file_system, zip_service, http_request_service, filename_extractor):
         """
         :param FileSystemService file_system:
+        :param zip_service:
+        :param HttpRequestService http_request_service:
+        :param filename_extractor:
         """
         self.file_system = file_system
         self.zip_service = zip_service
@@ -51,7 +55,7 @@ def _download(self, url, auth, logger, cancel_sampler):
         :return The downloaded file name
         """
         logger.info('Downloading file from \'%s\' ...'%url)
-        response = self.http_request_service.get_response(url, auth)
+        response = self.http_request_service.get_response(url, auth, logger)
         file_name = self.filename_extractor.get_filename(response)
 
         with self.file_system.create_file(file_name) as file:

package/requirements.txt

@@ -3,4 +3,3 @@ cloudshell-shell-core>=3.1.0,<3.2.0
 requests==2.14.2
 pywinrm==0.2.2
 paramiko==2.1.2
-