Relax requirement for specification of Git provider for Repos (#385)

* Relax requirement for specification of Git provider for Repos

Before this PR, the `--provider` option was required, but in many cases we can guess the
provider ID from the URL (for hosted providers as Gitlab, GitHub, etc.) - this PR is
making `--provider` option not required, but if we can't guess Git provider, it will error
out.

* addressing review comments

* add support for AWS CodeCommit repositories

Author: alexott

databricks_cli/repos/cli.py

@@ -52,7 +52,9 @@ def list_repos_cli(api_client, path_prefix, next_page_token):
 @click.command(context_settings=CONTEXT_SETTINGS,
                short_help='Create a repo and link it to the given remote Git repo')
 @click.option('--url', required=True, help="URL of the remote Git repo")
-@click.option('--provider', required=True, help="Git provider (case insensitive)")
+@click.option('--provider',
+              help="Git provider (case insensitive). Required if it couldn't be detected from "
+                   "host name")
 @click.option('--path', help="Desired workspace path of the repo object")
 @debug_option
 @profile_option

databricks_cli/sdk/service.py

@@ -24,6 +24,9 @@
 # limitations under the License.
 #
 import os
+import re
+
+from six.moves.urllib.parse import urlparse
 
 
 class JobsService(object):
@@ -1075,10 +1078,35 @@ def stop(self, pipeline_id=None, headers=None):
 
         return self.client.perform_query('POST', '/pipelines/{pipeline_id}/stop'.format(pipeline_id=pipeline_id),
                                          data=_data, headers=headers)
+
+
 class ReposService(object):
+    __git_providers__ = {
+        "github.com":    "gitHub",
+        "dev.azure.com": "azureDevOpsServices",
+        "gitlab.com":    "gitLab",
+        "bitbucket.org": "bitbucketCloud"
+    }
+    __aws_code_commit_regexp__ = re.compile(r"^git-codecommit\.[^.]+\.amazonaws.com$")
+
     def __init__(self, client):
         self.client = client
 
+    @staticmethod
+    def detect_repo_provider(url):
+        provider = None
+        try:
+            netloc = urlparse(url).netloc
+            idx = netloc.rfind("@")
+            if idx != -1:
+                netloc = netloc[(idx + 1):]
+            provider = ReposService.__git_providers__.get(netloc.lower())
+            if provider is None and ReposService.__aws_code_commit_regexp__.match(netloc):
+                provider = "awsCodeCommit"
+        except:
+            pass
+        return provider
+
     def list_repos(self, path_prefix=None, next_page_token=None, headers=None):
         _data = {}
         if path_prefix is not None:
@@ -1090,27 +1118,35 @@ def list_repos(self, path_prefix=None, next_page_token=None, headers=None):
     def get_repo(self, id, headers=None):
         _data = {}
 
-        return self.client.perform_query('GET', '/repos/{id}'.format(id=id), data=_data, headers=headers)
+        return self.client.perform_query('GET', '/repos/{id}'.format(id=id),
+                                         data=_data, headers=headers)
 
     def update_repo(self, id, branch=None, tag=None, headers=None):
         _data = {}
         if branch is not None:
             _data['branch'] = branch
         if tag is not None:
             _data['tag'] = tag
-        return self.client.perform_query('PATCH', '/repos/{id}'.format(id=id), data=_data, headers=headers)
+        return self.client.perform_query('PATCH', '/repos/{id}'.format(id=id),
+                                         data=_data, headers=headers)
 
     def create_repo(self, url, provider, path=None, headers=None):
         _data = {}
         if url is not None:
             _data['url'] = url
+        if provider is None or provider.trim() == "":
+            provider = self.detect_repo_provider(url)
         if provider is not None:
             _data['provider'] = provider
+        else:
+            raise ValueError("The Git provider parameter wasn't specified and we can't detect it "
+                             "from URL. Please pass 'provider' option")
         if path is not None:
             _data['path'] = path
         return self.client.perform_query('POST', '/repos', data=_data, headers=headers)
 
     def delete_repo(self, id, headers=None):
         _data = {}
 
-        return self.client.perform_query('DELETE', '/repos/{id}'.format(id=id), data=_data, headers=headers)
+        return self.client.perform_query('DELETE', '/repos/{id}'.format(id=id),
+                                         data=_data, headers=headers)

tests/repos/test_api.py

@@ -88,3 +88,9 @@ def test_get_id_other_error(self, repos_api_with_ws_service):
             side_effect=requests.exceptions.HTTPError(response=response))
         with pytest.raises(RuntimeError):
             repos_api_with_ws_service.get_repo_id(TEST_PATH)
+
+
+class TestCreateRepo(object):
+    def test_get(self, repos_api_with_ws_service):
+        with pytest.raises(ValueError):
+            repos_api_with_ws_service.create("https://github1.com/user/repo.git", None, None)

tests/repos/test_cli.py

@@ -75,6 +75,18 @@ def test_create_validation(repos_api_mock):
     )
 
 
+@provide_conf
+def test_create_validation_partial(repos_api_mock):
+    runner = CliRunner()
+    runner.invoke(cli.create_repo_cli,
+                  ["--url", TEST_URL])
+    repos_api_mock.create.assert_called_once_with(
+        TEST_URL,
+        None,
+        None,
+    )
+
+
 @provide_conf
 def test_get_validation(repos_api_mock):
     runner = CliRunner()

tests/sdk/test_api_client.py

@@ -26,6 +26,7 @@
 import requests
 import requests_mock
 
+from databricks_cli.sdk import ReposService
 from databricks_cli.sdk.api_client import ApiClient
 
 
@@ -123,3 +124,10 @@ def test_api_client_url_parsing():
     # databricks_cli.configure.cli
     client = ApiClient(host='http://databricks.com')
     assert client.get_url('') == 'http://databricks.com/api/2.0'
+
+
+def test_repos_provider_detection():
+    assert ReposService.detect_repo_provider("https://github.com/org/repo.git") == "gitHub"
+    assert ReposService.detect_repo_provider(
+        "https://git-codecommit.us-east-2.amazonaws.com/v1/repos/MyDemoRepo") == "awsCodeCommit"
+    assert ReposService.detect_repo_provider("https://github1.com/org/repo.git") is None