Support CodeGen for extension creation (#266)

Author: Yalin Li

azdev/help.py

@@ -168,14 +168,16 @@
 
 
 helps['extension create'] = """
-    short-summary: Create a new Azure CLI extension template.
+    short-summary: Create a new Azure CLI extension.
     examples:
-        - name: Scaffold a new CLI extension named 'contoso'.
-          text: azdev extension create contoso
-        - name: Scaffold a new CLI extension with the azure-mgmt-contoso SDK.
+        - name: Generate a new CLI extension named 'contoso' with local or remote azure-rest-api-specs repo.
+          text: azdev extension create contoso --azure-rest-api-specs {azure-rest-api-specs repo path}
+        - name: Generate a new CLI extension named 'contoso' with the default azure-rest-api-specs repo.
           text: >
-            azdev extension create contoso --local-sdk {sdkPath} --operation-name ContosoOperations
-            --client-name ContosoManagementClient --sdk-property contoso_name
+            azdev extension create contoso
+        - name: Generate a new CLI extension named 'contoso' with specified autorest.az release.
+          text: >
+            azdev extension create contoso --use=https://github.com/Azure/autorest.az/releases/download/1.4.0/autorest-az-1.4.0.tgz
 """
 
 

azdev/operations/code_gen.py

@@ -9,14 +9,20 @@
 import json
 import os
 import re
+import subprocess
 
 from knack.log import get_logger
 from knack.prompting import prompt_y_n, prompt
 from knack.util import CLIError
 
+import azdev.utilities.const as const
+
 from azdev.utilities import (
-    pip_cmd, display, heading, COMMAND_MODULE_PREFIX, EXTENSION_PREFIX, get_cli_repo_path, get_ext_repo_paths,
-    find_files)
+    pip_cmd, shell_cmd, display, heading, COMMAND_MODULE_PREFIX, EXTENSION_PREFIX, get_cli_repo_path,
+    get_ext_repo_paths, find_files, require_virtual_env)
+
+from urllib import request, error
+
 
 logger = get_logger(__name__)
 
@@ -55,24 +61,26 @@ def create_module(mod_name='test', display_name=None, display_name_plural=None,
     _display_success_message(COMMAND_MODULE_PREFIX + mod_name, mod_name)
 
 
-def create_extension(ext_name='test', repo_name='azure-cli-extensions',
-                     display_name=None, display_name_plural=None,
-                     required_sdk=None, client_name=None, operation_name=None, sdk_property=None,
-                     not_preview=False, github_alias=None, local_sdk=None):
-    repo_path = None
+def create_extension(ext_name, azure_rest_api_specs=const.GITHUB_SWAGGER_REPO_URL, branch=None, use=None):
+    if not azure_rest_api_specs.startswith('http') and branch:
+        raise CLIError('Cannot specify azure-rest-api-specs repo branch when using local one.')
+    if not branch:
+        branch = json.load(
+            request.urlopen('https://api.github.com/repos/Azure/azure-rest-api-specs')).get('default_branch')
+    require_virtual_env()
     repo_paths = get_ext_repo_paths()
-    repo_path = next((x for x in repo_paths if x.endswith(repo_name)), None)
-
+    repo_path = next(
+        (x for x in repo_paths if x.endswith(const.EXT_REPO_NAME) or x.endswith(const.EXT_REPO_NAME + '\\')), None)
     if not repo_path:
         raise CLIError('Unable to find `{}` repo. Have you cloned it and added '
-                       'with `azdev extension repo add`?'.format(repo_name))
-
-    _create_package(EXTENSION_PREFIX, os.path.join(repo_path, 'src'), True, ext_name, display_name,
-                    display_name_plural, required_sdk, client_name, operation_name, sdk_property, not_preview,
-                    local_sdk)
-    _add_to_codeowners(repo_path, EXTENSION_PREFIX, ext_name, github_alias)
+                       'with `azdev extension repo add`?'.format(const.EXT_REPO_NAME))
+    if not os.path.isdir(repo_path):
+        raise CLIError("Invalid path {} in .azure config.".format(repo_path))
+    swagger_readme_file_path = _get_swagger_readme_file_path(ext_name, azure_rest_api_specs, branch)
+    _generate_extension(ext_name, repo_path, swagger_readme_file_path, use)
+    _add_extension(ext_name, repo_path)
 
-    _display_success_message(EXTENSION_PREFIX + ext_name, ext_name)
+    _display_success_message(ext_name, ext_name)
 
 
 def _display_success_message(package_name, group_name):
@@ -300,3 +308,75 @@ def _create_package(prefix, repo_path, is_ext, name='test', display_name=None, d
         result = pip_cmd('install -e {}'.format(new_package_path), "Installing `{}{}`...".format(prefix, name))
         if result.error:
             raise result.error  # pylint: disable=raising-bad-type
+
+
+def _get_swagger_readme_file_path(ext_name, swagger_repo, branch):
+    swagger_readme_file_path = None
+    if swagger_repo == const.GITHUB_SWAGGER_REPO_URL or \
+            (swagger_repo.startswith('https://') and swagger_repo.endswith('azure-rest-api-specs')):
+        swagger_readme_file_path = '{}/blob/{}/specification/{}/resource-manager'.format(
+            swagger_repo, branch, ext_name)
+        # validate URL
+        try:
+            request.urlopen(swagger_readme_file_path)
+        except error.HTTPError as ex:
+            raise CLIError(
+                'HTTPError: {}\nNo swagger readme file found in this URL: {}'.format(ex.code, swagger_readme_file_path))
+    else:
+        swagger_readme_file_path = os.path.join(swagger_repo, 'specification', ext_name, 'resource-manager')
+        if not os.path.isdir(swagger_readme_file_path):
+            raise CLIError("The path {} does not exist.".format(swagger_readme_file_path))
+    return swagger_readme_file_path
+
+
+# pylint: disable=too-many-statements
+def _generate_extension(ext_name, repo_path, swagger_readme_file_path, use):
+    heading('Start generating extension {}.'.format(ext_name))
+    # check if npm is installed
+    try:
+        shell_cmd('npm --version', stdout=subprocess.DEVNULL, raise_ex=False)
+    except CLIError as ex:
+        raise CLIError('{}\nPlease install npm.'.format(ex))
+    display('Installing autorest...\n')
+    if const.IS_WINDOWS:
+        try:
+            shell_cmd('npm install -g autorest', raise_ex=False)
+        except CLIError as ex:
+            raise CLIError("Failed to install autorest.\n{}".format(ex))
+    else:
+        try:
+            shell_cmd('npm install -g autorest', stderr=subprocess.DEVNULL, raise_ex=False)
+        except CLIError as ex:
+            path = os.environ['PATH']
+            # check if npm is installed through nvm
+            if os.environ.get('NVM_DIR'):
+                raise ex
+            # check if user using specific node version and manually add it to the os env PATH
+            node_version = shell_cmd('node --version', capture_output=True).result
+            if 'node/' + node_version + '/bin' in path:
+                raise ex
+            # create a new directory for npm global installations, to avoid using sudo in installing autorest
+            npm_path = os.path.join(os.environ['HOME'], '.npm-packages')
+            if not os.path.isdir(npm_path):
+                os.mkdir(npm_path)
+            npm_prefix = shell_cmd('npm prefix -g', capture_output=True).result
+            shell_cmd('npm config set prefix ' + npm_path)
+            os.environ['PATH'] = path + ':' + os.path.join(npm_path, 'bin')
+            os.environ['MANPATH'] = os.path.join(npm_path, 'share', 'man')
+            shell_cmd('npm install -g autorest')
+            shell_cmd('npm config set prefix ' + npm_prefix)
+    # update autorest core
+    shell_cmd('autorest --latest')
+    if not use:
+        cmd = 'autorest --az --azure-cli-extension-folder={} {}'.format(repo_path, swagger_readme_file_path)
+    else:
+        cmd = 'autorest --az --azure-cli-extension-folder={} {} --use={}'.format(
+            repo_path, swagger_readme_file_path, use)
+    shell_cmd(cmd, message=True)
+
+
+def _add_extension(ext_name, repo_path):
+    new_package_path = os.path.join(repo_path, 'src', ext_name)
+    result = pip_cmd('install -e {}'.format(new_package_path), "Adding extension `{}`...".format(new_package_path))
+    if result.error:
+        raise result.error

azdev/operations/extensions/__init__.py

@@ -16,13 +16,13 @@
 
 from azdev.utilities import (
     cmd, py_cmd, pip_cmd, display, get_ext_repo_paths, find_files, get_azure_config, get_azdev_config,
-    require_azure_cli, heading, subheading, EXTENSION_PREFIX)
+    require_azure_cli, require_virtual_env, heading, subheading, EXTENSION_PREFIX)
 
 logger = get_logger(__name__)
 
 
 def add_extension(extensions):
-
+    require_virtual_env()
     ext_paths = get_ext_repo_paths()
     all_extensions = find_files(ext_paths, 'setup.py')
 
@@ -47,7 +47,7 @@ def add_extension(extensions):
 
 
 def remove_extension(extensions):
-
+    require_virtual_env()
     ext_paths = get_ext_repo_paths()
     installed_paths = find_files(ext_paths, '*.*-info')
     paths_to_remove = []
@@ -103,7 +103,7 @@ def _collect(path, depth=0, max_depth=3):
 
 def list_extensions():
     from glob import glob
-
+    require_virtual_env()
     azure_config = get_azure_config()
     dev_sources = azure_config.get('extension', 'dev_sources', None)
     dev_sources = dev_sources.split(',') if dev_sources else []
@@ -143,8 +143,8 @@ def _get_sha256sum(a_file):
 
 
 def add_extension_repo(repos):
-
     from azdev.operations.setup import _check_repo
+    require_virtual_env()
     az_config = get_azure_config()
     env_config = get_azdev_config()
     dev_sources = az_config.get('extension', 'dev_sources', None)
@@ -161,7 +161,7 @@ def add_extension_repo(repos):
 
 
 def remove_extension_repo(repos):
-
+    require_virtual_env()
     az_config = get_azure_config()
     env_config = get_azdev_config()
     dev_sources = az_config.get('extension', 'dev_sources', None)
@@ -177,7 +177,7 @@ def remove_extension_repo(repos):
 
 
 def list_extension_repos():
-
+    require_virtual_env()
     az_config = get_azure_config()
     dev_sources = az_config.get('extension', 'dev_sources', None)
     return dev_sources.split(',') if dev_sources else dev_sources
@@ -188,7 +188,7 @@ def update_extension_index(extensions):
     import tempfile
 
     from .util import get_ext_metadata, get_whl_from_url
-
+    require_virtual_env()
     ext_repos = get_ext_repo_paths()
     index_path = next((x for x in find_files(ext_repos, 'index.json') if 'azure-cli-extensions' in x), None)
     if not index_path:
@@ -245,6 +245,7 @@ def update_extension_index(extensions):
 
 
 def build_extensions(extensions, dist_dir='dist'):
+    require_virtual_env()
     ext_paths = get_ext_repo_paths()
     all_extensions = find_files(ext_paths, 'setup.py')
 
@@ -274,8 +275,8 @@ def publish_extensions(extensions, storage_account, storage_account_key, storage
                        dist_dir='dist', update_index=False, yes=False):
     from azure.storage.blob import BlockBlobService
 
+    require_virtual_env()
     heading('Publish Extensions')
-
     require_azure_cli()
 
     # rebuild the extensions

azdev/operations/help/__init__.py

@@ -13,19 +13,16 @@
 import tempfile
 
 from subprocess import check_call, check_output, CalledProcessError
-
-from knack.util import CLIError
-from knack.log import get_logger
-
-from azure.cli.core.extension.operations import list_available_extensions, list_extensions as list_cli_extensions  # pylint: disable=import-error
 from azdev.utilities import (
     display, heading, subheading,
     get_cli_repo_path, get_path_table,
     require_virtual_env
 )
-
 from azdev.utilities.tools import require_azure_cli
 from azdev.operations.extensions import list_extensions as list_dev_cli_extensions
+from knack.util import CLIError
+from knack.log import get_logger
+require_virtual_env()
 
 DOC_MAP_NAME = 'doc_source_map.json'
 HELP_FILE_NAME = '_help.py'
@@ -106,6 +103,7 @@ def generate_cli_ref_docs(output_dir=None, output_type=None, all_profiles=None):
 
 
 def generate_extension_ref_docs(output_dir=None, output_type=None):
+    require_virtual_env()
     # require that azure cli installed
     require_azure_cli()
     output_dir = _process_ref_doc_output_dir(output_dir)
@@ -261,6 +259,7 @@ def _get_profiles():
 
 
 def _warn_if_exts_installed():
+    from azure.cli.core.extension.operations import list_extensions as list_cli_extensions  # pylint: disable=import-error
     cli_extensions, dev_cli_extensions = list_cli_extensions(), list_dev_cli_extensions()
     if cli_extensions:
         _logger.warning("One or more CLI Extensions are installed and will be included in ref doc output.")
@@ -277,6 +276,7 @@ def _get_available_extension_urls():
 
     :return: list of 3-tuples in the form of '(extension_name, extension_file_name, extensions_download_url)'
     """
+    from azure.cli.core.extension.operations import list_available_extensions  # pylint: disable=import-error
     all_pub_extensions = list_available_extensions(show_details=True)
     compatible_extensions = list_available_extensions()
 

azdev/operations/legal.py

@@ -44,8 +44,12 @@ def check_license_headers():
 
     cli_path = get_cli_repo_path()
     all_paths = [cli_path]
-    for path in get_ext_repo_paths():
-        all_paths.append(path)
+    try:
+        ext_repo = get_ext_repo_paths()
+        for path in ext_repo:
+            all_paths.append(path)
+    except CLIError:
+        display("No CLI ext path, running check only on modules")
 
     files_without_header = []
     for path in all_paths:

azdev/params.py

@@ -141,18 +141,22 @@ def load_arguments(self, _):
     with ArgumentsContext(self, 'extension create') as c:
         c.positional('ext_name', help='Name of the extension to create.')
 
-    for scope in ['extension create', 'cli create']:
-        with ArgumentsContext(self, scope) as c:
-            c.argument('github_alias', help='Github alias for the individual who will be the code owner for this package.')
-            c.argument('not_preview', action='store_true', help='Do not create template commands under a "Preview" status.')
-            c.argument('required_sdk', help='Name and version of the underlying Azure SDK that is published on PyPI. (ex: azure-mgmt-contoso==0.1.0).', arg_group='SDK')
-            c.argument('local_sdk', help='Path to a locally saved SDK. Use if your SDK is not available on PyPI.', arg_group='SDK')
-            c.argument('client_name', help='Name of the Python SDK client object (ex: ContosoManagementClient).', arg_group='SDK')
-            c.argument('operation_name', help='Name of the principal Python SDK operation class (ex: ContosoOperations).', arg_group='SDK')
-            c.argument('sdk_property', help='The name of the Python variable that describes the main object name in the SDK calls (i.e.: account_name)', arg_group='SDK')
-            c.argument('repo_name', help='Name of the repo the extension will exist in.')
-            c.argument('display_name', arg_group='Help', help='Description to display in help text.')
-            c.argument('display_name_plural', arg_group='Help', help='Description to display in help text when plural.')
+    with ArgumentsContext(self, 'extension create') as c:
+        c.argument('azure_rest_api_specs', help='The local path or GitHub to azure-rest-api-specs repo.')
+        c.argument('branch', help='The repo branch when using remote azure-rest-api-specs repo. Default: the default branch set in this repo.')
+        c.argument('use', help='The URL for downloading autorest.az tgz file. You can get all releases here: https://github.com/Azure/autorest.az/releases')
+
+    with ArgumentsContext(self, 'cli create') as c:
+        c.argument('github_alias', help='Github alias for the individual who will be the code owner for this package.')
+        c.argument('not_preview', action='store_true', help='Do not create template commands under a "Preview" status.')
+        c.argument('required_sdk', help='Name and version of the underlying Azure SDK that is published on PyPI. (ex: azure-mgmt-contoso==0.1.0).', arg_group='SDK')
+        c.argument('local_sdk', help='Path to a locally saved SDK. Use if your SDK is not available on PyPI.', arg_group='SDK')
+        c.argument('client_name', help='Name of the Python SDK client object (ex: ContosoManagementClient).', arg_group='SDK')
+        c.argument('operation_name', help='Name of the principal Python SDK operation class (ex: ContosoOperations).', arg_group='SDK')
+        c.argument('sdk_property', help='The name of the Python variable that describes the main object name in the SDK calls (i.e.: account_name)', arg_group='SDK')
+        c.argument('repo_name', help='Name of the repo the extension will exist in.')
+        c.argument('display_name', arg_group='Help', help='Description to display in help text.')
+        c.argument('display_name_plural', arg_group='Help', help='Description to display in help text when plural.')
 
     with ArgumentsContext(self, 'cli generate-docs') as c:
         c.argument('all_profiles', action='store_true',

azdev/utilities/const.py

@@ -12,7 +12,9 @@
 COMMAND_MODULE_PREFIX = 'azure-cli-'
 CONFIG_NAME = 'config'
 EXTENSION_PREFIX = 'azext_'
+EXT_REPO_NAME = 'azure-cli-extensions'
 EXT_SECTION = 'extension'
+GITHUB_SWAGGER_REPO_URL = 'https://github.com/Azure/azure-rest-api-specs'
 IS_WINDOWS = sys.platform.lower() in ['windows', 'win32']
 PIP_E_CMD = 'pip install -e '
 PIP_R_CMD = 'pip install -r '