Add the pipelines list command to allow users to view all pipelines visible to them (#333)

This PR adds the pipelines list command to the CLI, which allows users to view metadata about all the pipelines that they own. The list() API is a paginated request, and this command continually requests new pages until they've been exhausted before displaying them.

This API also includes updates to the sdk/api_client.py and sdk/service.py files, which come from newly auto-generated versions of the files.

Author: aakash-db

databricks_cli/pipelines/api.py

@@ -60,6 +60,27 @@ def delete(self, pipeline_id, headers=None):
     def get(self, pipeline_id, headers=None):
         return self.client.get(pipeline_id, headers)
 
+    def list(self, headers=None):
+        def call(page_token=None, max_results=None, order_by=None):
+            _data = {}
+            if page_token:
+                _data["pagination.page_token"] = page_token
+            if max_results:
+                _data["pagination.max_results"] = max_results
+            if order_by:
+                _data["pagination.order_by"] = order_by
+
+            return self.client.client.perform_query(
+                'GET', '/pipelines', data=_data, headers=headers)
+
+        response = call()
+        pipelines = response["statuses"]
+
+        while "next_page_token" in response["pagination"]:
+            response = call(page_token=response["pagination"]["next_page_token"])
+            pipelines.extend(response["statuses"])
+        return pipelines
+
     def reset(self, pipeline_id, headers=None):
         self.client.reset(pipeline_id, headers)
 

databricks_cli/pipelines/cli.py

@@ -165,6 +165,16 @@ def get_cli(api_client, pipeline_id):
     click.echo(pretty_format(PipelinesApi(api_client).get(pipeline_id)))
 
 
+@click.command(context_settings=CONTEXT_SETTINGS,
+               short_help='Gets a delta pipeline\'s current spec and status')
+@debug_option
+@profile_option
+@pipelines_exception_eater
+@provide_api_client
+def list_cli(api_client):
+    click.echo(pretty_format(PipelinesApi(api_client).list()))
+
+
 @click.command(context_settings=CONTEXT_SETTINGS,
                short_help='Resets a delta pipeline so data can be reprocessed from scratch')
 @click.option('--pipeline-id', default=None, type=PipelineIdClickType(),
@@ -318,6 +328,7 @@ def pipelines_group():  # pragma: no cover
 pipelines_group.add_command(deploy_cli, name='deploy')
 pipelines_group.add_command(delete_cli, name='delete')
 pipelines_group.add_command(get_cli, name='get')
+pipelines_group.add_command(list_cli, name='list')
 pipelines_group.add_command(reset_cli, name='reset')
 pipelines_group.add_command(run_cli, name='run')
 pipelines_group.add_command(stop_cli, name='stop')

databricks_cli/sdk/api_client.py

@@ -123,10 +123,10 @@ def perform_query(self, method, path, data = {}, headers = None):
             if method == 'GET':
                 translated_data = {k: _translate_boolean_to_query_param(data[k]) for k in data}
                 resp = self.session.request(method, self.url + path, params = translated_data,
-                    verify = self.verify, headers = headers)
+                                            verify = self.verify, headers = headers)
             else:
                 resp = self.session.request(method, self.url + path, data = json.dumps(data),
-                    verify = self.verify, headers = headers)
+                                            verify = self.verify, headers = headers)
         try:
             resp.raise_for_status()
         except requests.exceptions.HTTPError as e: