MB-70103 Add backup-service repo-add command

The names of a few flags have been changed:

--s3-force-path-style has been changed to --cloud-force-path-style to
make it consistent with the body parameter.

--backup-archive has been changed to --location to avoid confusion with
the repo-archive command and to allow for the term "archive" to be
removed in the future without breaking backwards compatibility.

Duplicates the functionality of backup-service repository --add which
will be deprecated.

Change-Id: I3f462150463907540ab35c2efc55ccbd77a37a02
Reviewed-on: https://review.couchbase.org/c/couchbase-cli/+/238411
Reviewed-by: Safian Ali <safian.ali@couchbase.com>
Tested-by: Build Bot <build@couchbase.com>

Author: lubomarinski

cbmgr.py

@@ -1,5 +1,6 @@
 """A Couchbase  CLI subcommand"""
 
+import argparse
 import dataclasses
 import getpass
 import inspect
@@ -6997,11 +6998,12 @@ def __init__(self):
         self.repository_cmd = BackupServiceRepository(self.subparser)
         self.repo_list_cmd = BackupServiceRepoList(self.subparser)
         self.repo_get_cmd = BackupServiceRepoGet(self.subparser)
+        self.repo_add_cmd = BackupServiceRepoAdd(self.subparser)
         self.plan_cmd = BackupServicePlan(self.subparser)
         self.nodeThreads_cmd = BackupServiceNodeThreadsMap(self.subparser)
 
     def execute(self, opts):
-        subcommands = ['settings', 'repository', 'repo-list', 'repo-get', 'plan', 'node-threads']
+        subcommands = ['settings', 'repository', 'repo-list', 'repo-get', 'repo-add', 'plan', 'node-threads']
 
         if opts.sub_cmd is None or opts.sub_cmd not in subcommands:
             _exit_if_errors([f'<subcommand> must be one of {subcommands}'])
@@ -7014,6 +7016,8 @@ def execute(self, opts):
             self.repo_list_cmd.execute(opts)
         elif opts.sub_cmd == 'repo-get':
             self.repo_get_cmd.execute(opts)
+        elif opts.sub_cmd == 'repo-add':
+            self.repo_add_cmd.execute(opts)
         elif opts.sub_cmd == 'plan':
             self.plan_cmd.execute(opts)
         elif opts.sub_cmd == 'node-threads':
@@ -7205,6 +7209,32 @@ def human_friendly_print_repositories(repositories_map):
                   f" {healthy!s:<7}| {repository['repo']}")
 
 
+def check_cloud_params(location: str, opts: argparse.Namespace) -> Optional[List[str]]:
+    """Checks that inside kwargs there is a valid set of parameters to add a cloud repository
+    Args:
+        location (str): The location to use for the repository.
+        opts (argparse.Namespace): The options dict containing the cloud params.
+    """
+    if not location.startswith('s3://') and not location.startswith('az://') and not location.startswith('gs://'):
+        return None
+
+    creds_name = opts.cloud_credentials_name
+    region = opts.cloud_region
+    creds_id = opts.cloud_credentials_id
+    creds_key = opts.cloud_credentials_key
+    staging_dir = opts.cloud_staging_dir
+
+    if (creds_name and (creds_id or creds_key)) or (not creds_name and not (creds_id or creds_key)):
+        return ['must provide either --cloud-credentials-name or --cloud-credentials-key and '
+                '--cloud-credentials-id']
+    if not staging_dir:
+        return ['--cloud-staging-dir is required']
+    if not creds_name and not region:
+        return ['--cloud-credentials-region is required']
+
+    return None
+
+
 class BackupServiceRepoList:
     """List the backup repositories.
 
@@ -7283,6 +7313,93 @@ def get_description():
         return 'Retrieves a repository from the backup service'
 
 
+class BackupServiceRepoAdd:
+    """Adds a new active repository identified by a repository ID and that uses a plan as base.
+    """
+
+    def __init__(self, subparser):
+        """setup the parser"""
+        self.rest = None
+        repository_parser = subparser.add_parser('repo-add', help='Adds a new active repository', add_help=False,
+                                                 allow_abbrev=False)
+
+        group = repository_parser.add_argument_group('Backup service repository configuration')
+        group.add_argument('--id', metavar='<id>', help='The repository id', required=True)
+        group.add_argument('--plan', metavar='<plan_name>', help='The plan to use as base for the repository',
+                           required=True)
+        group.add_argument('--location', metavar='<location>', help='The location to store the backups in',
+                           required=True)
+        group.add_argument('--bucket-name', metavar='<bucket_name>', help='The bucket to backup')
+
+        # the cloud arguments are given the own group so that the short help is a bit more readable
+        cloud_group = repository_parser.add_argument_group('Backup repository cloud arguments')
+        cloud_group.add_argument('--cloud-credentials-name', metavar='<name>',
+                                 help='The stored clouds credential name to use for the new repository')
+        cloud_group.add_argument('--cloud-staging-dir', metavar='<path>', help='The path to the staging directory')
+        cloud_group.add_argument('--cloud-credentials-id', metavar='<id>',
+                                 help='The ID to use to communicate with the object store')
+        cloud_group.add_argument('--cloud-credentials-key', metavar='<key>',
+                                 help='The key to use to communicate with the object store')
+        cloud_group.add_argument('--cloud-credentials-refresh-token', metavar='<token>',
+                                 help='Used to refresh oauth2 tokens when accessing remote storage')
+        cloud_group.add_argument('--cloud-region', metavar='<region>',
+                                 help='The region for the object store')
+        cloud_group.add_argument('--cloud-endpoint', metavar='<endpoint>',
+                                 help='Overrides the default endpoint used to communicate with the cloud provider. '
+                                      'Use for object store compatible third party solutions')
+        cloud_group.add_argument('--cloud-force-path-style', action='store_true',
+                                 help='When using S3 or S3 compatible storage it will use the old path style.')
+
+    @rest_initialiser(version_check=True, enterprise_check=True, cluster_init_check=True)
+    def execute(self, opts):
+        """Run the backup-service repo-add subcommand"""
+        _exit_if_errors(check_cloud_params(opts.location, opts))
+
+        request_body = {
+            "plan": opts.plan,
+            "archive": opts.location,
+        }
+
+        if opts.bucket_name is not None:
+            request_body["bucket_name"] = opts.bucket_name
+
+        if opts.cloud_credentials_name is not None:
+            request_body["cloud_credential_name"] = opts.cloud_credentials_name
+
+        if opts.cloud_credentials_id is not None:
+            request_body["cloud_credentials_id"] = opts.cloud_credentials_id
+
+        if opts.cloud_credentials_key is not None:
+            request_body["cloud_credentials_key"] = opts.cloud_credentials_key
+
+        if opts.cloud_credentials_refresh_token is not None:
+            request_body["cloud_credentials_refresh_token"] = opts.cloud_credentials_refresh_token
+
+        if opts.cloud_region is not None:
+            request_body["cloud_region"] = opts.cloud_region
+
+        if opts.cloud_endpoint is not None:
+            request_body["cloud_endpoint"] = opts.cloud_endpoint
+
+        if opts.cloud_staging_dir is not None:
+            request_body["cloud_staging_dir"] = opts.cloud_staging_dir
+
+        if opts.cloud_force_path_style is not None:
+            request_body["cloud_force_path_style"] = opts.cloud_force_path_style
+
+        _, errors = self.rest.add_backup_active_repository(opts.id, request_body)
+        _exit_if_errors(errors)
+        _success('Added repository')
+
+    @staticmethod
+    def get_man_page_name():
+        return get_doc_page_name("couchbase-cli-backup-service-repo-add")
+
+    @staticmethod
+    def get_description():
+        return 'Adds a new active repository'
+
+
 class BackupServiceRepository:
     """This command manages backup services repositories.
 

docs/modules/cli/pages/_partials/cbcli/nav.adoc

@@ -4,6 +4,7 @@
  ** xref:cli:cbcli/couchbase-cli-backup-service.adoc[backup-service]
  ** xref:cli:cbcli/couchbase-cli-backup-service-nodes-threads-map.adoc[backup-service nodes-threads-map]
  ** xref:cli:cbcli/couchbase-cli-backup-service-plan.adoc[backup-service plan]
+ ** xref:cli:cbcli/couchbase-cli-backup-service-repo-add.adoc[backup-service repo-add]
  ** xref:cli:cbcli/couchbase-cli-backup-service-repo-get.adoc[backup-service repo-get]
  ** xref:cli:cbcli/couchbase-cli-backup-service-repo-list.adoc[backup-service repo-list]
  ** xref:cli:cbcli/couchbase-cli-backup-service-repository.adoc[backup-service repository]

docs/modules/cli/pages/cbcli/couchbase-cli-backup-service-repo-add.adoc

@@ -0,0 +1,190 @@
+= couchbase-cli-backup-service-repo-add(1)
+:description: Add a new backup service repository
+ifndef::doctype-manpage[:doctitle: backup-service repo-add]
+
+ifdef::doctype-manpage[]
+== NAME
+
+couchbase-cli-backup-service-repo-add -
+endif::[]
+Add a new backup service repository
+
+== SYNOPSIS
+
+[verse]
+_couchbase-cli backup-service_ [--cluster <url>] [--username <user>]
+    [--password <password>] [--client-cert <path>]
+    [--client-cert-password <password>] [--client-key <path>]
+    [--client-key-password <password>] _repo-add_ [--id <id>]
+    [--plan <plan_name>] [--location <location>] [--bucket-name <bucket_name>]
+    [--cloud-credentials-name <name>] [--cloud-staging-dir <path>]
+    [--cloud-credentials-id <id>] [--cloud-credentials-key <key>]
+    [--cloud-credentials-refresh-token <token>] [--cloud-region <region>]
+    [--cloud-endpoint <endpoint>] [--cloud-force-path-style]
+
+== DESCRIPTION
+
+Adds a new active backup repository to the backup service. The repository
+will be created with the specified ID and will use the given plan as its base
+configuration. The plan defines the backup schedules and which services to
+back up. Backups will be stored at the specified location.
+
+This command also supports creating cloud backup repositories that store data
+directly in object storage such as S3, Azure, GCP or compatible stores.
+
+== OPTIONS
+
+--id <id>::
+    The unique identifier for the new repository. This argument is required.
+
+--plan <plan_name>::
+    The name of the plan to use as the base configuration for this repository.
+    The plan defines backup schedules and which services to include. This
+    argument is required.
+
+--location <location>::
+    The location where backups will be stored. This location should be
+    accessible by all backup nodes. This argument is required.
+
+--bucket-name <bucket_name>::
+    Optionally specify a single bucket to back up. If not provided, all
+    buckets will be included in the backup.
+
+== CLOUD OPTIONS
+
+The following options are used when creating a cloud backup repository that
+stores data in object storage.
+
+--cloud-credentials-name <name>::
+    The name of a stored cloud credential set to use for this repository.
+    If specified, you do not need to provide `--cloud-credentials-id` and
+    `--cloud-credentials-key`.
+
+--cloud-staging-dir <path>::
+    The path to a staging directory used temporarily during cloud backups.
+    This path must be accessible by all backup nodes and should have enough
+    space for roughly 10% of the dataset size. Required when using cloud
+    storage.
+
+--cloud-credentials-id <id>::
+    The ID used to authenticate with the object store.
+
+--cloud-credentials-key <key>::
+    The key used to authenticate with the object store.
+
+--cloud-credentials-refresh-token <token>::
+    A refresh token used to obtain new OAuth2 tokens when accessing remote
+    storage.
+
+--cloud-region <region>::
+    The region where the object store bucket is located (e.g., `us-east-1`).
+
+--cloud-endpoint <endpoint>::
+    Overrides the default endpoint used to communicate with the cloud provider.
+    Use this for third-party object store solutions.
+
+--cloud-force-path-style::
+    When using S3 or S3-compatible storage, use the legacy path-style URLs
+    instead of virtual-hosted-style URLs. Some S3-compatible stores require
+    this option.
+
+include::{partialsdir}/cbcli/part-common-options.adoc[]
+
+include::{partialsdir}/cbcli/part-host-formats.adoc[]
+
+include::{partialsdir}/cbcli/part-certificate-authentication.adoc[]
+
+== EXAMPLES
+
+=== Adding a basic repository
+
+To add a new backup repository with ID 'weekly-backup' using the '_weekly'
+plan and storing backups in '/backup/data', run the following command.
+
+----
+$ couchbase-cli backup-service -c 127.0.0.1:8091 -u Administrator -p password \
+  repo-add --id weekly-backup --plan _weekly --location /backup/data
+----
+
+In the command above we are adding a new repository with name `weekly-backup`
+that is using a base plan `_weekly`. The base plan defines the schedules of
+the tasks that the repository will run as well as what services it will backup.
+The location is where the backups will be stored. This location is equivalent
+to a cbbackupmgr archive. Also `cbbackupmgr` should *not* be run on the
+archives managed by the service directly.
+
+=== Adding a repository for a specific bucket
+
+If you want a repository that only backs up one bucket or want different backup
+schedules for each bucket, this can be achieved by using the `--bucket-name`
+argument to specify which bucket the repository should backup.
+
+----
+$ couchbase-cli backup-service -c 127.0.0.1:8091 -u Administrator -p password \
+  repo-add --id beer-backup --plan _daily --location /backup/beer \
+  --bucket-name beer-sample
+----
+
+=== Adding a cloud backup repository
+
+The service supports creating cloud backup repositories. These are repositories
+that backup directly to object store. Currently supported object stores include
+S3, Azure, GCP, and S3-compatible stores. To create a cloud repository you will
+need to supply some additional details, as illustrated in the example below.
+
+----
+$ couchbase-cli backup-service -c 127.0.0.1:8091 -u Administrator -p password \
+  repo-add --id cloud-backup --plan _daily --location s3://bucket/archive \
+  --cloud-staging-dir /backup/staging --cloud-credentials-id id \
+  --cloud-credentials-key key --cloud-region us-east-1
+----
+
+In the command above we can see that the archive supplied for cloud starts
+with the schema `s3://` followed by the bucket name, after which the path to
+the archive in S3 must be given. A *staging directory* must also be supplied.
+This is a location where cbbackupmgr will temporarily store data whilst doing
+cloud backups. This path must be available on all backup nodes and should have
+space for roughly 10% of the data set size as reported by the UI. In the command
+above we have also supplied the cloud credential ID, key and region. These are
+the details that will be used to communicate with S3.
+
+=== Using stored cloud credentials
+
+Credentials can be stored and reused in the backup service. If you already have
+the correct set of credentials stored you can use `--cloud-credentials-name`
+instead of providing the ID, key and region flags.
+
+----
+$ couchbase-cli backup-service -c 127.0.0.1:8091 -u Administrator -p password \
+  repo-add --id cloud-backup --plan _weekly --location s3://bucket/archive \
+  --cloud-staging-dir /backup/staging --cloud-credentials-name creds
+----
+
+=== Using S3-compatible storage
+
+For S3-compatible stores you can use the `--cloud-endpoint` argument to override
+the endpoint used to communicate with S3 and point it to your storage solution
+address. Some S3-compatible storages only support the old S3 path styles; to use
+those supply the `--cloud-force-path-style` argument.
+
+----
+$ couchbase-cli backup-service -c 127.0.0.1:8091 -u Administrator -p password \
+  repo-add --id minio-backup --plan _daily --location s3://bucket/backups \
+  --cloud-staging-dir /backup/staging --cloud-credentials-id id \
+  --cloud-credentials-key key --cloud-endpoint http://localhost:9000 \
+  --cloud-force-path-style
+----
+
+== ENVIRONMENT AND CONFIGURATION VARIABLES
+
+include::{partialsdir}/cbcli/part-common-env.adoc[]
+
+== SEE ALSO
+
+man:couchbase-cli-backup-service[1],
+man:couchbase-cli-backup-service-repo-get[1],
+man:couchbase-cli-backup-service-repo-list[1],
+man:couchbase-cli-backup-service-plan[1]
+
+include::{partialsdir}/cbcli/part-footer.adoc[]
+

docs/modules/cli/pages/cbcli/couchbase-cli-backup-service-repo-get.adoc

@@ -123,6 +123,7 @@ include::{partialsdir}/cbcli/part-common-env.adoc[]
 
 man:couchbase-cli-backup-service[1],
 man:couchbase-cli-backup-service-repo-list[1],
+man:couchbase-cli-backup-service-repo-add[1]
 
 include::{partialsdir}/cbcli/part-footer.adoc[]
 

docs/modules/cli/pages/cbcli/couchbase-cli-backup-service-repo-list.adoc

@@ -112,7 +112,8 @@ include::{partialsdir}/cbcli/part-common-env.adoc[]
 == SEE ALSO
 
 man:couchbase-cli-backup-service[1],
-man:couchbase-cli-backup-service-repo-get[1]
+man:couchbase-cli-backup-service-repo-get[1],
+man:couchbase-cli-backup-service-repo-add[1]
 
 include::{partialsdir}/cbcli/part-footer.adoc[]
 

docs/modules/cli/pages/cbcli/couchbase-cli-backup-service-repository.adoc

@@ -158,8 +158,7 @@ In the command above we are adding a new repository with name `new-repository` t
 base profile `_weekly`. The base profile defines the schedules of the tasks that the
 repository will run as well as what services it will backup. Finally the backup archive is the
 location where the backups will stored. This location is equivalent to a cbbackupmgr
-archive. Two repository should *not* use the same archive. Also `cbbackupmgr` should *not*
-be run on the archives managed by the service directly.
+archive. Also `cbbackupmgr` should *not* be run on the archives managed by the service directly.
 
 If you want an repository that only backs up one bucket or want different backup schedules
 for each bucket this can be achieved by using the `--bucket-name` argument to specify
@@ -192,7 +191,7 @@ by the UI. In the command above we have also supplied the cloud credential ID, k
 region. This are the details that will be used to communicate with S3. Note that
 credentials can be stored and reused in the backup service so if you already have
 the correct set of credentials stored you can replace the id, key and region flags by
-`--cloud-credential-name` and give the name of the credential set you want to re-use.
+`--cloud-credentials-name` and give the name of the credential set you want to re-use.
 
 Finally, for S3 compatible stores you can use the `--cloud-endpoint` argument to override
 the endpoint use to communicate with S3 and point it to your storage solution address.