Merge branch 'neo' into master

* neo:
  MB-55715 Add support for configuring history_size_warning alert
  MB-55692 Add support for editing collections history field
  MB-55677 Fix the eshell error with -d switch

Change-Id: I62cabb4a9f8cca8fcb597915487559bb4c8422b8

Author: yonsken

cbmgr.py

@@ -2342,12 +2342,14 @@ def __init__(self):
                            help="Alert when the disk analyzer gets stuck")
         group.add_argument("--alert-memory-threshold", dest="alert_memory_threshold",
                            action="store_true", help="Alert when system memory usage exceeds threshold")
+        group.add_argument("--alert-bucket-history-size", dest="alert_bucket_history_size",
+                           action="store_true", help="Alert when history size for a bucket reaches 90%%")
 
     @rest_initialiser(cluster_init_check=True, version_check=True)
     def execute(self, opts):
         if opts.enabled == "1":
             if opts.email_recipients is None:
-                _exit_if_errors(["--email-recipient must be set when email alerts are enabled"])
+                _exit_if_errors(["--email-recipients must be set when email alerts are enabled"])
             if opts.email_sender is None:
                 _exit_if_errors(["--email-sender must be set when email alerts are enabled"])
             if opts.email_host is None:
@@ -2390,6 +2392,8 @@ def execute(self, opts):
             alerts.append('disk_usage_analyzer_stuck')
         if opts.alert_memory_threshold:
             alerts.append('memory_threshold')
+        if opts.alert_bucket_history_size:
+            alerts.append('history_size_warning')
 
         enabled = "true"
         if opts.enabled == "0":
@@ -4764,6 +4768,8 @@ def __init__(self):
                            help="List all of the scopes in the bucket")
         group.add_argument("--create-collection", dest="create_collection", metavar="<collection>", default=None,
                            help="The path to the collection to make")
+        group.add_argument("--edit-collection", dest="edit_collection", metavar="<collection>", default=None,
+                           help="The path to the collection to edit")
         group.add_argument("--drop-collection", dest="drop_collection", metavar="<collection>", default=None,
                            help="The path to the collection to remove")
         group.add_argument("--list-collections", dest="list_collections", metavar="<scope_list>", default=None,
@@ -4776,12 +4782,12 @@ def __init__(self):
 
     @rest_initialiser(cluster_init_check=True, version_check=True)
     def execute(self, opts):
-        cmds = [opts.create_scope, opts.drop_scope, opts.list_scopes, opts.create_collection, opts.drop_collection,
-                opts.list_collections]
+        cmds = [opts.create_scope, opts.drop_scope, opts.list_scopes, opts.create_collection, opts.edit_collection,
+                opts.drop_collection, opts.list_collections]
         cmd_total = sum(cmd is not None for cmd in cmds)
 
-        args = "--create-scope, --drop-scope, --list-scopes, --create-collection, --drop-collection, or " \
-               "--list-collections"
+        args = "--create-scope, --drop-scope, --list-scopes, --create-collection, --edit-collection, " \
+               "--drop-collection, or --list-collections"
         if cmd_total == 0:
             _exit_if_errors([f'Must specify one of the following: {args}'])
         elif cmd_total != 1:
@@ -4790,8 +4796,14 @@ def execute(self, opts):
         if opts.max_ttl is not None and opts.create_collection is None:
             _exit_if_errors(["--max-ttl can only be set with --create-collection"])
 
-        if opts.enable_history is not None and opts.create_collection is None:
-            _exit_if_errors(["--enable-history-retention can only be set with --create-collection"])
+        if opts.enable_history is not None and opts.create_collection is None and opts.edit_collection is None:
+            _exit_if_errors(["--enable-history-retention can only be set with --create-collection or "
+                             "--edit-collection"])
+
+        if opts.edit_collection is not None and opts.enable_history is None:
+            _exit_if_errors(["--enable-history-retention should be set with --edit-collection"])
+        if opts.edit_collection is not None and opts.max_ttl is not None:
+            _exit_if_errors(["--max-ttl cannot be set with --edit-collection"])
 
         if opts.create_scope:
             self._create_scope(opts)
@@ -4801,6 +4813,8 @@ def execute(self, opts):
             self._list_scopes(opts)
         if opts.create_collection:
             self._create_collection(opts)
+        if opts.edit_collection:
+            self._edit_collection(opts)
         if opts.drop_collection:
             self._drop_collection(opts)
         if opts.list_collections is not None:
@@ -4828,6 +4842,12 @@ def _create_collection(self, opts):
         _exit_if_errors(errors)
         _success("Collection created")
 
+    def _edit_collection(self, opts):
+        scope, collection = self._get_scope_collection(opts.edit_collection)
+        _, errors = self.rest.edit_collection(opts.bucket, scope, collection, opts.enable_history)
+        _exit_if_errors(errors)
+        _success("Collection edited")
+
     def _drop_collection(self, opts):
         scope, collection = self._get_scope_collection(opts.drop_collection)
         _, errors = self.rest.drop_collection(opts.bucket, scope, collection)

cluster_manager.py

@@ -2347,8 +2347,17 @@ def create_collection(self, bucket, scope, collection, max_ttl, enable_history):
 
         return self._post_form_encoded(url, params)
 
+    def edit_collection(self, bucket, scope, collection, enable_history):
+        url = f'{self.hostname}/pools/default/buckets/{urllib.parse.quote_plus(bucket)}/scopes/' \
+            f'{urllib.parse.quote_plus(scope)}/collections/{urllib.parse.quote_plus(collection)}'
+        params = {}
+        if enable_history:
+            params["history"] = one_zero_boolean_to_string(enable_history)
+
+        return self._patch_form_encoded(url, params)
+
     def drop_collection(self, bucket, scope, collection):
-        url = f'{self.hostname}/pools/default/buckets/{urllib.parse.quote_plus(bucket)}/scopes/'\
+        url = f'{self.hostname}/pools/default/buckets/{urllib.parse.quote_plus(bucket)}/scopes/' \
             f'{urllib.parse.quote_plus(scope)}/collections/{urllib.parse.quote_plus(collection)}'
         return self._delete(url, None)
 
@@ -2504,7 +2513,10 @@ def _get(self, url, params=None):
     @request
     def _post_form_encoded(self, url, params):
         if self.debug:
-            print(f'POST {url} {self._url_encode_params(params)}')
+            if len(params) and not isinstance(params[0], tuple):
+                print(f'POST {url} {params}')
+            else:
+                print(f'POST {url} {self._url_encode_params(params)}')
 
         return self._handle_response(self.session.post(url, auth=(self.username, self.password), data=params,
                                                        verify=self.ca_cert, timeout=self.timeout, headers=self.headers))
@@ -2517,6 +2529,21 @@ def _post_json(self, url, params):
         return self._handle_response(self.session.post(url, auth=(self.username, self.password), json=params,
                                                        verify=self.ca_cert, timeout=self.timeout, headers=self.headers))
 
+    @request
+    def _patch_form_encoded(self, url, params):
+        if self.debug:
+            print(f'PATCH {url} {self._url_encode_params(params)}')
+
+        return self._handle_response(
+            self.session.patch(
+                url, auth=(
+                    self.username,
+                    self.password),
+                data=params,
+                verify=self.ca_cert,
+                timeout=self.timeout,
+                headers=self.headers))
+
     @request
     def _patch_json(self, url, params):
         if self.debug:

docs/modules/cli/pages/cbcli/couchbase-cli-collection-manage.adoc

@@ -16,9 +16,9 @@ _couchbase-cli collection-manage_ [--cluster <url>] [--username <user>]
     [--password <password>] [--client-cert <path>] [--client-cert-password <password>]
     [--client-key <path>] [--client-key-password <password>] [--bucket <bucket>]
     [--create-scope <scope>] [--drop-scope <scope>] [--list-scopes]
-    [--create-collection <collection>] [--drop-collection <collection>]
-    [--list-collections [<scope_list>]] [--max-ttl <seconds>]
-    [--enable-history-retention <0|1>]
+    [--create-collection <collection>] [--edit-collection <collection>]
+    [--drop-collection <collection>] [--list-collections [<scope_list>]]
+    [--max-ttl <seconds>] [--enable-history-retention <0|1>]
 
 == DESCRIPTION
 
@@ -58,6 +58,11 @@ include::{partialsdir}/cbcli/part-common-options.adoc[]
   format (scope.collection), specifying the scope in which the
   collection and the name to be created.
 
+--edit-collection <collection>::
+  Edits the collection in the scope. The option takes a path in dot
+  format (scope.collection), specifying the collection in a scope which is
+  to be edited.
+
 --drop-collection <collection>::
   Removes the collection from the scope. The option takes a path in
   dot format (scope.collection), specifying the scope from which
@@ -73,12 +78,14 @@ include::{partialsdir}/cbcli/part-common-options.adoc[]
   collection, in seconds. If enabled and a document is mutated with
   no TTL or a TTL greater than than the maximum, its TTL will be set
   to the maximum TTL. Setting this option to 0 disables the use of
-  max-TTL, and the largest TTL that is allowed is 2147483647.
+  max-TTL, and the largest TTL that is allowed is 2147483647. This setting
+  cannot be edited after a collection has been created.
 
 --enable-history-retention <0|1>::
   Specifies whether or not the document history retention should be enabled for
   the collection. To enable the document history retention, set this option to
   "1". To disable the document history retention, set this option to "0".
+  This setting can be edited after a collection has been created.
 
 
 include::{partialsdir}/cbcli/part-host-formats.adoc[]
@@ -98,6 +105,13 @@ furniture bucket.
   $ couchbase-cli collection-manage -c 192.168.1.5 -u Administrator \
    -p password --bucket furniture --create-collection chairs.couches
 
+To edit the collection called "couches" in the chairs scope in the furniture
+bucket.
+
+  $ couchbase-cli collection-manage -c 192.168.1.5 -u Administrator \
+   -p password --bucket furniture --edit-collection chairs.couches
+   --enable-history-retention 1
+
 To list all of the collections in the chairs and tables scopes in the furniture
 bucket.
 

docs/modules/cli/pages/cbcli/couchbase-cli-setting-alert.adoc

@@ -24,7 +24,7 @@ _couchbase-cli setting-alert_ [--cluster <url>] [--username <user>] [--password
     [--alert-meta-overhead] [--alert-meta-oom] [--alert-write-failed]
     [--alert-audit-msg-dropped] [--alert-indexer-max-ram]
     [--alert-timestamp-drift-exceeded] [--alert-node-time] [--alert-disk-analyzer]
-    [--alert-memory-threshold]
+    [--alert-memory-threshold] [--alert-bucket-history-size]
 
 == DESCRIPTION
 
@@ -140,7 +140,11 @@ include::{partialsdir}/cbcli/part-common-options.adoc[]
 
 --alert-memory-threshold::
   Specifies that an email alert should be sent when any node's system memory
-  usage exceeds a threshold
+  usage exceeds a threshold.
+
+--alert-bucket-history-size::
+  Specifies that an email alert should be sent when the size of history for a
+  bucket reaches 90% of the maximum history size.
 
 include::{partialsdir}/cbcli/part-host-formats.adoc[]
 
@@ -172,7 +176,8 @@ following command below:
    --alert-ip-changed --alert-disk-space --alert-meta-overhead \
    --alert-meta-oom --alert-write-failed --alert-audit-msg-dropped \
    --alert-indexer-max-ram --alert-timestamp-drift-exceeded \
-   --alert-node-time --alert-disk-analyzer --alert-memory-threshold
+   --alert-node-time --alert-disk-analyzer --alert-memory-threshold \
+   --alert-bucket-history-size
 
 To disable email alerts run the following command:
 

test/mock_server.py

@@ -518,6 +518,7 @@ def export_eventing_functions(rest_params=None, server_args=None, path="", endpo
     (r'/pools/default/buckets/\w+/scopes/\w+$', {'DELETE': do_nothing}),
     (r'/pools/default/buckets/\w+/scopes/\w+/collections$', {'POST': do_nothing}),
     (r'/pools/default/buckets/\w+/scopes/\w+/collections/\w+$', {'DELETE': do_nothing}),
+    (r'/pools/default/buckets/\w+/scopes/\w+/collections/\w+$', {'PATCH': do_nothing}),
     (r'/pools/default/trustedCAs/\d+$', {'DELETE': get_by_path}),
     (r'/pools/nodes', {'GET': get_by_path}),
     (r'/settings/analytics', {'POST': do_nothing, 'GET': get_by_path}),

test/test_cli.py

@@ -1102,7 +1102,8 @@ def setUp(self):
                             '--alert-auto-failover-disable', '--alert-ip-changed', '--alert-disk-space',
                             '--alert-meta-overhead', '--alert-meta-oom', '--alert-write-failed',
                             '--alert-audit-msg-dropped', '--alert-indexer-max-ram', '--alert-timestamp-drift-exceeded',
-                            '--alert-communication-issue', '--alert-node-time', '--alert-disk-analyzer']
+                            '--alert-communication-issue', '--alert-node-time', '--alert-disk-analyzer',
+                            '--alert-bucket-history-size']
         self.server_args = {'enterprise': True, 'init': True, 'is_admin': True}
         super(TestSettingAlert, self).setUp()
 
@@ -1112,7 +1113,8 @@ def test_set_all_alerts(self):
                            '_nodes_down%2Cauto_failover_cluster_too_small%2Cauto_failover_disabled%2Cip%2Cdisk%2' +
                            'Coverhead%2Cep_oom_errors%2Cep_item_commit_failed%2Caudit_dropped_events%2Cindexer_ram_' +
                            'max_usage%2Cep_clock_cas_drift_threshold_exceeded%2Ccommunication_issue' +
-                           '%2Ctime_out_of_sync%2Cdisk_usage_analyzer_stuck', 'enabled=false', 'emailEncrypt=false']
+                           '%2Ctime_out_of_sync%2Cdisk_usage_analyzer_stuck%2Chistory_size_warning', 'enabled=false',
+                           'emailEncrypt=false']
 
         self.assertIn('POST:/settings/alerts', self.server.trace)
         self.rest_parameter_match(expected_params)
@@ -1123,7 +1125,7 @@ def test_set_all_alerts_and_email(self):
                            '_nodes_down%2Cauto_failover_cluster_too_small%2Cauto_failover_disabled%2Cip%2Cdisk%2' +
                            'Coverhead%2Cep_oom_errors%2Cep_item_commit_failed%2Caudit_dropped_events%2Cindexer_ram_' +
                            'max_usage%2Cep_clock_cas_drift_threshold_exceeded%2Ccommunication_issue' +
-                           '%2Ctime_out_of_sync%2Cdisk_usage_analyzer_stuck', 'enabled=true',
+                           '%2Ctime_out_of_sync%2Cdisk_usage_analyzer_stuck%2Chistory_size_warning', 'enabled=true',
                            'emailEncrypt=true', 'sender=email2', 'recipients=email1', 'emailUser=emailuser',
                            'emailPass=emailpwd', 'emailHost=emailhost', 'emailPort=3000']
 
@@ -2755,6 +2757,36 @@ def setUp(self):
         self.server_args = {'enterprise': True, 'init': True, 'is_admin': True}
         super(TestCollectionManage, self).setUp()
 
+    def test_missing_cmd(self):
+        self.system_exit_run(self.command + ['--max-ttl', '100'], self.server_args)
+        self.assertIn("Must specify one of the following: --create-scope, --drop-scope, --list-scopes, "
+                      "--create-collection, --edit-collection, --drop-collection, or --list-collections",
+                      self.str_output)
+
+    def test_more_than_one_cmd(self):
+        self.system_exit_run(self.command + ['--create-scope', 'scope_1', '--list-collections'], self.server_args)
+        self.assertIn("Only one of the following may be specified: --create-scope, --drop-scope, --list-scopes, "
+                      "--create-collection, --edit-collection, --drop-collection, or --list-collections",
+                      self.str_output)
+
+    def test_max_ttl_set_with_not_create_collection(self):
+        self.system_exit_run(self.command + ['--list-collections', '--max-ttl', '100'], self.server_args)
+        self.assertIn("--max-ttl can only be set with --create-collection", self.str_output)
+
+    def test_enable_history_set_with_not_create_collection_or_edit_collection(self):
+        self.system_exit_run(self.command + ['--list-collections', '--enable-history-retention', '1'], self.server_args)
+        self.assertIn("--enable-history-retention can only be set with --create-collection or --edit-collection",
+                      self.str_output)
+
+    def test_enable_history_should_be_set_with_edit_collection(self):
+        self.system_exit_run(self.command + ['--edit-collection', 'scope_1.collection_1'], self.server_args)
+        self.assertIn("--enable-history-retention should be set with --edit-collection", self.str_output)
+
+    def test_max_ttl_cannot_be_set_with_edit_collection(self):
+        self.system_exit_run(self.command + ['--edit-collection', 'scope_1.collection_1', "--max-ttl", "100"],
+                             self.server_args)
+        self.assertIn("--max-ttl can only be set with --create-collection", self.str_output)
+
     def test_create_scope(self):
         self.no_error_run(self.command + ['--create-scope', 'scope_1'], self.server_args)
         self.assertIn('POST:/pools/default/buckets/name/scopes', self.server.trace)
@@ -2780,6 +2812,13 @@ def test_create_collection(self):
         expected_params = ['name=collection_1', 'maxTTL=100', 'history=true']
         self.rest_parameter_match(expected_params)
 
+    def test_edit_collection(self):
+        self.no_error_run(self.command + ['--edit-collection', 'scope_1.collection_1', '--enable-history-retention',
+                                          '1'], self.server_args)
+        self.assertIn('PATCH:/pools/default/buckets/name/scopes/scope_1/collections/collection_1', self.server.trace)
+        expected_params = ['history=true']
+        self.rest_parameter_match(expected_params)
+
     def test_delete_collection(self):
         self.no_error_run(self.command + ['--drop-collection', 'scope_1.collection_1'], self.server_args)
         self.assertIn('DELETE:/pools/default/buckets/name/scopes/scope_1/collections/collection_1', self.server.trace)