Skip to content

Commit 5aa98d6

Browse files
jamesl33jamesl33
committed
Merge branch 'neo' into master
* neo: MB-55428 Add support to GCS link for analytics-link-setup MB-41477 Change method used to obtain erlang cookie MB-55341 Add history retention options to bucket create and edit DOC-10775 Docs infrastructure for 7.2.0 Change-Id: I26ea914e63796696d809c3dc67a11e682cc53c68
2 parents f9c8e69 + 38c3ce2 commit 5aa98d6

9 files changed

Lines changed: 275 additions & 17 deletions

‎cbmgr.py‎

Lines changed: 81 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -976,6 +976,14 @@ def __init__(self):
976976

977977
group.add_argument("--purge-interval", dest="purge_interval", type=(float),
978978
metavar="<float>", help="Sets the frequency of the tombstone purge interval")
979+
group.add_argument("--history-retention-bytes", dest="history_retention_bytes", default=None, type=(int),
980+
metavar="<bytes>", help="Set the maximum size of retained document history in bytes")
981+
group.add_argument("--history-retention-seconds", dest="history_retention_seconds", default=None, type=(int),
982+
metavar="<seconds>", help="Set the maximum age of retained document history in seconds")
983+
group.add_argument("--enable-history-retention-by-default",
984+
dest="enable_history_retention", metavar="<0|1>", choices=["0", "1"],
985+
help="Enable history retention for new collections created in this bucket by default "
986+
"(0 or 1)")
979987

980988
group.add_argument("--enable-point-in-time", dest="enable_pitr", metavar="<0|1>",
981989
choices=["0", "1"], help="Enable the Point-In-Time feature on this bucket, which allows "
@@ -1036,6 +1044,25 @@ def execute(self, opts):
10361044
if opts.storage == "magma":
10371045
storage_type = "magma"
10381046

1047+
if opts.type != "couchbase":
1048+
if opts.history_retention_bytes is not None:
1049+
_exit_if_errors([f"--history-retention-bytes cannot be specified for a {opts.type} bucket"])
1050+
if opts.history_retention_seconds is not None:
1051+
_exit_if_errors([f"--history-retention-seconds cannot be specified for a {opts.type} bucket"])
1052+
if opts.enable_history_retention is not None:
1053+
_exit_if_errors([f"--enable-history-retention-by-default cannot be specified for a {opts.type} bucket"])
1054+
1055+
if storage_type != "magma":
1056+
if opts.history_retention_bytes is not None:
1057+
_exit_if_errors([f"--history-retention-bytes cannot be specified for a bucket with {storage_type} "
1058+
"backend"])
1059+
if opts.history_retention_seconds is not None:
1060+
_exit_if_errors([f"--history-retention-seconds cannot be specified for a bucket with {storage_type} "
1061+
"backend"])
1062+
if opts.enable_history_retention is not None:
1063+
_exit_if_errors(["--enable-history-retention-by-default cannot be specified for a bucket with "
1064+
f"{storage_type} backend"])
1065+
10391066
priority = None
10401067
if opts.priority is not None:
10411068
if opts.priority == BUCKET_PRIORITY_HIGH_STR:
@@ -1057,7 +1084,9 @@ def execute(self, opts):
10571084
opts.db_frag_size, opts.view_frag_perc, opts.view_frag_size,
10581085
opts.from_hour, opts.from_min, opts.to_hour, opts.to_min,
10591086
opts.abort_outside, opts.paralleldb_and_view_compact, opts.purge_interval,
1060-
opts.enable_pitr, opts.pitr_granularity, opts.pitr_max_history_age)
1087+
opts.history_retention_bytes, opts.history_retention_seconds,
1088+
opts.enable_history_retention, opts.enable_pitr, opts.pitr_granularity,
1089+
opts.pitr_max_history_age)
10611090
_exit_if_errors(errors)
10621091
_success("Bucket created")
10631092

@@ -1158,6 +1187,15 @@ def __init__(self):
11581187
group.add_argument("--purge-interval", dest="purge_interval", type=(float),
11591188
metavar="<num>", help="Set the bucket metadata purge interval")
11601189

1190+
group.add_argument("--history-retention-bytes", dest="history_retention_bytes", default=None, type=(int),
1191+
metavar="<bytes>", help="Set the maximum size of retained document history in bytes")
1192+
group.add_argument("--history-retention-seconds", dest="history_retention_seconds", default=None, type=(int),
1193+
metavar="<seconds>", help="Set the maximum age of retained document history in seconds")
1194+
group.add_argument("--enable-history-retention-by-default",
1195+
dest="enable_history_retention", metavar="<0|1>", choices=["0", "1"],
1196+
help="Enable history retention for new collections created in this bucket by default "
1197+
"(0 or 1)")
1198+
11611199
group.add_argument("--enable-point-in-time", dest="enable_pitr", metavar="<0|1>",
11621200
choices=["0", "1"], help="Enable Point-In-Time backups and restores on this bucket (0 or 1)")
11631201
group.add_argument("--point-in-time-granularity", dest="pitr_granularity", default=None, type=(int),
@@ -1210,6 +1248,26 @@ def execute(self, opts):
12101248

12111249
is_couchbase_bucket = "bucketType" in bucket and bucket["bucketType"] == "membase"
12121250

1251+
if "bucketType" in bucket and bucket["bucketType"] != "membase":
1252+
if opts.history_retention_bytes is not None:
1253+
_exit_if_errors([f"--history-retention-bytes cannot be specified for a {bucket['bucketType']} bucket"])
1254+
if opts.history_retention_seconds is not None:
1255+
_exit_if_errors([f"--history-retention-seconds cannot be specified for a {bucket['bucketType']} "
1256+
"bucket"])
1257+
if opts.enable_history_retention is not None:
1258+
_exit_if_errors(["--enable-history-retention-by-default cannot be specified for a "
1259+
f"{bucket['bucketType']} bucket"])
1260+
if "storageBackend" in bucket and bucket["storageBackend"] != "magma":
1261+
if opts.history_retention_bytes is not None:
1262+
_exit_if_errors(["--history-retention-bytes cannot be specified for a bucket with "
1263+
f"{bucket['storageBackend']} backend"])
1264+
if opts.history_retention_seconds is not None:
1265+
_exit_if_errors(["--history-retention-seconds cannot be specified for a bucket with "
1266+
f"{bucket['storageBackend']} backend"])
1267+
if opts.enable_history_retention is not None:
1268+
_exit_if_errors(["--enable-history-retention-by-default cannot be specified for a bucket with "
1269+
f"{bucket['storageBackend']} backend"])
1270+
12131271
if not is_couchbase_bucket and (opts.enable_pitr is not None or opts.pitr_granularity is not None or
12141272
opts.pitr_max_history_age is not None):
12151273
_exit_if_errors(["Point-In-Time options are only supported for 'couchbase' buckets"])
@@ -1232,8 +1290,11 @@ def execute(self, opts):
12321290
opts.max_ttl, opts.compression_mode, opts.remove_port, opts.db_frag_perc,
12331291
opts.db_frag_size, opts.view_frag_perc, opts.view_frag_size, opts.from_hour,
12341292
opts.from_min, opts.to_hour, opts.to_min, opts.abort_outside,
1235-
opts.paralleldb_and_view_compact, opts.purge_interval, opts.enable_pitr,
1236-
opts.pitr_granularity, opts.pitr_max_history_age, is_couchbase_bucket)
1293+
opts.paralleldb_and_view_compact, opts.purge_interval,
1294+
opts.history_retention_bytes, opts.history_retention_seconds,
1295+
opts.enable_history_retention,
1296+
opts.enable_pitr, opts.pitr_granularity, opts.pitr_max_history_age,
1297+
is_couchbase_bucket)
12371298
_exit_if_errors(errors)
12381299

12391300
_success("Bucket edited")
@@ -2059,7 +2120,8 @@ def execute(self, opts):
20592120
_exit_if_errors(errors)
20602121

20612122
node = result['otpNode']
2062-
cookie = result['otpCookie']
2123+
cookie, errors = self.rest.get_ns_server_cookie()
2124+
_exit_if_errors(errors)
20632125

20642126
if opts.vm != 'ns_server':
20652127
cookie, errors = self.rest.get_babysitter_cookie()
@@ -4483,6 +4545,9 @@ def __init__(self):
44834545
help="The blob endpoint of the link (required)")
44844546

44854547
group = self.parser.add_argument_group("Analytics Service GCS link setup options")
4548+
group.add_argument("--application-default-credentials", dest="application_default_credentials",
4549+
action="store_true",
4550+
help="The option to use application default credentials for authentication (optional)")
44864551
group.add_argument("--json-credentials", dest="json_credentials", metavar="<key>",
44874552
help="The JSON credentials of the link (optional)")
44884553

@@ -4519,6 +4584,12 @@ def _set(self, opts):
45194584
if opts.type == 'azureblob' or opts.type == 'azuredatalake':
45204585
self._verify_azure_options(opts)
45214586

4587+
if opts.type == 'gcs':
4588+
# --application-default-credentials and --json-credentials are not allowed to be passed together
4589+
if opts.application_default_credentials and opts.json_credentials:
4590+
_exit_if_errors(['Parameter --json-credentials is not allowed if --application-default-credentials '
4591+
'is provided'])
4592+
45224593
if opts.dataverse:
45234594
opts.scope = opts.dataverse
45244595
if opts.certificates:
@@ -4700,6 +4771,8 @@ def __init__(self):
47004771
"are provided it will print all collections")
47014772
group.add_argument("--max-ttl", dest="max_ttl", metavar="<seconds>", type=int,
47024773
help="Set the maximum TTL the collection will accept")
4774+
group.add_argument("--enable-history-retention", dest="enable_history", metavar="<0|1>", choices=["0", "1"],
4775+
help="Enable history retention (0 or 1)")
47034776

47044777
@rest_initialiser(cluster_init_check=True, version_check=True)
47054778
def execute(self, opts):
@@ -4717,6 +4790,9 @@ def execute(self, opts):
47174790
if opts.max_ttl is not None and opts.create_collection is None:
47184791
_exit_if_errors(["--max-ttl can only be set with --create-collection"])
47194792

4793+
if opts.enable_history is not None and opts.create_collection is None:
4794+
_exit_if_errors(["--enable-history-retention can only be set with --create-collection"])
4795+
47204796
if opts.create_scope:
47214797
self._create_scope(opts)
47224798
if opts.drop_scope:
@@ -4748,7 +4824,7 @@ def _list_scopes(self, opts):
47484824

47494825
def _create_collection(self, opts):
47504826
scope, collection = self._get_scope_collection(opts.create_collection)
4751-
_, errors = self.rest.create_collection(opts.bucket, scope, collection, opts.max_ttl)
4827+
_, errors = self.rest.create_collection(opts.bucket, scope, collection, opts.max_ttl, opts.enable_history)
47524828
_exit_if_errors(errors)
47534829
_success("Collection created")
47544830

‎cluster_manager.py‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -833,7 +833,8 @@ def create_bucket(self, name, bucket_type, storage_type, memory_quota,
833833
max_ttl, compression_mode, sync, db_frag_perc, db_frag_size, view_frag_perc,
834834
view_frag_size, from_hour, from_min, to_hour, to_min,
835835
abort_outside, paralleldb_and_view_compact, purge_interval,
836-
pitr_enabled, pitr_granularity, pitr_max_history_age, timeout=60):
836+
history_retention_bytes, history_retention_seconds,
837+
history_retention_default, pitr_enabled, pitr_granularity, pitr_max_history_age, timeout=60):
837838
url = f'{self.hostname}/pools/default/buckets'
838839

839840
if name is None:
@@ -867,6 +868,12 @@ def create_bucket(self, name, bucket_type, storage_type, memory_quota,
867868
params["storageBackend"] = storage_type
868869
if durability_min_level is not None:
869870
params["durabilityMinLevel"] = durability_min_level
871+
if history_retention_bytes is not None:
872+
params["historyRetentionBytes"] = history_retention_bytes
873+
if history_retention_seconds is not None:
874+
params["historyRetentionSeconds"] = history_retention_seconds
875+
if history_retention_default is not None:
876+
params["historyRetentionCollectionDefault"] = one_zero_boolean_to_string(history_retention_default)
870877
if pitr_enabled is not None:
871878
params["pitrEnabled"] = one_zero_boolean_to_string(pitr_enabled)
872879
if pitr_granularity is not None:
@@ -941,6 +948,7 @@ def edit_bucket(self, name, memory_quota, durability_min_level, eviction_policy,
941948
compression_mode, remove_port, db_frag_perc, db_frag_size, view_frag_perc,
942949
view_frag_size, from_hour, from_min, to_hour, to_min,
943950
abort_outside, paralleldb_and_view_compact, purge_interval,
951+
history_retention_bytes, history_retention_seconds, history_retention_default,
944952
pitr_enabled, pitr_granularity, pitr_max_history_age, couchbase_bucket: bool = True):
945953
url = f'{self.hostname}/pools/default/buckets/{name}'
946954

@@ -994,6 +1002,12 @@ def edit_bucket(self, name, memory_quota, durability_min_level, eviction_policy,
9941002
params["purgeInterval"] = purge_interval
9951003
if durability_min_level is not None:
9961004
params["durabilityMinLevel"] = durability_min_level
1005+
if history_retention_bytes is not None:
1006+
params["historyRetentionBytes"] = history_retention_bytes
1007+
if history_retention_seconds is not None:
1008+
params["historyRetentionSeconds"] = history_retention_seconds
1009+
if history_retention_default is not None:
1010+
params["historyRetentionCollectionDefault"] = one_zero_boolean_to_string(history_retention_default)
9971011
if pitr_enabled is not None:
9981012
params["pitrEnabled"] = one_zero_boolean_to_string(pitr_enabled)
9991013
if pitr_granularity is not None:
@@ -1160,6 +1174,12 @@ def get_babysitter_cookie(self):
11601174
'{json, atom_to_binary(ns_server:get_babysitter_cookie(), latin1)}.'
11611175
return self._post_form_encoded(url, payload)
11621176

1177+
def get_ns_server_cookie(self):
1178+
url = f'{self.hostname}/diag/eval'
1179+
payload = \
1180+
'{json, atom_to_binary(erlang:get_cookie(), latin1)}.'
1181+
return self._post_form_encoded(url, payload)
1182+
11631183
def stop_rebalance(self):
11641184
params = {"allowUnsafe": "true"}
11651185
url = f'{self.hostname}/controller/stopRebalance'
@@ -2088,6 +2108,8 @@ def _get_analytics_link_params(opts):
20882108
params['endpoint'] = opts.endpoint
20892109

20902110
# gcs link params
2111+
if opts.application_default_credentials:
2112+
params['applicationDefaultCredentials'] = opts.application_default_credentials
20912113
if opts.json_credentials:
20922114
params['jsonCredentials'] = opts.json_credentials
20932115

@@ -2314,12 +2336,15 @@ def drop_scope(self, bucket, scope):
23142336
f'{urllib.parse.quote_plus(scope)}'
23152337
return self._delete(url, None)
23162338

2317-
def create_collection(self, bucket, scope, collection, max_ttl):
2339+
def create_collection(self, bucket, scope, collection, max_ttl, enable_history):
23182340
url = f'{self.hostname}/pools/default/buckets/{urllib.parse.quote_plus(bucket)}/scopes/' \
23192341
f'{urllib.parse.quote_plus(scope)}/collections'
23202342
params = {"name": collection}
23212343
if max_ttl:
23222344
params["maxTTL"] = max_ttl
2345+
if enable_history:
2346+
params["history"] = one_zero_boolean_to_string(enable_history)
2347+
23232348
return self._post_form_encoded(url, params)
23242349

23252350
def drop_collection(self, bucket, scope, collection):

‎docs/modules/cli/pages/_partials/cbcli/part-common-history-retention.adoc‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
--history-retention-bytes <bytes>::
2+
Specifies how many bytes of document history the bucket should aim to retain
3+
on disk. This option is valid for Couchbase buckets with Magma storage backend
4+
only. By default, history retention in bytes is set to 0, which means that no
5+
history is retained and all stale data is removed during compaction.
6+
7+
--history-retention-seconds <seconds>::
8+
Specifies how many seconds of document history the bucket should aim to retain
9+
on disk. This option is valid for Couchbase buckets with Magma storage backend
10+
only. By default, history retention in seconds is set to 0, which means that
11+
no history is retained and all stale data is removed during compaction.
12+
13+
--enable-history-retention-by-default <0|1>::
14+
Specifies whether or not the document history retention should be enabled by
15+
default for new collections in this bucket. This option is valid for Couchbase
16+
buckets with Magma storage backend only. To enable the document history
17+
retention for new collections, set this option to "1". To disable the document
18+
history retention for new collections, set this option to "0". By default, the
19+
document history retention for new collections is enabled. To create a
20+
collection with a non-default history retention setting use the *collection-manage*
21+
command with *--enable-history* flag set to either "0" or "1".

‎docs/modules/cli/pages/cbcli/couchbase-cli-analytics-link-setup.adoc‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ _couchbase-cli analytics-link-setup_ [--cluster <cluster>] [--username <username
2323
[--service-endpoint <url>] [--account-name <id>] [--account-key <key>]
2424
[--shared-access-signature <token>] [--managed-identity-id <id>] [--client-id <id>]
2525
[--tenant-id <id>] [--client-secret <key>] [--client-certificate <key>]
26-
[--client-certificate-password <key>] [--endpoint <url>] [--json-credentials <key>]
26+
[--client-certificate-password <key>] [--endpoint <url>] [--application-default-credentials]
27+
[--json-credentials <key>]
2728

2829
== DESCRIPTION
2930

@@ -236,11 +237,14 @@ The tenant id of the link. This property is provided together with the --client-
236237
--endpoint <url>::
237238
The endpoint of the link (required).
238239

239-
=== GCS Link Options (Developer Preview)
240-
Following are the options specific to *--type gcs*, i.e. Google Cloud Storage links
240+
=== GCS Link Options
241+
Following are the options specific to *--type gcs*, i.e. Google Cloud Storage links.
242+
243+
--application-default-credentials::
244+
Indicates that the link should use the Google Application Default Credentials for authenticating (optional).
241245

242246
--json-credentials <key>::
243-
The JSON credentials of the link.
247+
The JSON credentials of the link (optional).
244248

245249
--endpoint <url>::
246250
The endpoint of the link (optional).

‎docs/modules/cli/pages/cbcli/couchbase-cli-bucket-create.adoc‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@ _couchbase-cli bucket-create_ [--cluster <url>] [--username <user>]
2626
[--view-fragmentation-threshold-size <mebibytes>] [--from-hour <hour>]
2727
[--from-minute <min>] [--to-hour <hour>] [--to-minute <min>]
2828
[--abort-outside <0|1>] [--parallel-db-view-compaction <0|1>]
29-
[--purge-interval <float>]
29+
[--purge-interval <float>] [--history-retention-bytes <bytes>]
30+
[--history-retention-seconds <seconds>]
31+
[--enable-history-retention-by-default <0|1>]
3032

3133
== DESCRIPTION
3234

@@ -237,6 +239,8 @@ Run index and data compaction in parallel. Global setting only.
237239
Sets the frequency of the tombstone (metadata) purge interval. The default
238240
value is three days.
239241

242+
include::{partialsdir}/cbcli/part-common-history-retention.adoc[]
243+
240244
include::{partialsdir}/cbcli/part-common-pitr-options.adoc[]
241245

242246
include::{partialsdir}/cbcli/part-host-formats.adoc[]

‎docs/modules/cli/pages/cbcli/couchbase-cli-bucket-edit.adoc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ _couchbase-cli bucket-edit_ [--cluster <url>] [--username <user>]
2727
[--from-minute <min>] [--to-hour <hour>] [--to-minute <min>]
2828
[--abort-outside <0|1>] [--parallel-db-view-compaction <0|1>]
2929
[--purge-interval <num>] [--durability-min-level <level>]
30+
[--history-retention-bytes <bytes>] [--history-retention-seconds <seconds>]
31+
[--enable-history-retention-by-default <0|1>]
3032

3133
== DESCRIPTION
3234

@@ -201,6 +203,8 @@ Run index and data compaction in parallel. Global setting only.
201203
majority of the Data Service nodes. Accordingly, it will be written to disk
202204
on those nodes.
203205

206+
include::{partialsdir}/cbcli/part-common-history-retention.adoc[]
207+
204208
include::{partialsdir}/cbcli/part-common-pitr-options.adoc[]
205209

206210
include::{partialsdir}/cbcli/part-host-formats.adoc[]

0 commit comments

Comments
 (0)