Skip to content

Commit d17409c

Browse files
thejcfactorthejcfactor
committed
PYCBC-1722: Wrapper SDK Observability Improvements - Metrics
Motivation ---------- Align wrapper SDKs with Extended Observability RFC Changes ------- * Provide metrics interface that aligns with RFC * Add LoggingMeter * Update observability handler to also handle operation metrics Change-Id: Idb2e472ddb0b03883fb4b38a574e8b7236ed5a5a Reviewed-on: https://review.couchbase.org/c/couchbase-python-client/+/241739 Reviewed-by: Dimitris Christodoulou <dimitris.christodoulou@couchbase.com> Tested-by: Build Bot <build@couchbase.com>
1 parent 9583de7 commit d17409c

72 files changed

Lines changed: 7113 additions & 153 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CMakeLists.txt‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -271,15 +271,17 @@ if(WIN32)
271271
asio
272272
Microsoft.GSL::GSL
273273
taocpp::json
274-
spdlog::spdlog)
274+
spdlog::spdlog
275+
hdr_histogram_static)
275276
else()
276277
target_link_libraries(
277278
pycbc_core PRIVATE
278279
${COUCHBASE_CXX_CLIENT_TARGET}
279280
asio
280281
Microsoft.GSL::GSL
281282
taocpp::json
282-
spdlog::spdlog)
283+
spdlog::spdlog
284+
hdr_histogram_static)
283285
if(APPLE)
284286
target_link_options(
285287
pycbc_core

‎acouchbase/analytics.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,9 +102,9 @@ async def __anext__(self):
102102
self._process_core_span()
103103
return row
104104
except asyncio.QueueEmpty:
105-
self._process_core_span(with_error=True)
106105
exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException)
107106
excptn = exc_cls('Unexpected QueueEmpty exception caught when doing Analytics query.')
107+
self._process_core_span(exc_val=excptn)
108108
raise excptn
109109
except StopAsyncIteration:
110110
self._done_streaming = True
@@ -113,10 +113,10 @@ async def __anext__(self):
113113
self._get_metadata()
114114
raise
115115
except CouchbaseException as ex:
116-
self._process_core_span(with_error=True)
116+
self._process_core_span(exc_val=ex)
117117
raise ex
118118
except Exception as ex:
119-
self._process_core_span(with_error=True)
120119
exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException)
121120
excptn = exc_cls(str(ex))
121+
self._process_core_span(exc_val=excptn)
122122
raise excptn

‎acouchbase/datastructures.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,10 @@ async def _execute_op(self,
7676
create_type: Optional[bool] = None) -> Any:
7777
try:
7878
return await fn(req, obs_handler)
79-
except DocumentNotFoundException:
79+
except DocumentNotFoundException as ex:
8080
if create_type is True:
8181
orig_opt_type = obs_handler.op_type
82-
obs_handler.reset(KeyValueOperationType.Insert, with_error=True)
82+
obs_handler.reset(KeyValueOperationType.Insert, exc_val=ex)
8383
try:
8484
ins_req = self._impl.request_builder.build_insert_request(self._key,
8585
list(),
@@ -361,10 +361,10 @@ async def _execute_op(self,
361361
create_type: Optional[bool] = None) -> Any:
362362
try:
363363
return await fn(req, obs_handler)
364-
except DocumentNotFoundException:
364+
except DocumentNotFoundException as ex:
365365
if create_type is True:
366366
orig_opt_type = obs_handler.op_type
367-
obs_handler.reset(KeyValueOperationType.Insert, with_error=True)
367+
obs_handler.reset(KeyValueOperationType.Insert, exc_val=ex)
368368
try:
369369
ins_req = self._impl.request_builder.build_insert_request(self._key,
370370
dict(),
@@ -618,10 +618,10 @@ async def _execute_op(self,
618618
create_type: Optional[bool] = None) -> Any:
619619
try:
620620
return await fn(req, obs_handler)
621-
except DocumentNotFoundException:
621+
except DocumentNotFoundException as ex:
622622
if create_type is True:
623623
orig_opt_type = obs_handler.op_type
624-
obs_handler.reset(KeyValueOperationType.Insert, with_error=True)
624+
obs_handler.reset(KeyValueOperationType.Insert, exc_val=ex)
625625
try:
626626
ins_req = self._impl.request_builder.build_insert_request(self._key,
627627
list(),
@@ -827,10 +827,10 @@ async def _execute_op(self,
827827
create_type: Optional[bool] = None) -> Any:
828828
try:
829829
return await fn(req, obs_handler)
830-
except DocumentNotFoundException:
830+
except DocumentNotFoundException as ex:
831831
if create_type is True:
832832
orig_opt_type = obs_handler.op_type
833-
obs_handler.reset(KeyValueOperationType.Insert, with_error=True)
833+
obs_handler.reset(KeyValueOperationType.Insert, exc_val=ex)
834834
try:
835835
ins_req = self._impl.request_builder.build_insert_request(self._key,
836836
list(),

‎acouchbase/n1ql.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -104,20 +104,20 @@ async def __anext__(self):
104104
self._process_core_span()
105105
return row
106106
except asyncio.QueueEmpty:
107-
self._process_core_span(with_error=True)
108107
exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException)
109108
excptn = exc_cls('Unexpected QueueEmpty exception caught when doing N1QL query.')
109+
self._process_core_span(exc_val=excptn)
110110
raise excptn
111111
except StopAsyncIteration:
112112
self._done_streaming = True
113113
self._process_core_span()
114114
self._get_metadata()
115115
raise
116116
except CouchbaseException as ex:
117-
self._process_core_span(with_error=True)
117+
self._process_core_span(exc_val=ex)
118118
raise ex
119119
except Exception as ex:
120-
self._process_core_span(with_error=True)
121120
exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException)
122121
excptn = exc_cls(str(ex))
122+
self._process_core_span(exc_val=excptn)
123123
raise excptn

‎acouchbase/search.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,20 +100,20 @@ async def __anext__(self):
100100
self._process_core_span()
101101
return row
102102
except asyncio.QueueEmpty:
103-
self._process_core_span(with_error=True)
104103
exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException)
105104
excptn = exc_cls('Unexpected QueueEmpty exception caught when doing Search query.')
105+
self._process_core_span(exc_val=excptn)
106106
raise excptn
107107
except StopAsyncIteration:
108108
self._done_streaming = True
109109
self._process_core_span()
110110
self._get_metadata()
111111
raise
112112
except CouchbaseException as ex:
113-
self._process_core_span(with_error=True)
113+
self._process_core_span(exc_val=ex)
114114
raise ex
115115
except Exception as ex:
116-
self._process_core_span(with_error=True)
117116
exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException)
118117
excptn = exc_cls(str(ex))
118+
self._process_core_span(exc_val=excptn)
119119
raise excptn

‎acouchbase/tests/metrics_tests/__init__.py‎

Whitespace-only changes.

‎acouchbase/tests/metrics_tests/binary_t.py‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
# Copyright 2016-2026. Couchbase, Inc.
2+
# All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License")
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
from __future__ import annotations
17+
18+
from typing import Any, Optional
19+
20+
import pytest
21+
import pytest_asyncio
22+
23+
from couchbase.exceptions import InvalidArgumentException
24+
from couchbase.logic.observability import OpName
25+
from couchbase.options import (DecrementOptions,
26+
IncrementOptions,
27+
SignedInt64)
28+
from tests.environments.metrics import AsyncBinaryMetricsEnvironment
29+
from tests.environments.metrics.metrics_environment import MeterType
30+
31+
32+
class BinaryMetricsTestsSuite:
33+
34+
TEST_MANIFEST = [
35+
'test_binary_mutation_op',
36+
'test_binary_mutation_op_error',
37+
'test_binary_counter_op',
38+
'test_binary_counter_op_error',
39+
'test_binary_op_no_dispatch_failure',
40+
]
41+
42+
@pytest.fixture(autouse=True)
43+
def reset_validator(self, acb_env: AsyncBinaryMetricsEnvironment):
44+
yield
45+
acb_env.kv_meter_validator.reset()
46+
47+
@pytest.mark.parametrize('op_name', [OpName.Append, OpName.Prepend])
48+
@pytest.mark.asyncio
49+
async def test_binary_mutation_op(self, acb_env: AsyncBinaryMetricsEnvironment, op_name: OpName) -> None:
50+
key = acb_env.get_existing_binary_doc_by_type('bytes_empty', key_only=True)
51+
52+
validator = acb_env.kv_meter_validator
53+
validator.reset(op_name=op_name)
54+
operation = getattr(acb_env.collection.binary(), op_name.value)
55+
await operation(key, b'XXX')
56+
validator.validate_kv_op()
57+
58+
@pytest.mark.parametrize('op_name', [OpName.Append, OpName.Prepend])
59+
@pytest.mark.asyncio
60+
async def test_binary_mutation_op_error(self, acb_env: AsyncBinaryMetricsEnvironment, op_name: OpName) -> None:
61+
key = 'not-a-key'
62+
63+
validator = acb_env.kv_meter_validator
64+
validator.reset(op_name=op_name, validate_error=True)
65+
operation = getattr(acb_env.collection.binary(), op_name.value)
66+
try:
67+
await operation(key, b'XXX')
68+
except Exception:
69+
pass
70+
validator.validate_kv_op()
71+
72+
@pytest.mark.parametrize('op_name', [OpName.Increment, OpName.Decrement])
73+
@pytest.mark.asyncio
74+
async def test_binary_counter_op(self, acb_env: AsyncBinaryMetricsEnvironment, op_name: OpName) -> None:
75+
key = acb_env.get_existing_binary_doc_by_type('counter', key_only=True)
76+
77+
validator = acb_env.kv_meter_validator
78+
validator.reset(op_name=op_name)
79+
operation = getattr(acb_env.collection.binary(), op_name.value)
80+
await operation(key)
81+
validator.validate_kv_op()
82+
83+
@pytest.mark.parametrize('op_name', [OpName.Increment, OpName.Decrement])
84+
@pytest.mark.asyncio
85+
async def test_binary_counter_op_error(self, acb_env: AsyncBinaryMetricsEnvironment, op_name: OpName) -> None:
86+
key = 'not-a-key'
87+
88+
validator = acb_env.kv_meter_validator
89+
validator.reset(op_name=op_name, validate_error=True)
90+
operation = getattr(acb_env.collection.binary(), op_name.value)
91+
try:
92+
# if we don't provide the negative initial value, the counter doc will be created
93+
await operation(key, initial=SignedInt64(-1))
94+
except Exception:
95+
pass
96+
validator.validate_kv_op()
97+
98+
@pytest.mark.parametrize('op_name, opts', [
99+
(OpName.Append, None),
100+
(OpName.Decrement, DecrementOptions),
101+
(OpName.Increment, IncrementOptions),
102+
(OpName.Prepend, None),
103+
])
104+
@pytest.mark.asyncio
105+
async def test_binary_op_no_dispatch_failure(
106+
self,
107+
acb_env: AsyncBinaryMetricsEnvironment,
108+
op_name: OpName,
109+
opts: Optional[Any]
110+
) -> None:
111+
112+
key = acb_env.get_new_doc(key_only=True)
113+
validator = acb_env.kv_meter_validator
114+
validator.reset(op_name=op_name, validate_error=True)
115+
operation = getattr(acb_env.collection.binary(), op_name.value)
116+
try:
117+
if op_name is OpName.Append or op_name is OpName.Prepend:
118+
await operation(key, 123)
119+
else:
120+
await operation(key, opts(initial=123))
121+
except (InvalidArgumentException, ValueError):
122+
pass
123+
124+
validator.validate_kv_op()
125+
126+
127+
class ClassicBinaryMetricsTests(BinaryMetricsTestsSuite):
128+
129+
@pytest.fixture(scope='class', autouse=True)
130+
def manifest_validated(self):
131+
def valid_test_method(meth):
132+
attr = getattr(ClassicBinaryMetricsTests, meth)
133+
return callable(attr) and not meth.startswith('__') and meth.startswith('test')
134+
method_list = [meth for meth in dir(ClassicBinaryMetricsTests) if valid_test_method(meth)]
135+
test_list = set(BinaryMetricsTestsSuite.TEST_MANIFEST).symmetric_difference(method_list)
136+
if test_list:
137+
pytest.fail(f'Test manifest not validated. Missing/extra tests: {test_list}.')
138+
139+
@pytest_asyncio.fixture(scope='class', name='acb_env', params=[MeterType.Basic,
140+
MeterType.NoOp,
141+
])
142+
async def couchbase_test_environment(self, cb_base_env, request):
143+
# a new environment and cluster is created
144+
acb_env = await AsyncBinaryMetricsEnvironment.from_environment(cb_base_env, meter_type=request.param)
145+
await acb_env.setup_binary_data()
146+
acb_env.enable_bucket_mgmt()
147+
yield acb_env
148+
acb_env.disable_bucket_mgmt()
149+
await acb_env.teardown()

0 commit comments

Comments
 (0)