diff --git a/datadog_checks_base/tests/base/checks/openmetrics/test_v2/test_hotpath_bench.py b/datadog_checks_base/tests/base/checks/openmetrics/test_v2/test_hotpath_bench.py new file mode 100644 index 0000000000000..5ca2779d432cc --- /dev/null +++ b/datadog_checks_base/tests/base/checks/openmetrics/test_v2/test_hotpath_bench.py @@ -0,0 +1,239 @@ +# (C) Datadog, Inc. 2020-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +from prometheus_client.parser import text_string_to_metric_families + +from datadog_checks.base import OpenMetricsBaseCheckV2 + +SERVICES = ('checkout', 'catalog', 'payments', 'fulfillment', 'search') +REGIONS = ('us-east-1', 'us-west-2', 'eu-central-1', 'ap-southeast-1') +METHODS = ('GET', 'POST', 'PUT', 'DELETE') +STATUS_CLASSES = ('2xx', '3xx', '4xx', '5xx') +CONSUMERS = tuple(f'consumer-{i:02d}' for i in range(12)) + + +def make_labels(index, shape='current'): + if shape == 'none': + return {'sample': str(index)} + if shape == 'two': + return { + 'service': SERVICES[index % len(SERVICES)], + 'region': REGIONS[index % len(REGIONS)], + } + if shape == 'high': + return { + 'service': SERVICES[index % len(SERVICES)], + 'region': REGIONS[index % len(REGIONS)], + 'method': METHODS[index % len(METHODS)], + 'status_class': STATUS_CLASSES[index % len(STATUS_CLASSES)], + 'route': f'/api/v2/resource/{index:04d}', + 'consumer': f'consumer-{index:04d}', + 'pod': f'pod-{index:04d}', + 'instance': f'10.0.{index // 256}.{index % 256}:9100', + } + return { + 'service': SERVICES[index % len(SERVICES)], + 'region': REGIONS[index % len(REGIONS)], + 'method': METHODS[index % len(METHODS)], + 'status_class': STATUS_CLASSES[index % len(STATUS_CLASSES)], + 'route': f'/api/v1/resource/{index % 60:02d}', + 'consumer': CONSUMERS[index % len(CONSUMERS)], + } + + +def format_labels(labels): + return ','.join(f'{name}="{value}"' for name, value in labels.items()) + + +def make_payload(samples=962, families=1, label_shape='current'): + lines = [ + '# HELP target_info Synthetic target identity.', + '# TYPE target_info gauge', + 'target_info{job="openmetrics_hotpath_bench"} 1', + ] + sample_count = samples - 1 + per_family = sample_count // families + remainder = sample_count % families + sample_index = 0 + for family_index in range(families): + metric = f'om_hotpath_family_{family_index:04d}' if families > 1 else 'om_hotpath_gauge' + lines.append(f'# HELP {metric} Synthetic gauge.') + lines.append(f'# TYPE {metric} gauge') + count = per_family + (1 if family_index < remainder else 0) + for _ in range(count): + labels = make_labels(sample_index, label_shape) + lines.append(f'{metric}{{{format_labels(labels)}}} {sample_index % 997}') + sample_index += 1 + lines.append('') + return '\n'.join(lines) + + +def parse_payload(payload): + return list(text_string_to_metric_families(payload)) + + +def check_for_payload(metrics=None): + check = OpenMetricsBaseCheckV2( + 'test', + {}, + [ + { + 'openmetrics_endpoint': 'foo', + 'namespace': 'hotpath', + 'metrics': metrics or ['.*'], + 'max_returned_metrics': 10000, + } + ], + ) + check.configure_scrapers() + return check + + +def run_parsed_metrics(check, metrics): + scraper = check.scrapers['foo'] + runtime_data = {'flush_first_value': True, 'static_tags': scraper.static_tags} + for metric in metrics: + transformer = scraper.metric_transformer.get(metric) + if transformer is not None: + transformer(metric, scraper.generate_sample_data(metric), runtime_data) + + +def run_parsed_metrics_cold(metrics): + check = check_for_payload() + run_parsed_metrics(check, metrics) + + +def run_parsed_metrics_cold_scrapers(metrics, scraper_count=25): + for _ in range(scraper_count): + check = check_for_payload() + run_parsed_metrics(check, metrics) + + +def run_transformer_get(check, metrics): + scraper = check.scrapers['foo'] + for metric in metrics: + scraper.metric_transformer.get(metric) + + +def run_transformer_get_cold(metrics): + check = check_for_payload() + run_transformer_get(check, metrics) + + +def bench_full_check(benchmark, dd_run_check, mock_http_response, payload, metrics=None): + mock_http_response(payload) + check = OpenMetricsBaseCheckV2( + 'test', + {}, + [ + { + 'openmetrics_endpoint': 'foo', + 'namespace': 'hotpath', + 'metrics': metrics or ['.*'], + 'max_returned_metrics': 10000, + } + ], + ) + dd_run_check(check) + benchmark(check.check, None) + + +def test_hotpath_parse_one_family_two_labels(benchmark): + payload = make_payload(samples=962, families=1, label_shape='two') + benchmark(parse_payload, payload) + + +def test_hotpath_parse_current_labels(benchmark): + payload = make_payload(samples=962, families=1, label_shape='current') + benchmark(parse_payload, payload) + + +def test_hotpath_parse_high_labels(benchmark): + payload = make_payload(samples=962, families=1, label_shape='high') + benchmark(parse_payload, payload) + + +def test_hotpath_parse_many_families(benchmark): + payload = make_payload(samples=962, families=960, label_shape='two') + benchmark(parse_payload, payload) + + +def test_hotpath_transform_current_labels(benchmark): + payload = make_payload(samples=962, families=1, label_shape='current') + metrics = parse_payload(payload) + check = check_for_payload() + benchmark(run_parsed_metrics, check, metrics) + + +def test_hotpath_transform_high_labels(benchmark): + payload = make_payload(samples=962, families=1, label_shape='high') + metrics = parse_payload(payload) + check = check_for_payload() + benchmark(run_parsed_metrics, check, metrics) + + +def test_hotpath_transform_many_families(benchmark): + payload = make_payload(samples=962, families=960, label_shape='two') + metrics = parse_payload(payload) + check = check_for_payload() + benchmark(run_parsed_metrics, check, metrics) + + +def test_hotpath_transformer_get_cold_many_families(benchmark): + payload = make_payload(samples=962, families=960, label_shape='two') + metrics = parse_payload(payload) + benchmark(run_transformer_get_cold, metrics) + + +def test_hotpath_transformer_get_warm_many_families(benchmark): + payload = make_payload(samples=962, families=960, label_shape='two') + metrics = parse_payload(payload) + check = check_for_payload() + run_transformer_get(check, metrics) + benchmark(run_transformer_get, check, metrics) + + +def test_hotpath_transform_cold_one_family_two_labels(benchmark): + payload = make_payload(samples=962, families=1, label_shape='two') + metrics = parse_payload(payload) + benchmark(run_parsed_metrics_cold, metrics) + + +def test_hotpath_transform_cold_current_labels(benchmark): + payload = make_payload(samples=962, families=1, label_shape='current') + metrics = parse_payload(payload) + benchmark(run_parsed_metrics_cold, metrics) + + +def test_hotpath_transform_cold_many_families(benchmark): + payload = make_payload(samples=962, families=960, label_shape='two') + metrics = parse_payload(payload) + benchmark(run_parsed_metrics_cold, metrics) + + +def test_hotpath_transform_cold_many_families_multi_scraper(benchmark): + payload = make_payload(samples=962, families=960, label_shape='two') + metrics = parse_payload(payload) + benchmark(run_parsed_metrics_cold_scrapers, metrics) + + +def test_hotpath_full_one_family_two_labels(benchmark, dd_run_check, mock_http_response): + payload = make_payload(samples=962, families=1, label_shape='two') + bench_full_check(benchmark, dd_run_check, mock_http_response, payload) + + +def test_hotpath_full_current_labels(benchmark, dd_run_check, mock_http_response): + payload = make_payload(samples=962, families=1, label_shape='current') + bench_full_check(benchmark, dd_run_check, mock_http_response, payload) + + +def test_hotpath_full_high_labels(benchmark, dd_run_check, mock_http_response): + payload = make_payload(samples=962, families=1, label_shape='high') + bench_full_check(benchmark, dd_run_check, mock_http_response, payload) + + +def test_hotpath_full_many_families(benchmark, dd_run_check, mock_http_response): + payload = make_payload(samples=962, families=960, label_shape='two') + bench_full_check(benchmark, dd_run_check, mock_http_response, payload)