From 0ce067c392ffd2dbcec110b495005a032ae10d43 Mon Sep 17 00:00:00 2001 From: Heston Hoffman Date: Tue, 26 May 2026 08:21:57 -0700 Subject: [PATCH 01/44] (docs) Style fixes for n8n integration (#23816) * (docs) Style fixes for n8n integration * Fix typo * tiny edits --- n8n/README.md | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/n8n/README.md b/n8n/README.md index 9e9d410984a93..18da75615dd84 100644 --- a/n8n/README.md +++ b/n8n/README.md @@ -4,18 +4,17 @@ This check monitors [n8n][1] through the Datadog Agent. -Collect n8n metrics including: +This integration collects n8n metrics including: - Cache metrics: hit, miss, and update counts. -- Workflow metrics: started, success, failed counters, audit workflow lifecycle counters; in n8n 2.x, an execution-duration histogram. -- Node metrics: per-node started and finished counters emitted by worker processes in queue mode. +- Workflow metrics: Started, success, and failed counters. Audit workflow life cycle counters. In n8n 2.x, an execution-duration histogram. +- Node metrics: per-node counters (started and finished) emitted by worker processes in queue mode. - Queue metrics: queue depth; enqueued, dequeued, completed, failed, and stalled counters; and scaling-mode worker gauges. - HTTP metrics: request duration histograms tagged with status code. - Process and Node.js runtime metrics. - ## Setup -Follow the instructions below to install and configure this check for an Agent running on a host. For containerized environments, see the [Autodiscovery Integration Templates][3] for guidance on applying these instructions. +Follow the instructions below to install and configure this check for an Agent running on a host. For containerized environments, see the [Autodiscovery integration templates][3] for guidance on applying these instructions. ### Installation @@ -26,7 +25,9 @@ No additional installation is needed on your server. #### Enable the n8n metrics endpoint -The `/metrics` endpoint is disabled by default and must be enabled in your n8n configuration. Note that the `/metrics` endpoint is only available for self-hosted instances and is not available on n8n Cloud. +The `/metrics` endpoint is disabled by default and must be enabled in your n8n configuration. + +**Note**: The `/metrics` endpoint is only available for self-hosted instances and is not available on n8n Cloud. Set the following environment variables to enable metrics: @@ -51,7 +52,7 @@ N8N_METRICS_PREFIX=n8n_ For more details, see the n8n documentation on [enabling Prometheus metrics][10]. -If you change `N8N_METRICS_PREFIX` from its default of `n8n_`, you **must** also set `raw_metric_prefix` in the integration's `conf.yaml` to the same value. Otherwise the check will not recognize the exposed metric names and will silently submit nothing: +If you change `N8N_METRICS_PREFIX` from its default of `n8n_`, you **must** also set `raw_metric_prefix` in the integration's `conf.yaml` to the same value. Otherwise the check does not recognize the exposed metric names and silently submits nothing: ```yaml instances: @@ -63,12 +64,12 @@ instances: Most n8n counters are registered dynamically the first time their underlying event fires. The integration ships mappings for around 70 of these event-bus counters, including: -- Workflow lifecycle: `n8n.workflow.started.count`, `n8n.workflow.success.count`, `n8n.workflow.failed.count`, `n8n.workflow.cancelled.count` +- Workflow life cycle: `n8n.workflow.started.count`, `n8n.workflow.success.count`, `n8n.workflow.failed.count`, `n8n.workflow.cancelled.count` - Audit (workflow, user, credentials, package, variable, execution data): `n8n.audit.workflow.executed.count`, `n8n.audit.user.login.success.count`, `n8n.audit.user.credentials.created.count`, and similar - AI nodes: `n8n.ai.tool.called.count`, `n8n.ai.llm.generated.count`, `n8n.ai.vector.store.searched.count`, and similar -- Runner, queue, and node lifecycle: `n8n.runner.task.requested.count`, `n8n.queue.job.completed.count`, `n8n.node.started.count`, `n8n.node.finished.count` +- Runner, queue, and node life cycle: `n8n.runner.task.requested.count`, `n8n.queue.job.completed.count`, `n8n.node.started.count`, `n8n.node.finished.count` -These counters do not appear on the `/metrics` endpoint until the corresponding event has occurred. A healthy idle deployment will not produce data points for them until that activity fires. The complete list is in [`metadata.csv`][7]. +These counters do not appear on the `/metrics` endpoint until the corresponding event has occurred. A healthy idle deployment does not produce datapoints for them until that activity fires. The complete list is in [`metadata.csv`][7]. If a future n8n release exposes a new event-driven counter that is not yet covered by this integration, add it to the `extra_metrics` option in your instance configuration: @@ -85,7 +86,9 @@ The left-hand side is the Prometheus counter name as n8n exposes it (keep the `_ In queue mode, n8n runs separate worker processes that execute jobs picked up from a Redis-backed queue. Each worker exposes its own `/metrics` endpoint and emits a different subset of metrics than the main process. Worker-observed metrics include `n8n.queue.job.dequeued.count`, `n8n.queue.job.stalled.count`, `n8n.node.started.count`, `n8n.node.finished.count`, and `n8n.runner.task.requested.count`. Main-only metrics include `n8n.instance.role.leader` and the `n8n.scaling.mode.queue.jobs.*` family. -To expose worker metrics, set `QUEUE_HEALTH_CHECK_ACTIVE=true` and `QUEUE_HEALTH_CHECK_PORT=` on each worker. **In n8n 2.x, port `5679` is reserved for the task runner broker, so pick a different port (for example `5680`).** +To expose worker metrics, set `QUEUE_HEALTH_CHECK_ACTIVE=true` and `QUEUE_HEALTH_CHECK_PORT=` on each worker. + +**Note**: In n8n 2.x, port `5679` is reserved for the task runner broker. Pick a different port (for example `5680`). For full coverage in queue deployments, configure one Datadog instance per n8n process exposing `/metrics`, including main and worker processes: @@ -107,11 +110,11 @@ Several metric families were introduced in n8n 2.x and are not emitted on n8n 1. - The `n8n.{production,manual,production.root}.executions`, `n8n.users.total`, `n8n.enabled.users`, `n8n.workflows.total`, and `n8n.credentials.total` family. Only emitted when `N8N_METRICS_INCLUDE_WORKFLOW_STATISTICS=true` is set. - The `n8n.expression.*` family (`evaluation.duration.seconds`, `code.cache.{hit,miss,eviction,size}`, `pool.{acquired,replenish.failed,scaled.up,scaled.to.zero}`). Only emitted when n8n is running the new VM-isolated expression engine *and* observability for it is on. Set `N8N_EXPRESSION_ENGINE=vm` and `N8N_EXPRESSION_ENGINE_OBSERVABILITY_ENABLED=true` on the n8n process; both default to off (the engine defaults to `legacy`). These metrics surface the per-expression evaluation latency, the compiled-expression LRU cache hit and miss rates, and the V8-isolate pool's idle scaling behavior. They are most useful for troubleshooting workflow latency that traces back to slow `{{ ... }}` evaluation. -Some metrics only emit samples after the corresponding runtime event occurs. For example, failures-only counters (`*.failures.count`) need an authentication failure, audit workflow counters need the matching workflow state transition, and the libuv `n8n.nodejs.active.requests` gauge needs an in-flight libuv request. A healthy idle deployment may not produce data points for these metrics until that activity occurs. +Some metrics only emit samples after the corresponding runtime event occurs. For example, failures-only counters (`*.failures.count`) need an authentication failure, audit workflow counters need the matching workflow state transition, and the libuv `n8n.nodejs.active.requests` gauge needs an in-flight libuv request. A healthy idle deployment may not produce datapoints for these metrics until that activity occurs. #### Tag cardinality -When `N8N_METRICS_INCLUDE_WORKFLOW_ID_LABEL=true`, http and workflow execution histograms are tagged with `workflow_id` (and similar labels for nodes). On deployments with many distinct workflows or nodes, this can produce high-cardinality metrics. Drop the label via `exclude_labels` or omit `N8N_METRICS_INCLUDE_WORKFLOW_ID_LABEL` to keep tag cardinality bounded. +When `N8N_METRICS_INCLUDE_WORKFLOW_ID_LABEL=true`, http and workflow execution histograms are tagged with `workflow_id` (and similar labels for nodes). On deployments with many distinct workflows or nodes, this can produce high-cardinality metrics. Drop the label through `exclude_labels` or omit `N8N_METRICS_INCLUDE_WORKFLOW_ID_LABEL` to keep tag cardinality bounded. #### Configure the Datadog Agent @@ -121,7 +124,7 @@ When `N8N_METRICS_INCLUDE_WORKFLOW_ID_LABEL=true`, http and workflow execution h ### Log collection -_Available for Agent versions >6.0_ +**Note**: Available for Agent versions 6.0 and later. #### Enable n8n logging From 9e89c6f1eedf5fb82c6f03bd471ecb7568f4ee20 Mon Sep 17 00:00:00 2001 From: Juanpe Araque Date: Tue, 26 May 2026 18:51:06 +0100 Subject: [PATCH 02/44] Detect stale and unreleased entries in Agent release requirements (#23813) * Validate stale Agent release requirements * Skip unreleased integrations in Agent release output * Tighten unreleased-integrations parsing and broaden tests - Validate that by-agent-version-range keys contain the '..' separator and raise a clear ValueError naming the offending key. - Drop the redundant else branch in exclude_unreleased_integrations and move the historical folder-name comment back next to normalize_catalog. - mkdir(exist_ok=True) in the write_repo_config helper for consistency with neighbours. - Parametrize test_agent_version_in_range_is_inclusive (now covers below/above bounds and the malformed-range error path). - Add a direct unit test on get_unreleased_integrations that exercises by-integration and by-agent-version-range together. - Add a clean-pass test for validate agent-reqs and pull the set_root teardown into an isolated_root yield fixture. - Add changelog entries for ddev and datadog_checks_dev. * Surface malformed version ranges as a clean ddev abort - Drop the redundant empty parent table header in .ddev/config.toml; the two sub-tables imply it. - Catch ValueError from agent_version_in_range at every command entry point that triggers the lookup (integrations, changelog, integrations_changelog) and surface it via app.abort so config authors get a clean message instead of a Python traceback. - Document that exclude_unreleased_integrations accepts both raw and folder-normalized catalog keys. * Scope stale-entry detection to whole-file invocations When the user runs `ddev validate agent-reqs `, only the requested check should be validated; previously the new stale-entry detection still scanned the whole requirements file and surfaced unrelated stale packages, defeating per-check pre-commit usage. --- .ddev/config.toml | 18 +++++ datadog_checks_dev/changelog.d/23813.added | 1 + .../tooling/commands/validate/agent_reqs.py | 36 +++++++++- .../commands/validate/test_agent_reqs.py | 66 +++++++++++++++++++ ddev/changelog.d/23813.added | 1 + ddev/src/ddev/cli/release/agent/changelog.py | 5 +- ddev/src/ddev/cli/release/agent/common.py | 54 +++++++++++++-- .../ddev/cli/release/agent/integrations.py | 8 ++- .../release/agent/integrations_changelog.py | 5 +- ddev/tests/cli/release/agent/conftest.py | 13 ++++ .../tests/cli/release/agent/test_changelog.py | 26 ++++++++ .../cli/release/agent/test_integrations.py | 61 +++++++++++++++++ 12 files changed, 283 insertions(+), 11 deletions(-) create mode 100644 datadog_checks_dev/changelog.d/23813.added create mode 100644 datadog_checks_dev/tests/tooling/commands/validate/test_agent_reqs.py create mode 100644 ddev/changelog.d/23813.added diff --git a/.ddev/config.toml b/.ddev/config.toml index eabffa06e2919..92c39bef62c2b 100644 --- a/.ddev/config.toml +++ b/.ddev/config.toml @@ -240,6 +240,24 @@ trace-captures = false ## Just in case __pycache__ is present in the root of the repo __pycache__ = false +# Integrations that were pinned in requirements-agent-release.txt but were not shipped +# in the listed Agent releases. Agent release generation uses these entries to skip +# false positives when building AGENT_INTEGRATIONS.md and Agent changelog data. +# Use by-integration for one-off skips: +# integration_name = ["7.78.0", "7.79.0"] +# Use by-agent-version-range for inclusive Agent version ranges: +# "7.74.0..7.78.0" = ["datadog-first-integration", "datadog-second-integration"] +[overrides.release.agent.unreleased-integrations.by-integration] + +[overrides.release.agent.unreleased-integrations.by-agent-version-range] +"7.74.0..7.78.0" = [ + "datadog-control-m", + "datadog-krakend", + "datadog-lustre", + "datadog-n8n", + "datadog-prefect", +] + # Explicitely add the platforms supported by an integration for those where the manifest has been # removed. # This is a temporary fix while we implement a metadata.json file that we can add to each integration diff --git a/datadog_checks_dev/changelog.d/23813.added b/datadog_checks_dev/changelog.d/23813.added new file mode 100644 index 0000000000000..d7bc9bd0f1aaf --- /dev/null +++ b/datadog_checks_dev/changelog.d/23813.added @@ -0,0 +1 @@ +Fail `ddev validate agent-reqs` when `requirements-agent-release.txt` pins a `datadog-*` package whose integration folder is no longer present in the repo. diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/agent_reqs.py b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/agent_reqs.py index e9905670273d3..0d5d1bdcf4093 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/agent_reqs.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/agent_reqs.py @@ -13,9 +13,14 @@ echo_warning, ) from datadog_checks.dev.tooling.constants import AGENT_V5_ONLY, NOT_CHECKS, get_agent_release_requirements -from datadog_checks.dev.tooling.release import get_package_name +from datadog_checks.dev.tooling.release import get_folder_name, get_package_name from datadog_checks.dev.tooling.testing import process_checks_option -from datadog_checks.dev.tooling.utils import complete_valid_checks, get_version_string, parse_agent_req_file +from datadog_checks.dev.tooling.utils import ( + complete_valid_checks, + get_valid_checks, + get_version_string, + parse_agent_req_file, +) from datadog_checks.dev.utils import read_file @@ -63,6 +68,33 @@ def agent_reqs(check): if unreleased_checks: joined_checks = ', '.join(unreleased_checks) echo_warning(f"{len(unreleased_checks)} unreleased checks: {joined_checks}") + if check is None or check.lower() == 'all': + stale_released_checks = find_stale_released_checks(agent_reqs_content) + if stale_released_checks: + failed_checks += len(stale_released_checks) + for package_name in stale_released_checks: + folder_name = get_folder_name(package_name) + message = ( + f"{package_name} is pinned in requirements-agent-release.txt " + f"but `{folder_name}` is not present in the repo" + ) + echo_failure(message) + annotate_error(release_requirements_file, message) if failed_checks: echo_failure(f"{failed_checks} checks out of sync") abort() + + +def find_stale_released_checks(agent_reqs_content: dict[str, str]) -> list[str]: + """Return pinned Agent packages that no longer match a repo check.""" + expected_packages = { + get_package_name(check_name) + for check_name in get_valid_checks() + if check_name not in AGENT_V5_ONLY | NOT_CHECKS + } + + return sorted( + package_name + for package_name in agent_reqs_content + if package_name.startswith('datadog-') and package_name not in expected_packages + ) diff --git a/datadog_checks_dev/tests/tooling/commands/validate/test_agent_reqs.py b/datadog_checks_dev/tests/tooling/commands/validate/test_agent_reqs.py new file mode 100644 index 0000000000000..96ebed600a267 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/commands/validate/test_agent_reqs.py @@ -0,0 +1,66 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os + +import pytest +from click.testing import CliRunner + +from datadog_checks.dev.tooling.commands.validate.agent_reqs import agent_reqs +from datadog_checks.dev.tooling.constants import get_root, set_root + + +@pytest.fixture +def isolated_root(): + runner = CliRunner() + previous_root = get_root() + with runner.isolated_filesystem(): + set_root(os.getcwd()) + try: + yield runner + finally: + set_root(previous_root) + + +def test_validate_agent_reqs_fails_on_stale_release_entry(isolated_root): + write_check('foo', '1.0.0') + with open('requirements-agent-release.txt', 'w', encoding='utf-8') as f: + f.write('# DO NOT PASS THIS TO PIP DIRECTLY\ndatadog-foo==1.0.0\ndatadog-snowflake==7.13.0\n') + + result = isolated_root.invoke(agent_reqs) + + assert result.exit_code == 1 + assert ( + 'datadog-snowflake is pinned in requirements-agent-release.txt but `snowflake` is not present in the repo' + ) in result.output + assert 'datadog-foo is pinned' not in result.output + + +def test_validate_agent_reqs_passes_when_every_entry_has_a_folder(isolated_root): + write_check('foo', '1.0.0') + with open('requirements-agent-release.txt', 'w', encoding='utf-8') as f: + f.write('# DO NOT PASS THIS TO PIP DIRECTLY\ndatadog-foo==1.0.0\n') + + result = isolated_root.invoke(agent_reqs) + + assert result.exit_code == 0 + assert 'pinned in requirements-agent-release.txt' not in result.output + + +def test_validate_agent_reqs_does_not_report_stale_entries_when_scoped_to_a_check(isolated_root): + write_check('foo', '1.0.0') + with open('requirements-agent-release.txt', 'w', encoding='utf-8') as f: + f.write('# DO NOT PASS THIS TO PIP DIRECTLY\ndatadog-foo==1.0.0\ndatadog-snowflake==7.13.0\n') + + result = isolated_root.invoke(agent_reqs, ['foo']) + + assert result.exit_code == 0 + assert 'datadog-snowflake is pinned' not in result.output + + +def write_check(name: str, version: str) -> None: + """Create the minimum check structure needed by agent-reqs.""" + check_package = os.path.join(name, 'datadog_checks', name) + os.makedirs(check_package) + with open(os.path.join(check_package, '__about__.py'), 'w', encoding='utf-8') as f: + f.write(f'__version__ = "{version}"\n') diff --git a/ddev/changelog.d/23813.added b/ddev/changelog.d/23813.added new file mode 100644 index 0000000000000..d1c722f785ec2 --- /dev/null +++ b/ddev/changelog.d/23813.added @@ -0,0 +1 @@ +Skip integrations pinned in Agent release requirements but not actually shipped in a given Agent release, configurable under `[overrides.release.agent.unreleased-integrations]` in `.ddev/config.toml`. diff --git a/ddev/src/ddev/cli/release/agent/changelog.py b/ddev/src/ddev/cli/release/agent/changelog.py index fba92bc29375d..114a7b295f721 100644 --- a/ddev/src/ddev/cli/release/agent/changelog.py +++ b/ddev/src/ddev/cli/release/agent/changelog.py @@ -58,7 +58,10 @@ def changelog(app: Application, since: str, to: str, write: bool, force: bool): app.repo.git.fetch_tags() - changes_per_agent = get_changes_per_agent(app.repo, since, to) + try: + changes_per_agent = get_changes_per_agent(app.repo, since, to) + except ValueError as exc: + app.abort(str(exc)) # store the changelog in memory changelog_contents = StringIO() diff --git a/ddev/src/ddev/cli/release/agent/common.py b/ddev/src/ddev/cli/release/agent/common.py index a1c8673d812fa..71cbb55c69f50 100644 --- a/ddev/src/ddev/cli/release/agent/common.py +++ b/ddev/src/ddev/cli/release/agent/common.py @@ -11,6 +11,7 @@ AgentChangelog = dict[str, dict[str, tuple[str, bool, bool]]] DATADOG_PACKAGE_PREFIX = 'datadog-' +UNRELEASED_INTEGRATIONS_CONFIG = '/overrides/release/agent/unreleased-integrations' def get_agent_tags(repo: Repository, since: str, to: str) -> list[str]: @@ -73,11 +74,8 @@ def get_changes_per_agent(repo: Repository, since: str, to: str) -> AgentChangel file_contents = repo.git.show_file(req_file_name, agent_tags[i]) catalog_prev = parse_agent_req_file(file_contents) - # at some point in the git history, the requirements file erroneously - # contained the folder name instead of the package name for each check, - # let's be resilient by normalizing all entries to be folder names - catalog_now = normalize_catalog(catalog_now) - catalog_prev = normalize_catalog(catalog_prev) + catalog_now = exclude_unreleased_integrations(repo, normalize_catalog(catalog_now), current_tag) + catalog_prev = exclude_unreleased_integrations(repo, normalize_catalog(catalog_prev), agent_tags[i]) changes_per_agent[current_tag] = {} @@ -94,10 +92,56 @@ def get_changes_per_agent(repo: Repository, since: str, to: str) -> AgentChangel return changes_per_agent +# at some point in the git history, the requirements file erroneously +# contained the folder name instead of the package name for each check, +# let's be resilient by normalizing all entries to be folder names def normalize_catalog(catalog: dict[str, str]) -> dict[str, str]: return {normalize_package_name(k): v for k, v in catalog.items()} +def exclude_unreleased_integrations(repo: Repository, catalog: dict[str, str], agent_version: str) -> dict[str, str]: + """Filter integrations listed as unreleased for ``agent_version``; catalog keys may be raw or folder-normalized.""" + skipped_integrations = get_unreleased_integrations(repo, agent_version) + if not skipped_integrations: + return catalog + return { + name: version for name, version in catalog.items() if normalize_package_name(name) not in skipped_integrations + } + + +def get_unreleased_integrations(repo: Repository, agent_version: str) -> set[str]: + unreleased_integrations = repo.config.get(UNRELEASED_INTEGRATIONS_CONFIG, default={}) + by_integration = unreleased_integrations.get('by-integration', {}) + by_agent_version_range = unreleased_integrations.get('by-agent-version-range', {}) + + skipped_integrations = { + normalize_package_name(name) for name, versions in by_integration.items() if agent_version in versions + } + for version_range, integration_names in by_agent_version_range.items(): + if agent_version_in_range(agent_version, version_range): + skipped_integrations.update(normalize_package_name(name) for name in integration_names) + + return skipped_integrations + + +def agent_version_in_range(agent_version: str, version_range: str) -> bool: + from packaging.version import parse as parse_version + + parts = version_range.split('..', 1) + if len(parts) != 2: + raise ValueError( + f"Invalid version range {version_range!r} in " + f"{UNRELEASED_INTEGRATIONS_CONFIG}/by-agent-version-range; " + "expected format: 'START..END'" + ) + start, end = parts + version = parse_version(agent_version) + start_version = parse_version(start) + end_version = parse_version(end) + + return start_version <= version <= end_version + + def normalize_package_name(name: str) -> str: """ Given a Python package name for a check, return the corresponding folder diff --git a/ddev/src/ddev/cli/release/agent/integrations.py b/ddev/src/ddev/cli/release/agent/integrations.py index 9c8281baebe4c..156a3aa0b60e7 100644 --- a/ddev/src/ddev/cli/release/agent/integrations.py +++ b/ddev/src/ddev/cli/release/agent/integrations.py @@ -29,7 +29,7 @@ def integrations(app: Application, since: str, to: str, write: bool, force: bool tool will generate the list for every Agent since version 6.3.0 (before that point we don't have enough information to build the log). """ - from ddev.cli.release.agent.common import get_agent_tags, parse_agent_req_file + from ddev.cli.release.agent.common import exclude_unreleased_integrations, get_agent_tags, parse_agent_req_file agent_tags = get_agent_tags(app.repo, since, to) # get the list of integrations shipped with the agent from the requirements file @@ -40,7 +40,11 @@ def integrations(app: Application, since: str, to: str, write: bool, force: bool integrations_contents.write(f'## Datadog Agent version {tag}\n\n') # Requirements for current tag file_contents = app.repo.git.show_file(req_file_name, tag) - for name, ver in parse_agent_req_file(file_contents).items(): + try: + catalog = exclude_unreleased_integrations(app.repo, parse_agent_req_file(file_contents), tag) + except ValueError as exc: + app.abort(str(exc)) + for name, ver in catalog.items(): integrations_contents.write(f'* {name}: {ver}\n') integrations_contents.write('\n') diff --git a/ddev/src/ddev/cli/release/agent/integrations_changelog.py b/ddev/src/ddev/cli/release/agent/integrations_changelog.py index b3f70a33e86ec..343fa385964ea 100644 --- a/ddev/src/ddev/cli/release/agent/integrations_changelog.py +++ b/ddev/src/ddev/cli/release/agent/integrations_changelog.py @@ -39,7 +39,10 @@ def integrations_changelog(app: Application, integrations: tuple[str], since: st if not integrations: integrations = [integration.name for integration in app.repo.integrations.iter_all('all')] - changes_per_agent = get_changes_per_agent(app.repo, since, to) + try: + changes_per_agent = get_changes_per_agent(app.repo, since, to) + except ValueError as exc: + app.abort(str(exc)) integrations_versions: dict[str, dict[str, str]] = defaultdict(dict) for agent_version, version_changes in changes_per_agent.items(): diff --git a/ddev/tests/cli/release/agent/conftest.py b/ddev/tests/cli/release/agent/conftest.py index 35172b2b57118..d0a72f41395ce 100644 --- a/ddev/tests/cli/release/agent/conftest.py +++ b/ddev/tests/cli/release/agent/conftest.py @@ -1,9 +1,12 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +from collections.abc import Callable + import pytest from ddev.repo.core import Repository +from ddev.utils.fs import Path @pytest.fixture @@ -55,6 +58,16 @@ def commit(msg): yield repo +@pytest.fixture +def write_repo_config() -> Callable[[Path, str], None]: + def write_config(repo_path: Path, contents: str) -> None: + config_dir = repo_path / '.ddev' + config_dir.mkdir(exist_ok=True) + (config_dir / 'config.toml').write_text(contents) + + return write_config + + def write_agent_requirements(repo_path, requirements): with open(repo_path / 'requirements-agent-release.txt', 'w') as req_file: req_file.write('\n'.join(requirements)) diff --git a/ddev/tests/cli/release/agent/test_changelog.py b/ddev/tests/cli/release/agent/test_changelog.py index d4b1396895e3f..3ddff8fdc7fd9 100644 --- a/ddev/tests/cli/release/agent/test_changelog.py +++ b/ddev/tests/cli/release/agent/test_changelog.py @@ -90,6 +90,32 @@ def test_new_integration_with_non_initial_version(repo_with_new_integration_patc assert mock_fetch_tags.call_count == 1 +def test_changelog_skips_unreleased_integrations(repo_with_history, config_file, ddev, mocker, write_repo_config): + config_file.model.repos['core'] = str(repo_with_history.path) + config_file.save() + write_repo_config( + repo_with_history.path, + """ +[overrides.release.agent.unreleased-integrations.by-integration] +bar = ["7.38.0"] +""", + ) + mock_fetch_tags = mocker.patch('ddev.utils.git.GitRepository.fetch_tags') + + result = ddev('release', 'agent', 'changelog', '--since', '7.37.0', '--to', '7.38.0') + assert result.exit_code == 0 + + expected_output = """## Datadog Agent version [7.38.0](https://github.com/DataDog/datadog-agent/blob/master/CHANGELOG.rst#7380) + +### New Integrations +* datadog_checks_base [2.1.3](https://github.com/DataDog/integrations-core/blob/master/datadog_checks_base/CHANGELOG.md) +### Integration Updates +* foo [1.5.0](https://github.com/DataDog/integrations-core/blob/master/foo/CHANGELOG.md) +""" + assert result.output.rstrip('\n') == expected_output.strip('\n') + assert mock_fetch_tags.call_count == 1 + + @pytest.fixture def repo_with_fake_changelog(repo_with_history, config_file): config_file.model.repos['core'] = str(repo_with_history.path) diff --git a/ddev/tests/cli/release/agent/test_integrations.py b/ddev/tests/cli/release/agent/test_integrations.py index 10589324d25d3..7ef443b2a5fc9 100644 --- a/ddev/tests/cli/release/agent/test_integrations.py +++ b/ddev/tests/cli/release/agent/test_integrations.py @@ -2,9 +2,16 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) import re +from types import SimpleNamespace import pytest +from ddev.cli.release.agent.common import ( + UNRELEASED_INTEGRATIONS_CONFIG, + agent_version_in_range, + get_unreleased_integrations, +) + def test_integrations_without_arguments(fake_integrations, ddev): result = ddev('release', 'agent', 'integrations') @@ -53,6 +60,60 @@ def test_integrations_since_to(fake_integrations, ddev): assert result.output.rstrip('\n') == expected_output.strip('\n') +@pytest.mark.parametrize( + ('version', 'expected'), + [ + pytest.param('7.74.0', True, id='lower-bound-inclusive'), + pytest.param('7.78.0', True, id='upper-bound-inclusive'), + pytest.param('7.77.0', True, id='middle'), + pytest.param('7.73.0', False, id='below-range'), + pytest.param('7.79.0', False, id='above-range'), + ], +) +def test_agent_version_in_range_is_inclusive(version, expected): + assert agent_version_in_range(version, '7.74.0..7.78.0') is expected + + +def test_agent_version_in_range_raises_on_malformed_range(): + with pytest.raises(ValueError, match="Invalid version range '7.74.0'"): + agent_version_in_range('7.74.0', '7.74.0') + + +def test_get_unreleased_integrations_combines_both_keys(): + config_data = { + 'by-integration': {'datadog-bar': ['7.78.0']}, + 'by-agent-version-range': {'7.74.0..7.78.0': ['datadog-foo']}, + } + repo = SimpleNamespace( + config=SimpleNamespace( + get=lambda key, default=None: config_data if key == UNRELEASED_INTEGRATIONS_CONFIG else default, + ) + ) + + assert get_unreleased_integrations(repo, '7.78.0') == {'bar', 'foo'} + + +def test_integrations_skips_unreleased_integrations(repo_with_history, config_file, ddev, write_repo_config): + config_file.model.repos['core'] = str(repo_with_history.path) + config_file.save() + write_repo_config( + repo_with_history.path, + """ +[overrides.release.agent.unreleased-integrations.by-agent-version-range] +"7.38.0..7.39.0" = ["datadog-bar"] +""", + ) + + result = ddev('release', 'agent', 'integrations', '--since', '7.38.0', '--to', '7.38.0') + assert result.exit_code == 0 + + expected_output = """## Datadog Agent version 7.38.0 + +* foo: 1.5.0 +* datadog_checks_base: 2.1.3""" + assert result.output.rstrip('\n') == expected_output.strip('\n') + + @pytest.fixture def repo_with_fake_integrations(repo_with_history, config_file): config_file.model.repos['core'] = str(repo_with_history.path) From 9a6d48614d5f4dc6f6aafeb3927a654db290e60e Mon Sep 17 00:00:00 2001 From: Florian Veaux Date: Wed, 27 May 2026 10:12:20 +0200 Subject: [PATCH 03/44] [snmp] Re-categorize aruba-clearpass metric type fix as breaking change (#23844) --- snmp/changelog.d/{23791.fixed => 23791.changed} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename snmp/changelog.d/{23791.fixed => 23791.changed} (100%) diff --git a/snmp/changelog.d/23791.fixed b/snmp/changelog.d/23791.changed similarity index 100% rename from snmp/changelog.d/23791.fixed rename to snmp/changelog.d/23791.changed From 12c11b54b2cfc0ad281605616b5a2c980ae3ad1e Mon Sep 17 00:00:00 2001 From: Juanpe Araque Date: Wed, 27 May 2026 09:34:58 +0100 Subject: [PATCH 04/44] Show exact run URL and add lifecycle comment to wheel promotion (#23828) * Show exact run URL and add lifecycle comment to wheel promotion - Extend dispatch_workflow with return_run_details so callers can get back the new run's html_url instead of a generic recent-runs link. - ddev dep promote now prints the exact workflow run URL and suppresses noisy httpx request logs around the API calls. - Replace the single success comment in dependency-wheel-promotion.yaml with a lifecycle comment that updates on start, success, and failure, scoped per (PR, head SHA) via a hidden marker so re-dispatches edit the same comment. * Harden lifecycle comment chaining and github-script inputs - Started-comment step now references find_comment.outputs.comment-id (the previous version pointed at its own step output, so re-dispatches for the same SHA would not have updated the existing comment). - Pass inputs.head_sha into actions/github-script via env: HEAD_SHA and read process.env.HEAD_SHA in the script body, so a hostile workflow_dispatch input cannot break out of the JS string literal and execute arbitrary code. * Type-narrow dispatch_workflow and bail out cleanly on missing run details - Add Literal[True]/Literal[False] overloads to GitHubManager.dispatch_workflow so callers asking for run details get a non-nullable dict back at the type level. - Replace the bare assert in ddev dep promote with an explicit app.abort, run the validity check before printing the success message, and keep the success output inside the httpx-suppression scope. - Add ddev/changelog.d/23828.added so the PR-changelog check passes for the ddev source changes. - Lift the github credentials setup into ddev/tests/cli/dep/conftest.py as an autouse fixture, hoist the test-side logging import, and add coverage for the no-run-details abort path and the failure-path httpx level restoration. - Match the cleaner api_post.call_args.kwargs form already used in the companion test in tests/utils/test_github.py. * Trim runtime imports and share httpx-debug fixture across promote tests - Move Any and Literal under TYPE_CHECKING in github.py; they are only used inside annotations that PEP 563 keeps as strings, so they have no runtime cost. The overload decorator stays at module scope because it runs at class definition time. - Add an httpx_at_debug fixture in tests/cli/dep/conftest.py and use it from both httpx-suppression tests so the get-logger/set-DEBUG/restore boilerplate lives in one place. * Type-annotate the new ddev/tests/cli/dep fixtures --- .../workflows/dependency-wheel-promotion.yaml | 59 +++++++++++--- ddev/changelog.d/23828.added | 1 + ddev/src/ddev/cli/dep/promote.py | 35 ++++---- ddev/src/ddev/utils/github.py | 48 +++++++++-- ddev/tests/cli/dep/conftest.py | 32 ++++++++ ddev/tests/cli/dep/test_promote.py | 79 +++++++++++++++++++ ddev/tests/utils/test_github.py | 45 +++++++++++ 7 files changed, 268 insertions(+), 31 deletions(-) create mode 100644 ddev/changelog.d/23828.added create mode 100644 ddev/tests/cli/dep/conftest.py create mode 100644 ddev/tests/cli/dep/test_promote.py diff --git a/.github/workflows/dependency-wheel-promotion.yaml b/.github/workflows/dependency-wheel-promotion.yaml index 5e26cb45bef8a..e5b522f43586e 100644 --- a/.github/workflows/dependency-wheel-promotion.yaml +++ b/.github/workflows/dependency-wheel-promotion.yaml @@ -27,6 +27,25 @@ jobs: - name: Checkout trusted code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Find existing lifecycle comment + id: find_comment + uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 + with: + issue-number: ${{ inputs.pr_number }} + body-includes: "" + + - name: Post lifecycle comment (started) + id: started_comment + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 + with: + issue-number: ${{ inputs.pr_number }} + comment-id: ${{ steps.find_comment.outputs.comment-id }} + edit-mode: replace + body: | + + Wheel promotion started for commit `${{ inputs.head_sha }}` by @${{ github.actor }}. + Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + - name: Checkout PR lockfiles only uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -62,41 +81,57 @@ jobs: - name: Set dependency-wheel-promotion status to success uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + HEAD_SHA: ${{ inputs.head_sha }} with: script: | await github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, - sha: '${{ inputs.head_sha }}', + sha: process.env.HEAD_SHA, state: 'success', context: 'dependency-wheel-promotion', description: 'Wheels promoted to stable storage.', target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, }); - - name: Post success comment - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + - name: Update lifecycle comment (success) + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 with: - script: | - const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: ${{ inputs.pr_number }}, - body: `Wheels promoted to stable storage for commit ${{ inputs.head_sha }} by @${context.actor}. [Workflow run](${runUrl}).`, - }); + issue-number: ${{ inputs.pr_number }} + comment-id: ${{ steps.started_comment.outputs.comment-id }} + edit-mode: replace + body: | + + Wheels promoted to stable storage for commit `${{ inputs.head_sha }}` by @${{ github.actor }}. + Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - name: Set dependency-wheel-promotion status to error if: failure() uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + HEAD_SHA: ${{ inputs.head_sha }} with: script: | await github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, - sha: '${{ inputs.head_sha }}', + sha: process.env.HEAD_SHA, state: 'error', context: 'dependency-wheel-promotion', description: 'Wheel promotion failed. Check the Actions tab for details.', target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, }); + + - name: Update lifecycle comment (failure) + if: failure() + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 + with: + issue-number: ${{ inputs.pr_number }} + comment-id: ${{ steps.started_comment.outputs.comment-id }} + edit-mode: replace + body: | + + Wheel promotion failed for commit `${{ inputs.head_sha }}` by @${{ github.actor }}. + Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + Check the workflow logs before retrying. diff --git a/ddev/changelog.d/23828.added b/ddev/changelog.d/23828.added new file mode 100644 index 0000000000000..1a808e1d73e89 --- /dev/null +++ b/ddev/changelog.d/23828.added @@ -0,0 +1 @@ +Print the exact workflow run URL when dispatching `ddev dep promote`, via a new `return_run_details` option on `GitHubManager.dispatch_workflow`. diff --git a/ddev/src/ddev/cli/dep/promote.py b/ddev/src/ddev/cli/dep/promote.py index d298e7bbe1c23..c7f3210de3671 100644 --- a/ddev/src/ddev/cli/dep/promote.py +++ b/ddev/src/ddev/cli/dep/promote.py @@ -3,6 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from __future__ import annotations +import logging import re from typing import TYPE_CHECKING @@ -39,20 +40,26 @@ def promote(app: Application, pr_url: str): pr_number = int(match.group(1)) - with app.status(f'Fetching PR #{pr_number} head...'): - head_sha, head_ref = app.github.get_pr_head(pr_number) + httpx_logger = logging.getLogger('httpx') + previous_level = httpx_logger.level + httpx_logger.setLevel(logging.WARNING) + try: + with app.status(f'Fetching PR #{pr_number} head...'): + head_sha, head_ref = app.github.get_pr_head(pr_number) - app.display_info(f'PR #{pr_number} — branch: {head_ref}, SHA: {head_sha}') + app.display_info(f'PR #{pr_number}: branch {head_ref}, SHA {head_sha}') - with app.status('Dispatching promote workflow...'): - app.github.dispatch_workflow( - workflow_id=PROMOTE_WORKFLOW, - ref=PROMOTE_WORKFLOW_REF, - inputs={'pr_number': str(pr_number), 'head_sha': head_sha}, - ) + with app.status('Dispatching promote workflow...'): + run_details = app.github.dispatch_workflow( + workflow_id=PROMOTE_WORKFLOW, + ref=PROMOTE_WORKFLOW_REF, + inputs={'pr_number': str(pr_number), 'head_sha': head_sha}, + return_run_details=True, + ) - runs_url = ( - f'https://github.com/{app.github.repo_id}/actions/workflows/{PROMOTE_WORKFLOW}?query=event%3Aworkflow_dispatch' - ) - app.display_success(f'Promote workflow dispatched for PR #{pr_number}.') - app.display_info(f'Recent runs: {runs_url}') + if not run_details: + app.abort('Workflow dispatched but no run details were returned.') + app.display_success(f'Promote workflow dispatched for PR #{pr_number}.') + app.display_info(f'Workflow run: {run_details["html_url"]}') + finally: + httpx_logger.setLevel(previous_level) diff --git a/ddev/src/ddev/utils/github.py b/ddev/src/ddev/utils/github.py index ef314fb7d9fe7..bae40dc9ff23c 100644 --- a/ddev/src/ddev/utils/github.py +++ b/ddev/src/ddev/utils/github.py @@ -6,9 +6,11 @@ import json from functools import cached_property from time import time -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, overload if TYPE_CHECKING: + from typing import Any, Literal + from httpx import Client from ddev.cli.terminal import BorrowedStatus @@ -217,12 +219,48 @@ def get_pull_request_labels(self, pr_number: int) -> list[str] | None: return None return [label['name'] for label in response.json().get('labels', [])] - def dispatch_workflow(self, workflow_id: str, ref: str, inputs: dict[str, Any]) -> None: - """Trigger a workflow_dispatch event.""" - self.__api_post( + @overload + def dispatch_workflow( + self, + workflow_id: str, + ref: str, + inputs: dict[str, Any], + return_run_details: Literal[False] = False, + ) -> None: ... + + @overload + def dispatch_workflow( + self, + workflow_id: str, + ref: str, + inputs: dict[str, Any], + return_run_details: Literal[True], + ) -> dict[str, Any]: ... + + def dispatch_workflow( + self, + workflow_id: str, + ref: str, + inputs: dict[str, Any], + return_run_details: bool = False, + ) -> dict[str, Any] | None: + """Trigger a workflow_dispatch event. + + When ``return_run_details`` is true, request the new run's details from + the API and return the parsed JSON response (``workflow_run_id``, + ``run_url``, ``html_url``). The default keeps the prior fire-and-forget + behavior and returns ``None``. + """ + payload: dict[str, Any] = {'ref': ref, 'inputs': inputs} + if return_run_details: + payload['return_run_details'] = True + response = self.__api_post( self.WORKFLOW_DISPATCH_API.format(repo_id=self.repo_id, workflow_id=workflow_id), - content=json.dumps({'ref': ref, 'inputs': inputs}), + content=json.dumps(payload), ) + if not return_run_details: + return None + return response.json() def get_pull_request_comments(self, pr_number: int) -> list[dict]: response = self.__api_get( diff --git a/ddev/tests/cli/dep/conftest.py b/ddev/tests/cli/dep/conftest.py new file mode 100644 index 0000000000000..8758a8cbc804c --- /dev/null +++ b/ddev/tests/cli/dep/conftest.py @@ -0,0 +1,32 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +import logging +from collections.abc import Generator +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from ddev.config.file import ConfigFileWithOverrides + + +@pytest.fixture(autouse=True) +def configure_github_credentials(config_file: ConfigFileWithOverrides) -> None: + """Provide github credentials so commands that touch app.github do not abort.""" + config_file.model.github = {'user': 'test-user', 'token': 'test-token'} + config_file.save() + + +@pytest.fixture +def httpx_at_debug() -> Generator[logging.Logger, None, None]: + """Force the httpx logger to DEBUG and restore its previous level on teardown.""" + logger = logging.getLogger('httpx') + previous_level = logger.level + logger.setLevel(logging.DEBUG) + try: + yield logger + finally: + logger.setLevel(previous_level) diff --git a/ddev/tests/cli/dep/test_promote.py b/ddev/tests/cli/dep/test_promote.py new file mode 100644 index 0000000000000..2d55db0660c63 --- /dev/null +++ b/ddev/tests/cli/dep/test_promote.py @@ -0,0 +1,79 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import logging + +import pytest + +RUN_DETAILS = { + 'workflow_run_id': 999, + 'run_url': 'https://api.github.com/repos/DataDog/integrations-core/actions/runs/999', + 'html_url': 'https://github.com/DataDog/integrations-core/actions/runs/999', +} + + +def test_promote_dispatches_workflow_and_prints_run_url(ddev, mocker): + mocker.patch('ddev.utils.github.GitHubManager.get_pr_head', return_value=('deadbeef', 'feature-branch')) + dispatch = mocker.patch('ddev.utils.github.GitHubManager.dispatch_workflow', return_value=RUN_DETAILS) + + result = ddev('dep', 'promote', 'https://github.com/DataDog/integrations-core/pull/12345') + + assert result.exit_code == 0, result.output + dispatch.assert_called_once_with( + workflow_id='dependency-wheel-promotion.yaml', + ref='master', + inputs={'pr_number': '12345', 'head_sha': 'deadbeef'}, + return_run_details=True, + ) + assert 'PR #12345' in result.output + assert 'feature-branch' in result.output + assert 'deadbeef' in result.output + assert RUN_DETAILS['html_url'] in result.output + assert 'Recent runs' not in result.output + assert 'query=event%3Aworkflow_dispatch' not in result.output + + +def test_promote_invalid_pr_url_aborts(ddev): + result = ddev('dep', 'promote', 'https://example.invalid/not-a-pr') + + assert result.exit_code != 0 + assert 'Could not extract a PR number' in result.output + + +def test_promote_aborts_when_no_run_details_returned(ddev, mocker): + mocker.patch('ddev.utils.github.GitHubManager.get_pr_head', return_value=('deadbeef', 'feature-branch')) + mocker.patch('ddev.utils.github.GitHubManager.dispatch_workflow', return_value=None) + + result = ddev('dep', 'promote', 'https://github.com/DataDog/integrations-core/pull/12345') + + assert result.exit_code != 0 + assert 'no run details were returned' in result.output + assert 'Promote workflow dispatched' not in result.output + + +def test_promote_suppresses_httpx_logs_and_restores_level(ddev, mocker, httpx_at_debug): + captured_levels = [] + + def capture_level(*_args, **_kwargs): + captured_levels.append(httpx_at_debug.level) + return ('deadbeef', 'feature-branch') + + mocker.patch('ddev.utils.github.GitHubManager.get_pr_head', side_effect=capture_level) + mocker.patch('ddev.utils.github.GitHubManager.dispatch_workflow', return_value=RUN_DETAILS) + + result = ddev('dep', 'promote', 'https://github.com/DataDog/integrations-core/pull/12345') + + assert result.exit_code == 0, result.output + assert captured_levels == [logging.WARNING] + assert httpx_at_debug.level == logging.DEBUG + + +def test_promote_restores_httpx_log_level_on_failure(ddev, mocker, httpx_at_debug): + """Ensure the finally branch restores the previous httpx logger level even when an API call raises.""" + mocker.patch('ddev.utils.github.GitHubManager.get_pr_head', side_effect=RuntimeError('boom')) + mocker.patch('ddev.utils.github.GitHubManager.dispatch_workflow') + + with pytest.raises(RuntimeError, match='boom'): + ddev('dep', 'promote', 'https://github.com/DataDog/integrations-core/pull/12345') + + assert httpx_at_debug.level == logging.DEBUG diff --git a/ddev/tests/utils/test_github.py b/ddev/tests/utils/test_github.py index 59b67d6a97fa5..e1f103dadcb08 100644 --- a/ddev/tests/utils/test_github.py +++ b/ddev/tests/utils/test_github.py @@ -1,6 +1,8 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import json + import pytest from ddev.utils.github import PullRequest @@ -83,3 +85,46 @@ def test_create_label(self, network_replay, github_manager): assert label.json()['name'] == 'my_custom_label' assert label.json()['color'] == 'ff0000' + + +def test_dispatch_workflow_default_returns_none(github_manager, mocker): + """Default dispatch_workflow keeps the prior fire-and-forget behavior.""" + response = mocker.MagicMock() + api_post = mocker.patch('ddev.utils.github.GitHubManager._GitHubManager__api_post', return_value=response) + + result = github_manager.dispatch_workflow( + workflow_id='example.yaml', + ref='master', + inputs={'pr_number': '123', 'head_sha': 'deadbeef'}, + ) + + assert result is None + api_post.assert_called_once() + payload = json.loads(api_post.call_args.kwargs['content']) + assert payload == {'ref': 'master', 'inputs': {'pr_number': '123', 'head_sha': 'deadbeef'}} + assert 'return_run_details' not in payload + + +def test_dispatch_workflow_return_run_details_sends_flag_and_returns_json(github_manager, mocker): + """When return_run_details is true, the payload includes the flag and the parsed JSON is returned.""" + run_details = { + 'workflow_run_id': 42, + 'run_url': 'https://api.github.com/repos/o/r/actions/runs/42', + 'html_url': 'https://github.com/o/r/actions/runs/42', + } + response = mocker.MagicMock() + response.json.return_value = run_details + api_post = mocker.patch('ddev.utils.github.GitHubManager._GitHubManager__api_post', return_value=response) + + result = github_manager.dispatch_workflow( + workflow_id='example.yaml', + ref='master', + inputs={'pr_number': '123', 'head_sha': 'deadbeef'}, + return_run_details=True, + ) + + assert result == run_details + payload = json.loads(api_post.call_args.kwargs['content']) + assert payload['return_run_details'] is True + assert payload['ref'] == 'master' + assert payload['inputs'] == {'pr_number': '123', 'head_sha': 'deadbeef'} From 3345d4d211fa6f28c5be57d5325dfc52130502ff Mon Sep 17 00:00:00 2001 From: Juanpe Araque Date: Wed, 27 May 2026 13:10:17 +0100 Subject: [PATCH 05/44] Pin coverage datadog action to the latest one (#23845) --- .github/workflows/master-windows.yml | 2 +- .github/workflows/master.yml | 88 ++++++------ .github/workflows/pr-all-windows.yml | 72 +++++----- .github/workflows/pr-all.yml | 78 +++++------ .github/workflows/pr-test.yml | 74 +++++----- .github/workflows/test-fips-e2e.yml | 201 +++++++++++++-------------- 6 files changed, 257 insertions(+), 258 deletions(-) diff --git a/.github/workflows/master-windows.yml b/.github/workflows/master-windows.yml index 7f858efd91012..e1cdd3c037d71 100644 --- a/.github/workflows/master-windows.yml +++ b/.github/workflows/master-windows.yml @@ -102,7 +102,7 @@ jobs: - name: Upload coverage to Datadog if: always() continue-on-error: true - uses: DataDog/coverage-upload-github-action@9bbbf86d16f7db1b14c5b885e61cf0d96053686a # v1.0.0 + uses: DataDog/coverage-upload-github-action@6c4bd935248daa6f0ef94e3e6ba71ad5ad079998 # v1.0.3 with: api_key: ${{ secrets.DD_API_KEY }} files: coverage-reports diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 7fcc2f903aaf0..45786b66173ae 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -3,27 +3,27 @@ name: Master on: push: branches: - - master + - master paths: # List of files/paths that should trigger the run. The intention is to avoid running all tests if the commit only includes changes on assets or README - - '*/datadog_checks/**' - - '*/tests/**' - - 'ddev/**' - - 'datadog_checks_base/**' - - 'datadog_checks_dev/**' - # Contains overrides for testing - - '.ddev/**' - # Want to ensure any change in workflows is validated - - '.github/workflows/**' - # Test matrices and dependencies - - '*/hatch.toml' - - '*/pyproject.toml' - # Some integrations might use this file to validate metrics emission - - '*/metadata.csv' - # In case some linting formatting config has changed - - 'pyproject.toml' + - "*/datadog_checks/**" + - "*/tests/**" + - "ddev/**" + - "datadog_checks_base/**" + - "datadog_checks_dev/**" + # Contains overrides for testing + - ".ddev/**" + # Want to ensure any change in workflows is validated + - ".github/workflows/**" + # Test matrices and dependencies + - "*/hatch.toml" + - "*/pyproject.toml" + # Some integrations might use this file to validate metrics emission + - "*/metadata.csv" + # In case some linting formatting config has changed + - "pyproject.toml" schedule: - - cron: '0 2 * * *' + - cron: "0 2 * * *" jobs: cache: @@ -31,7 +31,7 @@ jobs: test: needs: - - cache + - cache uses: ./.github/workflows/test-all.yml with: @@ -48,12 +48,12 @@ jobs: secrets: inherit permissions: - # needed for compute-matrix in test-target.yml - contents: read + # needed for compute-matrix in test-target.yml + contents: read publish-test-results: needs: - - test + - test if: success() || failure() concurrency: @@ -69,7 +69,7 @@ jobs: upload-coverage: needs: - - test + - test if: > !github.event.repository.private && (success() || failure()) @@ -80,27 +80,27 @@ jobs: contents: read steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Download all coverage artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 - with: - pattern: coverage-* - path: coverage-reports - merge-multiple: false + - name: Download all coverage artifacts + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + pattern: coverage-* + path: coverage-reports + merge-multiple: false - - name: Upload coverage to Codecov - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - directory: coverage-reports - fail_ci_if_error: false + - name: Upload coverage to Codecov + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de + with: + use_oidc: true + directory: coverage-reports + fail_ci_if_error: false - - name: Upload coverage to Datadog - if: always() - continue-on-error: true - uses: DataDog/coverage-upload-github-action@9bbbf86d16f7db1b14c5b885e61cf0d96053686a # v1.0.0 - with: - api_key: ${{ secrets.DD_API_KEY }} - files: coverage-reports - format: cobertura + - name: Upload coverage to Datadog + if: always() + continue-on-error: true + uses: DataDog/coverage-upload-github-action@6c4bd935248daa6f0ef94e3e6ba71ad5ad079998 # v1.0.3 + with: + api_key: ${{ secrets.DD_API_KEY }} + files: coverage-reports + format: cobertura diff --git a/.github/workflows/pr-all-windows.yml b/.github/workflows/pr-all-windows.yml index 8f1d9c0e34268..a3a5c824a1d7c 100644 --- a/.github/workflows/pr-all-windows.yml +++ b/.github/workflows/pr-all-windows.yml @@ -5,17 +5,17 @@ name: PR All Windows on: pull_request: paths: - - datadog_checks_base/datadog_checks/** - - datadog_checks_dev/datadog_checks/dev/*.py - - ddev/src/** - - "!agent_requirements.in" - # Also run if we modify the workflow files - - '.github/workflows/pr-all-windows.yml' - - '.github/workflows/test-target.yml' - - '.github/workflows/test-all-windows.yml' - # Also run in the action to install test-target scripts changes - - '.github/actions/setup-test-target-scripts/**' - - '.github/actions/setup-ddev/**' + - datadog_checks_base/datadog_checks/** + - datadog_checks_dev/datadog_checks/dev/*.py + - ddev/src/** + - "!agent_requirements.in" + # Also run if we modify the workflow files + - ".github/workflows/pr-all-windows.yml" + - ".github/workflows/test-target.yml" + - ".github/workflows/test-all-windows.yml" + # Also run in the action to install test-target scripts changes + - ".github/actions/setup-test-target-scripts/**" + - ".github/actions/setup-ddev/**" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.head_ref }} @@ -26,8 +26,8 @@ jobs: uses: ./.github/workflows/test-all-windows.yml permissions: - # needed for compute-matrix in test-target.yml - contents: read + # needed for compute-matrix in test-target.yml + contents: read with: repo: core @@ -39,14 +39,14 @@ jobs: save-event: needs: - - test + - test if: success() || failure() uses: ./.github/workflows/save-event.yml upload-coverage: needs: - - test + - test if: > !github.event.repository.private && (success() || failure()) @@ -57,27 +57,27 @@ jobs: contents: read steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Download all coverage artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 - with: - pattern: coverage-* - path: coverage-reports - merge-multiple: false + - name: Download all coverage artifacts + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + pattern: coverage-* + path: coverage-reports + merge-multiple: false - - name: Upload coverage to Codecov - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - directory: coverage-reports - fail_ci_if_error: false + - name: Upload coverage to Codecov + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de + with: + use_oidc: true + directory: coverage-reports + fail_ci_if_error: false - - name: Upload coverage to Datadog - if: always() - continue-on-error: true - uses: DataDog/coverage-upload-github-action@9bbbf86d16f7db1b14c5b885e61cf0d96053686a # v1.0.0 - with: - api_key: ${{ secrets.DD_API_KEY }} - files: coverage-reports - format: cobertura + - name: Upload coverage to Datadog + if: always() + continue-on-error: true + uses: DataDog/coverage-upload-github-action@6c4bd935248daa6f0ef94e3e6ba71ad5ad079998 # v1.0.3 + with: + api_key: ${{ secrets.DD_API_KEY }} + files: coverage-reports + format: cobertura diff --git a/.github/workflows/pr-all.yml b/.github/workflows/pr-all.yml index 9ea6dce99667e..fb9fe8ca4cb30 100644 --- a/.github/workflows/pr-all.yml +++ b/.github/workflows/pr-all.yml @@ -3,20 +3,20 @@ name: PR All on: pull_request: paths: - - datadog_checks_base/datadog_checks/** - - datadog_checks_base/pyproject.toml - - datadog_checks_dev/datadog_checks/dev/*.py - - datadog_checks_dev/pyproject.toml - - ddev/src/** - - ddev/pyproject.toml - - "!agent_requirements.in" - # Also run if we modify the workflow files - - '.github/workflows/pr-all.yml' - - '.github/workflows/test-target.yml' - - '.github/workflows/test-all.yml' - # Also run if the action to install test-target scripts changes - - '.github/actions/setup-test-target-scripts/**' - - '.github/actions/setup-ddev/**' + - datadog_checks_base/datadog_checks/** + - datadog_checks_base/pyproject.toml + - datadog_checks_dev/datadog_checks/dev/*.py + - datadog_checks_dev/pyproject.toml + - ddev/src/** + - ddev/pyproject.toml + - "!agent_requirements.in" + # Also run if we modify the workflow files + - ".github/workflows/pr-all.yml" + - ".github/workflows/test-target.yml" + - ".github/workflows/test-all.yml" + # Also run if the action to install test-target scripts changes + - ".github/actions/setup-test-target-scripts/**" + - ".github/actions/setup-ddev/**" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.head_ref }} @@ -27,8 +27,8 @@ jobs: uses: ./.github/workflows/test-all.yml permissions: - # needed for compute-matrix in test-target.yml - contents: read + # needed for compute-matrix in test-target.yml + contents: read with: repo: core @@ -42,14 +42,14 @@ jobs: save-event: needs: - - test + - test if: success() || failure() uses: ./.github/workflows/save-event.yml upload-coverage: needs: - - test + - test if: > !github.event.repository.private && (success() || failure()) @@ -60,27 +60,27 @@ jobs: contents: read steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Download all coverage artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 - with: - pattern: coverage-* - path: coverage-reports - merge-multiple: false + - name: Download all coverage artifacts + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + pattern: coverage-* + path: coverage-reports + merge-multiple: false - - name: Upload coverage to Codecov - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - directory: coverage-reports - fail_ci_if_error: false + - name: Upload coverage to Codecov + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de + with: + use_oidc: true + directory: coverage-reports + fail_ci_if_error: false - - name: Upload coverage to Datadog - if: always() - continue-on-error: true - uses: DataDog/coverage-upload-github-action@9bbbf86d16f7db1b14c5b885e61cf0d96053686a # v1.0.0 - with: - api_key: ${{ secrets.DD_API_KEY }} - files: coverage-reports - format: cobertura + - name: Upload coverage to Datadog + if: always() + continue-on-error: true + uses: DataDog/coverage-upload-github-action@6c4bd935248daa6f0ef94e3e6ba71ad5ad079998 # v1.0.3 + with: + api_key: ${{ secrets.DD_API_KEY }} + files: coverage-reports + format: cobertura diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 8f3a14ece990e..824471469d4dc 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -33,7 +33,7 @@ jobs: test: needs: - - compute-matrix + - compute-matrix if: needs.compute-matrix.outputs.matrix != '[]' && github.event_name != 'merge_group' strategy: fail-fast: false @@ -64,7 +64,7 @@ jobs: test-minimum-base-package: needs: - - compute-matrix + - compute-matrix if: needs.compute-matrix.outputs.matrix != '[]' && github.event_name != 'merge_group' strategy: fail-fast: false @@ -96,16 +96,16 @@ jobs: save-event: needs: - - test - - test-minimum-base-package + - test + - test-minimum-base-package if: success() || failure() uses: ./.github/workflows/save-event.yml upload-coverage: needs: - - test - - test-minimum-base-package + - test + - test-minimum-base-package if: > !github.event.repository.private && (success() || failure()) && @@ -117,35 +117,35 @@ jobs: contents: read steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Download all coverage artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 - with: - pattern: coverage-* - path: coverage-reports - merge-multiple: false - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - directory: coverage-reports - fail_ci_if_error: false - - - name: Upload coverage to Datadog - if: always() - continue-on-error: true - uses: DataDog/coverage-upload-github-action@9bbbf86d16f7db1b14c5b885e61cf0d96053686a # v1.0.0 - with: - api_key: ${{ secrets.DD_API_KEY }} - files: coverage-reports - format: cobertura + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Download all coverage artifacts + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + pattern: coverage-* + path: coverage-reports + merge-multiple: false + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de + with: + use_oidc: true + directory: coverage-reports + fail_ci_if_error: false + + - name: Upload coverage to Datadog + if: always() + continue-on-error: true + uses: DataDog/coverage-upload-github-action@6c4bd935248daa6f0ef94e3e6ba71ad5ad079998 # v1.0.3 + with: + api_key: ${{ secrets.DD_API_KEY }} + files: coverage-reports + format: cobertura check: needs: - - test - - test-minimum-base-package + - test + - test-minimum-base-package # In integrations-core and integrations-extras repos the tests are flaky enough that # it would be a pain to merge PRs with the Merge Queue enabled. # While we work on the tests, we skip the job if it's triggered by Merge Queue. @@ -154,8 +154,8 @@ jobs: runs-on: ubuntu-latest steps: - - name: Check status of required jobs - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 - with: - jobs: ${{ toJSON(needs) }} - allowed-skips: test, test-minimum-base-package + - name: Check status of required jobs + uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 + with: + jobs: ${{ toJSON(needs) }} + allowed-skips: test, test-minimum-base-package diff --git a/.github/workflows/test-fips-e2e.yml b/.github/workflows/test-fips-e2e.yml index 1035573ea9424..23ae8619da71f 100644 --- a/.github/workflows/test-fips-e2e.yml +++ b/.github/workflows/test-fips-e2e.yml @@ -17,10 +17,10 @@ on: type: string pull_request: paths: - - datadog_checks_base/datadog_checks/** - - datadog_checks_base/pyproject.toml + - datadog_checks_base/datadog_checks/** + - datadog_checks_base/pyproject.toml schedule: - - cron: '0 0,8,16 * * *' + - cron: "0 0,8,16 * * *" defaults: run: @@ -43,103 +43,102 @@ jobs: DD_TRACE_ANALYTICS_ENABLED: "true" permissions: - # needed for dd-sts and codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write - # needed for compute-matrix in test-target.yml - contents: read + # needed for dd-sts and codecov in test-target.yml, allows the action to get a JWT signed by Github + id-token: write + # needed for compute-matrix in test-target.yml + contents: read steps: - - - name: Set environment variables with sanitized paths - run: | - JOB_NAME="test-fips-e2e" - - echo "TEST_RESULTS_DIR=$TEST_RESULTS_BASE_DIR/$JOB_NAME" >> $GITHUB_ENV - echo "TRACE_CAPTURE_FILE=$TRACE_CAPTURE_BASE_DIR/$JOB_NAME" >> $GITHUB_ENV - - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: "${{ env.PYTHON_VERSION }}" - - - name: Get Datadog credentials - id: dd-sts - uses: DataDog/dd-sts-action@2e8187910199bd93129520183c093e19aa585c75 # v1.0.0 - with: - policy: integrations-core-api-key - - - name: Install ddev from local folder - uses: ./.github/actions/setup-ddev - with: - install-mode: local - cache-profile: local-ddev-base - - - name: Configure ddev - run: |- - ddev config set upgrade_check false - ddev config set repos.core . - ddev config set repo core - - - name: Prepare for testing - env: - PYTHONUNBUFFERED: "1" - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} - ORACLE_DOCKER_USERNAME: ${{ secrets.ORACLE_DOCKER_USERNAME }} - ORACLE_DOCKER_PASSWORD: ${{ secrets.ORACLE_DOCKER_PASSWORD }} - DD_GITHUB_USER: ${{ github.actor }} - DD_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: ddev ci setup ${{ inputs.target || 'tls' }} - - - name: Run E2E tests with FIPS disabled - env: - DDEV_E2E_AGENT: "${{ inputs.agent-image || 'registry.datadoghq.com/agent-dev:master-py3' }}" - DD_API_KEY: "${{ steps.dd-sts.outputs.api_key }}" - run: | - ddev env test --base --new-env --junit ${{ inputs.target || 'tls' }} -- all -m "fips_off" - - - name: Run E2E tests with FIPS enabled - env: - DDEV_E2E_AGENT: "${{ inputs.agent-image-fips || 'registry.datadoghq.com/agent-dev:master-fips' }}" - DD_API_KEY: "${{ steps.dd-sts.outputs.api_key }}" - run: | - ddev env test --base --new-env --junit ${{ inputs.target || 'tls' }} -- all -k "fips_on" - - - name: Finalize test results - if: always() - run: |- - mkdir -p "${{ env.TEST_RESULTS_DIR }}" - if [[ -d ${{ inputs.target || 'tls' }}/junit ]]; then - mv ${{ inputs.target || 'tls' }}/junit/*.xml "${{ env.TEST_RESULTS_DIR }}" - fi - - - name: Upload test results - if: always() - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: "test-results-${{ inputs.target || 'tls' }}" - path: "${{ env.TEST_RESULTS_BASE_DIR }}" - - - name: Upload coverage data - if: > - !github.event.repository.private && - always() - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - files: "${{ inputs.target || 'tls' }}/coverage.xml" - flags: "${{ inputs.target || 'tls' }}" - - - name: Upload coverage to Datadog - if: > - !github.event.repository.private && - always() - continue-on-error: true - uses: DataDog/coverage-upload-github-action@9bbbf86d16f7db1b14c5b885e61cf0d96053686a # v1.0.0 - with: - api_key: ${{ secrets.DD_API_KEY }} - files: "${{ inputs.target || 'tls' }}/coverage.xml" - format: cobertura - flags: "${{ inputs.target || 'tls' }}" + - name: Set environment variables with sanitized paths + run: | + JOB_NAME="test-fips-e2e" + + echo "TEST_RESULTS_DIR=$TEST_RESULTS_BASE_DIR/$JOB_NAME" >> $GITHUB_ENV + echo "TRACE_CAPTURE_FILE=$TRACE_CAPTURE_BASE_DIR/$JOB_NAME" >> $GITHUB_ENV + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Get Datadog credentials + id: dd-sts + uses: DataDog/dd-sts-action@2e8187910199bd93129520183c093e19aa585c75 # v1.0.0 + with: + policy: integrations-core-api-key + + - name: Install ddev from local folder + uses: ./.github/actions/setup-ddev + with: + install-mode: local + cache-profile: local-ddev-base + + - name: Configure ddev + run: |- + ddev config set upgrade_check false + ddev config set repos.core . + ddev config set repo core + + - name: Prepare for testing + env: + PYTHONUNBUFFERED: "1" + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} + ORACLE_DOCKER_USERNAME: ${{ secrets.ORACLE_DOCKER_USERNAME }} + ORACLE_DOCKER_PASSWORD: ${{ secrets.ORACLE_DOCKER_PASSWORD }} + DD_GITHUB_USER: ${{ github.actor }} + DD_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: ddev ci setup ${{ inputs.target || 'tls' }} + + - name: Run E2E tests with FIPS disabled + env: + DDEV_E2E_AGENT: "${{ inputs.agent-image || 'registry.datadoghq.com/agent-dev:master-py3' }}" + DD_API_KEY: "${{ steps.dd-sts.outputs.api_key }}" + run: | + ddev env test --base --new-env --junit ${{ inputs.target || 'tls' }} -- all -m "fips_off" + + - name: Run E2E tests with FIPS enabled + env: + DDEV_E2E_AGENT: "${{ inputs.agent-image-fips || 'registry.datadoghq.com/agent-dev:master-fips' }}" + DD_API_KEY: "${{ steps.dd-sts.outputs.api_key }}" + run: | + ddev env test --base --new-env --junit ${{ inputs.target || 'tls' }} -- all -k "fips_on" + + - name: Finalize test results + if: always() + run: |- + mkdir -p "${{ env.TEST_RESULTS_DIR }}" + if [[ -d ${{ inputs.target || 'tls' }}/junit ]]; then + mv ${{ inputs.target || 'tls' }}/junit/*.xml "${{ env.TEST_RESULTS_DIR }}" + fi + + - name: Upload test results + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: "test-results-${{ inputs.target || 'tls' }}" + path: "${{ env.TEST_RESULTS_BASE_DIR }}" + + - name: Upload coverage data + if: > + !github.event.repository.private && + always() + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de + with: + use_oidc: true + files: "${{ inputs.target || 'tls' }}/coverage.xml" + flags: "${{ inputs.target || 'tls' }}" + + - name: Upload coverage to Datadog + if: > + !github.event.repository.private && + always() + continue-on-error: true + uses: DataDog/coverage-upload-github-action@6c4bd935248daa6f0ef94e3e6ba71ad5ad079998 # v1.0.3 + with: + api_key: ${{ secrets.DD_API_KEY }} + files: "${{ inputs.target || 'tls' }}/coverage.xml" + format: cobertura + flags: "${{ inputs.target || 'tls' }}" From b5362c71c254c4a65f7acfb036522dd7e5f893b0 Mon Sep 17 00:00:00 2001 From: dkirov-dd <166512750+dkirov-dd@users.noreply.github.com> Date: Wed, 27 May 2026 14:21:12 +0200 Subject: [PATCH 06/44] feat(downloader): add TUFPointerDownloader for v2 pointer-file format (#23144) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(downloader): add TUFPointerDownloader for v2 pointer-file format The new agent-integrations-tuf pipeline produces TUF targets as JSON pointer files (targets//.json) rather than the old HTML simple index + in-toto approach. This commit adds: - TUFPointerDownloader in download_v2.py: TUF-verifies the pointer file, then fetches and sha256-verifies the wheel from S3. - DigestMismatch exception for sha256/length failures. - --format v2 CLI flag: routes through TUFPointerDownloader. --unsafe-disable-verification carries forward; --type and --ignore-python-version are no-ops in v2 with a warning. - 8 offline unit tests covering happy path, missing target, digest mismatch, length mismatch, and disable_verification mode. Co-Authored-By: Claude Sonnet 4.6 * fix(downloader): use --repository URL for wheel fetch, not pointer's baked value The pointer file always contains the prod S3 repository URL. When validating staging, the caller passes --repository to point at the staging bucket; that URL should be used for both the TUF metadata fetch AND the wheel download, not just the metadata. Adds a test that asserts the wheel is fetched from the caller-supplied URL even when the pointer contains a different (prod) repository value. Co-Authored-By: Claude Sonnet 4.6 * refactor(downloader): resolve latest via S3 listing, drop latest.json reliance Replace the ``latest.json`` rolling pointer fetch with an S3 ``ListObjectsV2`` walk over ``targets//``: filter keys to PEP 440 stable versions and pick the maximum. The chosen version is then fetched through TUF as before, so the pointer file the client trusts is still cryptographically verified. Why list S3 instead of parsing the signed targets metadata: once ``path_hash_prefixes`` delegations are in use, a client cannot tell from metadata alone which delegation signs the latest version of a given project. Listing the bucket sidesteps that — TUF still authoritatively verifies the chosen version's pointer. The publisher counterpart in agent-integrations-tuf drops ``latest.json`` entirely; see DataDog/agent-integrations-tuf PR #9. - ``_resolve_latest_version`` lists ``targets//`` via the S3 REST API (no boto3 dep), parses the XML response, follows the continuation-token pagination, and applies a PEP 440 stable filter - ``get_pointer(project, version=None)`` resolves ``version`` itself before delegating to the TUF Updater - 6 new offline tests cover max-version selection, pre-release/dev filtering, post-release support, the no-stable error, paginated listings, and non-pointer key skipping Co-Authored-By: Claude Opus 4.7 (1M context) * Revert "refactor(downloader): resolve latest via S3 listing, drop latest.json reliance" This reverts commit 70688d8d5971d7d0b41f284e4d7ffea75b8c231e. * feat(downloader): bundle 1.root.json; rename --format to --index; drop --root-json - Bundle metadata/root_history/1.root.json from agent-integrations-tuf as a package resource; TUFPointerDownloader loads it via importlib.resources — no TOFU, no --root-json flag needed - Rename --format v2 to --index (boolean flag); v1 remains the default when --index is absent - Remove trust_anchor parameter from TUFPointerDownloader.__init__ - Drop --format and --root-json from instantiate_downloader (v1 path) - Register 1.root.json as a wheel artifact in pyproject.toml - Update tests to match new interface Co-Authored-By: Claude Sonnet 4.6 (1M context) * fix(downloader): rename --index to --v2 Co-Authored-By: Claude Sonnet 4.6 (1M context) * feat(downloader): default to v2 with v1 fallback; add prod URL constant Without any flag the downloader now attempts v2 (against the prod S3 bucket) and falls back to v1 on any failure, so callers get the new format automatically without code changes. Passing --v2 explicitly keeps the strict v2 path with no fallback (used by the pipeline's validate- staging step). V2_REPOSITORY_URL is the prod bucket constant used for the default repository value in _download_v2(); callers can still override it with --repository. Co-Authored-By: Claude Sonnet 4.6 (1M context) * feat(downloader): resolve hash-prefixed targets via N.targets.json The v2 TUF repository uses consistent-snapshot format: pointer files are stored as {sha256}.{version}.json on S3. Two changes to support this: 1. _make_updater now sets UpdaterConfig(prefix_targets_with_hash=True) so the TUF Updater resolves hash-prefixed paths automatically when calling download_target(). 2. get_pointer() now parses N.targets.json (after Updater.refresh()) to enumerate available versions for the project. This replaces the removed latest.json: when version=None, _resolve_version() scans all /.json entries in targets metadata and returns the highest stable PEP 440 version. The disable_verification path fetches the metadata chain (timestamp → snapshot → targets) without verifying signatures to find the hash-prefixed URL, then fetches the pointer directly. Co-Authored-By: Claude Sonnet 4.6 (1M context) * feat(downloader): resolve latest via latest pointer target * Move v2 TUF root metadata * Simplify v2 downloader implementation * feat(downloader): add MissingVersion and MalformedPointerError exceptions Dedicated types replace the prior reuse of TargetNotFoundError for argument validation (which mislabeled the failure category) and the unchecked KeyError raised on a malformed pointer JSON. Co-Authored-By: Claude Opus 4.7 * fix(downloader): harden v2 wheel fetch and pointer handling - Add explicit 60s timeout to urllib.request.urlopen so a stalled wheel fetch does not hang the Agent installer indefinitely. - Validate required pointer JSON keys (digest, length, wheel_path) and raise the new MalformedPointerError instead of an opaque KeyError. - Raise MissingVersion (a CLIError subclass) when --unsafe-disable-verification is set without --version, so the v1 fallback log reports the actual cause instead of "target not found". - Extract _verify_content to drop the pointer-is-None sentinel and make the verified and direct-download branches structurally parallel. - Add `from __future__ import annotations` so the PEP 604 unions stay compatible with the declared requires-python = ">=3.8". - Move logging.basicConfig out of the constructor and into the CLI entry point (separate commit); the class no longer mutates the root logger. Co-Authored-By: Claude Opus 4.7 * fix(downloader): make v2/v1 fallback handle validation errors and --force - Split _download_v2() into instantiate_v2_downloader() and run_v2_downloader() to mirror the v1 instantiate/run split and let the warning/validation branches be tested without patching sys.argv. - Re-raise user-input errors (CLIError, MissingVersion) before the broad except so they propagate as-is instead of triggering a spurious v1 retry and a misleading "v2 download failed" log line. - Add --force as a no-op compat stub on the v2 parser so v1-only callers do not trip parse_args -> SystemExit and silently skip the fallback. - Hoist `import logging` to module top (was lazy-imported in the except block) and own the verbose-to-level + logging.basicConfig setup that used to live inside TUFPointerDownloader.__init__. - Drop the meaningless `--v2 default=True` re-declaration; rename underscore-prefixed argparse dests to plain names. - Note in the fallback block that v1 offline tests now traverse v2 first on every invocation. Co-Authored-By: Claude Opus 4.7 * test(downloader): broaden v2 coverage and parametrize failure categories - Parametrize _v2_failure_category across all five (exc, category) cases and add DownloadError / TimeoutError coverage that the categorizer already handles but previous tests never asserted. - Replace direct calls to TUFPointerDownloader._target_path with a parametrized test that drives get_pointer and asserts on Updater.get_targetinfo so the behavior, not the private helper, is what's pinned. - Add failure-mode tests for malformed pointer JSON (one per required key), urllib HTTPError/URLError mid-download, and wheel_path without a leading slash so the URL-composition contract is visible. - Update test_direct_download_requires_explicit_version to expect MissingVersion now that argument-validation no longer reuses TargetNotFoundError. - Move @pytest.mark.offline from each class to a module-level pytestmark; drop the leading-underscore prefix on module constants to match AGENTS.md style. Co-Authored-By: Claude Opus 4.7 * style(downloader): sort test imports per project ruff config Ruff in CI uses the root ../pyproject.toml which treats datadog_checks as first-party. Reorder the test imports to match. Co-Authored-By: Claude Opus 4.7 * refactor(downloader): address PR #23144 review feedback - exceptions.py: type-hint MalformedPointerError/DigestMismatch __init__; add LengthMismatch (split from the overloaded DigestMismatch). - download_v2.py: drop underscore from WHEEL_FETCH_TIMEOUT_SECONDS and REQUIRED_POINTER_KEYS per AGENTS.md; validate wheel_path leading slash via MalformedPointerError; verify length first (cheap early-out) before the sha256 digest check. - cli.py: add type hints on download(), _v2_parser(), instantiate_v2_downloader(), run_v2_downloader(); drop the unused _args parameter from run_v2_downloader; collapse the redundant (CLIError, MissingVersion) except clause to just CLIError. - test_v2_downloader.py: assert MalformedPointerError when wheel_path lacks a leading slash; split TestLengthMismatch from TestDigestMismatch; cover instantiate_v2_downloader validation/warning branches and the cli.download() v2-then-v1 fallback orchestration; drop the inline Updater patch in TestDisableVerification in favour of the fixture. * Fix v2 downloader blockers: narrow fallback, future import Narrow the v1 fallback in download() to a tuple of network/lookup errors. Previously every non-CLIError exception triggered v1 retry, including DigestMismatch / LengthMismatch / MalformedPointerError — i.e. integrity failures the v2 path is meant to surface were silently masked. Now those propagate; only TargetNotFoundError, DownloadError, TimeoutError, and urllib.error.URLError fall back. Add `from __future__ import annotations` to cli.py: the new module uses PEP 604 unions and PEP 585 subscripted generics at definition time, which crash on Python 3.8/3.9 (pyproject.toml declares requires-python = ">=3.8"). download_v2.py already had the import. Add parametrized test pinning the new behavior — DigestMismatch, LengthMismatch, and MalformedPointerError propagate without invoking the v1 downloader. Other review feedback (refactor download(), gate compat warnings on --v2, validate pointer field types, split download() into verified / direct, etc.) is deferred to a follow-up to keep this PR focused. * Preserve v1 downloader fallback behavior * Format v2 downloader tests * Add v2 downloader reviewer test coverage * Reuse v2 downloader test wheel name * Restore unsafe v1 fallback regression test --------- Co-authored-by: Claude Sonnet 4.6 --- .../changelog.d/23144.added | 1 + .../datadog_checks/downloader/cli.py | 112 +++++- .../downloader/data/v2/metadata/root.json | 191 +++++++++++ .../datadog_checks/downloader/download_v2.py | 140 ++++++++ .../datadog_checks/downloader/exceptions.py | 39 +++ datadog_checks_downloader/pyproject.toml | 3 + datadog_checks_downloader/tests/test_unit.py | 21 ++ .../tests/test_v2_downloader.py | 319 ++++++++++++++++++ 8 files changed, 822 insertions(+), 4 deletions(-) create mode 100644 datadog_checks_downloader/changelog.d/23144.added create mode 100644 datadog_checks_downloader/datadog_checks/downloader/data/v2/metadata/root.json create mode 100644 datadog_checks_downloader/datadog_checks/downloader/download_v2.py create mode 100644 datadog_checks_downloader/tests/test_v2_downloader.py diff --git a/datadog_checks_downloader/changelog.d/23144.added b/datadog_checks_downloader/changelog.d/23144.added new file mode 100644 index 0000000000000..2b3e21333eff5 --- /dev/null +++ b/datadog_checks_downloader/changelog.d/23144.added @@ -0,0 +1 @@ +Add v2 TUF pointer downloader support. diff --git a/datadog_checks_downloader/datadog_checks/downloader/cli.py b/datadog_checks_downloader/datadog_checks/downloader/cli.py index be3776c3d3682..8cdd29f44af20 100644 --- a/datadog_checks_downloader/datadog_checks/downloader/cli.py +++ b/datadog_checks_downloader/datadog_checks/downloader/cli.py @@ -2,16 +2,30 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations # 1st party. import argparse +import logging import os import re import sys +import urllib.error # 2nd party. +from tuf.api.exceptions import DownloadError + from .download import DEFAULT_ROOT_LAYOUT_TYPE, REPOSITORY_URL_PREFIX, ROOT_LAYOUTS, TUFDownloader -from .exceptions import NonCanonicalVersion, NonDatadogPackage +from .download_v2 import V2_REPOSITORY_URL, TUFPointerDownloader +from .exceptions import CLIError, MissingVersion, NonCanonicalVersion, NonDatadogPackage, TargetNotFoundError + +V2_FALLBACK_ERRORS: tuple[type[BaseException], ...] = ( + MissingVersion, + TargetNotFoundError, + DownloadError, + TimeoutError, + urllib.error.URLError, +) # Private module functions. @@ -25,6 +39,14 @@ def __is_canonical(version): return re.match(P, version) is not None +def _v2_failure_category(exc: Exception) -> str: + if isinstance(exc, TargetNotFoundError): + return 'target version not found' + if isinstance(exc, (DownloadError, TimeoutError, urllib.error.URLError)): + return 'network error' + return 'other' + + def __find_shipped_integrations(): # Recurse up from site-packages until we find the Agent root directory. # The relative path differs between operating systems. @@ -142,6 +164,88 @@ def run_downloader(tuf_downloader, standard_distribution_name, version, ignore_p # Public module functions. -def download(): - tuf_downloader, standard_distribution_name, version, ignore_python_version = instantiate_downloader() - run_downloader(tuf_downloader, standard_distribution_name, version, ignore_python_version) +def download() -> None: + downloader, name, version, args = instantiate_v2_downloader() + + if args.v2: + warn_v2_ignored_args(args) + run_v2_downloader(downloader, name, version) + return + + try: + run_v2_downloader(downloader, name, version) + except V2_FALLBACK_ERRORS as exc: + # Integrity failures (DigestMismatch / LengthMismatch / MalformedPointerError) are + # intentionally not in V2_FALLBACK_ERRORS — they must propagate, not be masked by v1. + logging.getLogger(__name__).info( + 'v2 download failed (%s, %s: %s), falling back to v1', + _v2_failure_category(exc), + type(exc).__name__, + exc, + ) + run_downloader(*instantiate_downloader()) + except CLIError: + # NonDatadogPackage and NonCanonicalVersion: v1 would raise the same. + raise + + +def _v2_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + + parser.add_argument( + 'standard_distribution_name', + type=str, + help='Standard distribution name of the desired Datadog check, e.g. datadog-postgres.', + ) + parser.add_argument( + '--repository', type=str, default=V2_REPOSITORY_URL, help='HTTPS base URL of the v2 TUF repository.' + ) + parser.add_argument('--version', type=str, default=None, help='Version to download (default: latest stable).') + parser.add_argument( + '--unsafe-disable-verification', + action='store_true', + help='Disable TUF verification and wheel digest checks; requires --version and downloads /wheels directly.', + ) + parser.add_argument('-v', '--verbose', action='count', default=0) + parser.add_argument('--v2', action='store_true', default=False) + + # v1 compat flags accepted as no-ops so callers upgrading from v1 get a warning, not an error. + parser.add_argument('--type', type=str, default=None, dest='ignored_type') + parser.add_argument('--ignore-python-version', action='store_true', dest='ignored_ignore_python_version') + parser.add_argument('--force', action='store_true', dest='ignored_force') + + return parser + + +def warn_v2_ignored_args(args: argparse.Namespace) -> None: + if args.ignored_type is not None: + sys.stderr.write('WARNING: --type is not applicable with --v2 and will be ignored.\n') + if args.ignored_ignore_python_version: + sys.stderr.write( + 'NOTE: --ignore-python-version is not applicable with --v2 (wheel selection happens at publish time).\n' + ) + + +def instantiate_v2_downloader() -> tuple[TUFPointerDownloader, str, str | None, argparse.Namespace]: + args = _v2_parser().parse_args() + + if not args.standard_distribution_name.startswith('datadog-'): + raise NonDatadogPackage(args.standard_distribution_name) + + if args.version and not __is_canonical(args.version): + raise NonCanonicalVersion(args.version) + + remainder = min(args.verbose, 5) % 6 + level = (6 - remainder) * 10 + logging.basicConfig(format='%(levelname)-8s: %(message)s', level=level) + + downloader = TUFPointerDownloader( + repository_url=args.repository, + disable_verification=args.unsafe_disable_verification, + ) + return downloader, args.standard_distribution_name, args.version, args + + +def run_v2_downloader(downloader: TUFPointerDownloader, name: str, version: str | None) -> None: + wheel_path = downloader.download(name, version=version) + print(wheel_path) # pylint: disable=print-statement diff --git a/datadog_checks_downloader/datadog_checks/downloader/data/v2/metadata/root.json b/datadog_checks_downloader/datadog_checks/downloader/data/v2/metadata/root.json new file mode 100644 index 0000000000000..e053044657c99 --- /dev/null +++ b/datadog_checks_downloader/datadog_checks/downloader/data/v2/metadata/root.json @@ -0,0 +1,191 @@ +{ + "signatures": [ + { + "keyid": "ac5d650bc9aa17fdad54753fbf64e083f6f613286d0feef991ff61ec26874f2b", + "sig": "3066023100beb16cde4c9e17c725713c6020cb5b11a65dd60a7b46f6842068815593e8f39cfa547ea89b6169474a8ee6a98c22f934023100f215434d25181f6f0d6a75a1bae4f09814678d3409cc0aaf0f8e1cb2b0a7bcb29acd5394b4df1751f7e73c641a17256b" + }, + { + "keyid": "6f0f52eb4cb14d590aafd5f7eb8d9a79477ac89794be2d3caade9fc39b3735e6", + "sig": "306502306130792e890c5257cc8fc951e7c9a4009a5552affbd6bdf5cef3bac647de18f82918adbd4edfaa6d1624e2c378ee0ca4023100db6cff59e145b0113560116eac4466e52fc24e3ec3f02ee39fe7213718c2184d58e5473a5f29429b1a4c63e7ff87b83b" + }, + { + "keyid": "2d019dcc7a3e8da4d22bf364f0e0cb87937b2ced68339f3c53d305d1a9aadcce", + "sig": "3066023100ed93223b4ab9784c00b73937cd431fe9d6af8906548124a10215ad93432523476a3265e712fe99b7555ea30ce5aeff30023100e55ae85f68da6ac322f912cae2a28facb6b3f014770d348a7c9daee100e5eb8ae78cc86321b20711f3b357d900a075a4" + }, + { + "keyid": "e942404daa3e8cb1143ab5f275df2f8c741ae002194147806bd6f05b8e2e816f", + "sig": "3066023100a1a75a85dbe43db459e3d8c1bd935f2717bae0b1cba79ea5b9a5e785b7eb08cc30e08f96ba5fc0ccb9b9c97b9af456450231008dcc197a5de9a93649dfbd27e3f112321441913138c3377487ae85353a982cbffdc88029d681e432e86cfc14c51196ab" + }, + { + "keyid": "1286a08794005a5f1d679e56322f45fd3b55aa198f87bdc699f8213048602000", + "sig": "3064023075188913725a1c2e9af59f8663b6a178156b64d87da126a5970a3b6a3399bdfe7f5c357099f2f1a4e83d52294551c41e0230080ef2ff77b7d558879cbe0eda409c3ba2fe080860506d4f2ede314374e39dc0b2bc466af51fdd258eb76171344d42af" + }, + { + "keyid": "65ccb05ff16285a3b65ea2db2581ed083bb19acfcbd130d5484c151baf28541f", + "sig": "306602310086b9d6f39f795ad188223318f02e1d78b5798d34e333c1933e55891cc1b11cd6771b2d9ab1f5c1fc707d4815e3cc200d023100eb63f35f7cc2d0166357f2c209ecca63b82fc6bc9c310b9a0fa345957b1a0df102036ea6a6d3825d787eb7d3b3131e70" + }, + { + "keyid": "b59ade3245077bd622dc7bf41163a877e05272590cb4830632dc0d034717d735", + "sig": "3066023100fa26ef91f1bb3cdf779cb6bbc43d70bab67a7c66103e61b8998698f469fad0d44002d4a9399ceac304b8ee1a8823fd99023100ecd415e58696ab4778f4bdf5187be3743ac372b29cd139111b3461a0da42f8f44e5bb3ec83c2bd0ce4b6281e585b8889" + }, + { + "keyid": "a07e905cad57b71374ef5e408d61936c31957b35026de0b8db3938878ccad637", + "sig": "3066023100f4802957c21a0916677154494c4360260f5994c35c435d2bf2df39bc7cccca7fb437563d21ae128bcaa7909ead7d6e7802310097513f90e5e7dbe4bcb3f9b20308e966ec38960e8cad4869a4b32be8bd98726ded4a68c671d5f22858dd10ba3b56b04a" + }, + { + "keyid": "a442c20904f96e3a367e16037665bfb2e002bb2e9586cec4c96d83697a49fa2a", + "sig": "30660231008d58d822ee1accd6bff07e79f171d61d122d35c1d51c86b2f2ada76cff695090fdf859127889f9a8d90e539277b5ab5a023100f0ad8d7ba6a25e27316a91bbc61c9b4d31f42c5c93662ea53d660af6b8ab9ee111b135b6844901ccd281fd9246d5f786" + }, + { + "keyid": "8969905969a712d54c9b327939aead62784587b54d1d03cbaa835f79205069bf", + "sig": "3066023100894ecac9291e64ea6b84d168b886ef5829f4ad5b57c83b0ec745b644e4d19983e29c4681b5f744070a679e71fb4325af023100b6365d50a44e59932ea24d17a9fbc975cc7e7b44539d36dc7208470b1ddc9572b06266e149d1793a12a98cd62b803a95" + } + ], + "signed": { + "_type": "root", + "consistent_snapshot": true, + "expires": "2027-04-21T15:31:05Z", + "keys": { + "1286a08794005a5f1d679e56322f45fd3b55aa198f87bdc699f8213048602000": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEruzzCikai9w8LqTLE4cxf0qRIFU6AQve\nnMmudDdNo22MCiOwbuYjJJ1dvRlMiSVrAGyv1+37h8aXGa5Qbx5nb4TEIRfaDth8\nhMbKJcQ7OOK/6SaltjNZh3VaZ396/WIC\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@nouemankhal" + }, + "2d019dcc7a3e8da4d22bf364f0e0cb87937b2ced68339f3c53d305d1a9aadcce": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAErfozI3wqaB8k6o6Mc7SPFiw8s1dLTaxk\nMmhMsdkk7QIl3t+gFzWNdXANEjN027g4S6Ty2CvdzovU37yD24td9pQBh8LGmfPa\nmU5cxtzRaXkCibibJrrvLxyyZTWZXW6C\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@alexeypilyugin" + }, + "4542ee95093cb434e0d80a4bb9dd9d96e6b67cda12759fa2648a7786f822e97d": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEUV4g/gyxCdXKHK07QWO5z6S9lRhL88DO\nOb22g0dCOtxBB2sKojAUw3wXXz+SaUZRFgqfVvezbtsC4LSkkIlwA5MrJDA83kP2\nJRo4BQPtW8wZmtSvkkRQPSfAdXv975pg\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-online-uri": "awskms:arn:aws:kms:us-east-1:510233252802:key/9efe9e34-88f3-4ad3-8828-5340561e7c42" + }, + "65ccb05ff16285a3b65ea2db2581ed083bb19acfcbd130d5484c151baf28541f": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEi44mg+tnJn41Cy4Lr42lQNRuZaHDY4d+\nB/oYkBRTiHl6n6hc6alGLS/1rWijAfSL7x7wgVeOrA5fp1ornW27vPOkRVWJO5Lv\nZcZXwJYi7svVFBkFjBAtAOF6DGuAEWc9\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@lucia-sb" + }, + "6f0f52eb4cb14d590aafd5f7eb8d9a79477ac89794be2d3caade9fc39b3735e6": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEAbWHH4rfNiJFz9gXLPV/QJK0tky4/nW1\nyMPnUe1GRac6UfGcjZvGA7mpmns4FYG1KuHbPhWlEDOQnLjiIiJkY2+Z96tywq6y\n+/e+0Gc2KSsVr0IAWALkTzQE+Q6ru+lj\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@nubtron" + }, + "8969905969a712d54c9b327939aead62784587b54d1d03cbaa835f79205069bf": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEkztjp5ixZrKt94qnSn4bisyEdgs0Wre/\nheazr1zx7MJUCLiHim0lEDWCB64m/YLru+W3/PLwTiQSavO62lB6y3ggjcq/ygwA\n5yxi0bP/MAJBZ0Hl+y+Q8BfKTZSrTb6j\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@hadhemidd" + }, + "a07e905cad57b71374ef5e408d61936c31957b35026de0b8db3938878ccad637": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEEilQwnno5GxJpoyxulKzkkHa0x0/ERDa\nf3m1ZCpF9SoT2B98T+BwT6noD+qlOwX7VKLFSQwl4/od53tu6Wt3s3P70zFviq+Y\n+chUOSCbA5y/TCvfwx4mLBruXI1QbVOh\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@aarakke" + }, + "a442c20904f96e3a367e16037665bfb2e002bb2e9586cec4c96d83697a49fa2a": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEUJ7k4tiIZrWNLhrNrcBBMh4we3GiMlpo\ntwVy72lNw7aMxisK6ttP0mV30Yh1rX37DO6UUdeiWImrYBVfXFkP7z2QD9qKetny\nCeVHycA7uNby7yb7pljv2l2SpTgXACZk\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@iliakur" + }, + "ac5d650bc9aa17fdad54753fbf64e083f6f613286d0feef991ff61ec26874f2b": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEAWOvhm6nk7iY+EYK8ZnrxS49yqLf/ZTR\nJ74WY9Kz3ikjyXASkD4IgqJyyrmbMoqS9k6/RM/Zk6CAfPeZneDh1puVAlxy9nJD\nZp/OW78dVOqrlw1uQ0d+gfe7b4TcUNG4\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@dkirov-dd" + }, + "b59ade3245077bd622dc7bf41163a877e05272590cb4830632dc0d034717d735": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEd/9wooA4OKbC7hUO1OTZN3pnFbc85PDs\n+izKkDDSqj3yk8Pa39OJstT2BHvrn/B0BKMHhE6T/PN/rhorKVIVZ3UZErn1QCgG\nkkcFfA5MQm92SjIr9zAJea9bVUJhZ+PA\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@sarah-witt" + }, + "e942404daa3e8cb1143ab5f275df2f8c741ae002194147806bd6f05b8e2e816f": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE3VG/DJn/wmXh3bQ/LLjGMyKubQ1f5/1P\nJTVDYgTh5AC5zWxDSD26PoNpS29MecItPoM+pMy5YC99mwkEkxjNdwIke1Aons92\n8SVtL3BYH311oC6jLtFt+oqEunL5EdgJ\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@kyle-neale" + } + }, + "roles": { + "root": { + "keyids": [ + "ac5d650bc9aa17fdad54753fbf64e083f6f613286d0feef991ff61ec26874f2b", + "6f0f52eb4cb14d590aafd5f7eb8d9a79477ac89794be2d3caade9fc39b3735e6", + "2d019dcc7a3e8da4d22bf364f0e0cb87937b2ced68339f3c53d305d1a9aadcce", + "e942404daa3e8cb1143ab5f275df2f8c741ae002194147806bd6f05b8e2e816f", + "1286a08794005a5f1d679e56322f45fd3b55aa198f87bdc699f8213048602000", + "65ccb05ff16285a3b65ea2db2581ed083bb19acfcbd130d5484c151baf28541f", + "b59ade3245077bd622dc7bf41163a877e05272590cb4830632dc0d034717d735", + "a07e905cad57b71374ef5e408d61936c31957b35026de0b8db3938878ccad637", + "a442c20904f96e3a367e16037665bfb2e002bb2e9586cec4c96d83697a49fa2a", + "8969905969a712d54c9b327939aead62784587b54d1d03cbaa835f79205069bf" + ], + "threshold": 2 + }, + "snapshot": { + "keyids": [ + "4542ee95093cb434e0d80a4bb9dd9d96e6b67cda12759fa2648a7786f822e97d" + ], + "threshold": 1, + "x-tuf-on-ci-expiry-period": 365, + "x-tuf-on-ci-signing-period": 60 + }, + "targets": { + "keyids": [ + "ac5d650bc9aa17fdad54753fbf64e083f6f613286d0feef991ff61ec26874f2b", + "6f0f52eb4cb14d590aafd5f7eb8d9a79477ac89794be2d3caade9fc39b3735e6", + "2d019dcc7a3e8da4d22bf364f0e0cb87937b2ced68339f3c53d305d1a9aadcce", + "e942404daa3e8cb1143ab5f275df2f8c741ae002194147806bd6f05b8e2e816f", + "1286a08794005a5f1d679e56322f45fd3b55aa198f87bdc699f8213048602000", + "65ccb05ff16285a3b65ea2db2581ed083bb19acfcbd130d5484c151baf28541f", + "b59ade3245077bd622dc7bf41163a877e05272590cb4830632dc0d034717d735", + "a07e905cad57b71374ef5e408d61936c31957b35026de0b8db3938878ccad637", + "a442c20904f96e3a367e16037665bfb2e002bb2e9586cec4c96d83697a49fa2a", + "8969905969a712d54c9b327939aead62784587b54d1d03cbaa835f79205069bf" + ], + "threshold": 1 + }, + "timestamp": { + "keyids": [ + "4542ee95093cb434e0d80a4bb9dd9d96e6b67cda12759fa2648a7786f822e97d" + ], + "threshold": 1, + "x-tuf-on-ci-expiry-period-hours": 48, + "x-tuf-on-ci-signing-period-hours": 24 + } + }, + "spec_version": "1.0.31", + "version": 1, + "x-tuf-on-ci-expiry-period": 365, + "x-tuf-on-ci-signing-period": 60 + } +} \ No newline at end of file diff --git a/datadog_checks_downloader/datadog_checks/downloader/download_v2.py b/datadog_checks_downloader/datadog_checks/downloader/download_v2.py new file mode 100644 index 0000000000000..82557de31f4c9 --- /dev/null +++ b/datadog_checks_downloader/datadog_checks/downloader/download_v2.py @@ -0,0 +1,140 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +"""TUF pointer-file downloader for the v2 repository format.""" + +from __future__ import annotations + +import hashlib +import importlib.resources +import json +import logging +import tempfile +import urllib.request +from pathlib import Path + +from tuf.ngclient import Updater +from tuf.ngclient.config import UpdaterConfig + +from .exceptions import ( + DigestMismatch, + LengthMismatch, + MalformedPointerError, + MissingVersion, + TargetNotFoundError, +) + +logger = logging.getLogger(__name__) + +V2_REPOSITORY_URL = "https://agent-integration-wheels-prod.s3.amazonaws.com" + +# tuf.ngclient sets its own fetcher timeout; this applies only to the raw wheel urlopen(). +WHEEL_FETCH_TIMEOUT_SECONDS = 60 + +REQUIRED_POINTER_KEYS = ('digest', 'length', 'wheel_path') + + +class TUFPointerDownloader: + """Downloads Datadog integration wheels from a v2 TUF repository.""" + + def __init__(self, repository_url: str, disable_verification: bool = False): + self._repository_url = repository_url.rstrip('/') + self._disable_verification = disable_verification + + if disable_verification: + logger.warning('Running with TUF verification disabled. Integrity is protected only by TLS (HTTPS).') + + def _bootstrap_metadata_dir(self, metadata_dir: Path) -> None: + dest = metadata_dir / 'root.json' + metadata = importlib.resources.files('datadog_checks.downloader') / 'data' / 'v2' / 'metadata' + dest.write_bytes((metadata / 'root.json').read_bytes()) + + def _make_updater(self, metadata_dir: Path, target_dir: Path) -> Updater: + return Updater( + metadata_dir=str(metadata_dir), + metadata_base_url=f'{self._repository_url}/metadata/', + target_base_url=f'{self._repository_url}/targets/', + target_dir=str(target_dir), + config=UpdaterConfig(prefix_targets_with_hash=True), + ) + + @staticmethod + def _target_path(project: str, version: str | None) -> str: + name = version if version is not None else 'latest' + return f'{project}/{name}.json' + + @staticmethod + def _wheel_filename(project: str, version: str) -> str: + distribution = project.replace('-', '_') + return f'{distribution}-{version}-py3-none-any.whl' + + def _direct_wheel_url(self, project: str, version: str) -> str: + return f'{self._repository_url}/wheels/{project}/{self._wheel_filename(project, version)}' + + @staticmethod + def _validate_pointer(project: str, pointer: dict) -> None: + for key in REQUIRED_POINTER_KEYS: + if key not in pointer: + raise MalformedPointerError(project, key) + if not pointer['wheel_path'].startswith('/'): + raise MalformedPointerError(project, 'wheel_path') + + @staticmethod + def _verify_content(project: str, content: bytes, pointer: dict) -> None: + if len(content) != pointer['length']: + raise LengthMismatch(project, pointer['length'], len(content)) + actual_digest = hashlib.sha256(content).hexdigest() + if actual_digest != pointer['digest']: + raise DigestMismatch(project, pointer['digest'], actual_digest) + + def get_pointer(self, project: str, version: str | None = None) -> dict: + """Return the pointer JSON for *project* at *version* (or 'latest' when None).""" + with tempfile.TemporaryDirectory() as tmp: + metadata_dir = Path(tmp) / 'metadata' + target_dir = Path(tmp) / 'targets' + metadata_dir.mkdir() + target_dir.mkdir() + + target_path = self._target_path(project, version) + self._bootstrap_metadata_dir(metadata_dir) + updater = self._make_updater(metadata_dir, target_dir) + updater.refresh() + + target_info = updater.get_targetinfo(target_path) + if target_info is None: + label = version if version is not None else 'latest stable' + raise TargetNotFoundError(f'No TUF target for {project!r} version {label!r}') + + pointer_path = target_dir / target_path + pointer_path.parent.mkdir(parents=True, exist_ok=True) + updater.download_target(target_info, pointer_path) + + return json.loads(pointer_path.read_text(encoding='utf-8')) + + def download(self, project: str, version: str | None = None, dest_dir: Path | None = None) -> Path: + """Download and verify the wheel for *project* at *version*; return its path.""" + if self._disable_verification: + if version is None: + raise MissingVersion('unsafe-disable-verification requires an explicit --version') + wheel_url = self._direct_wheel_url(project, version) + wheel_filename = self._wheel_filename(project, version) + pointer: dict | None = None + else: + pointer = self.get_pointer(project, version) + self._validate_pointer(project, pointer) + wheel_url = self._repository_url + pointer['wheel_path'] + wheel_filename = Path(pointer['wheel_path']).name + + dest = (dest_dir or Path(tempfile.mkdtemp())) / wheel_filename + + logger.info('Downloading wheel from %s', wheel_url) + with urllib.request.urlopen(wheel_url, timeout=WHEEL_FETCH_TIMEOUT_SECONDS) as resp: + content = resp.read() + + if pointer is not None: + self._verify_content(project, content, pointer) + + dest.write_bytes(content) + logger.info('Wrote %s to %s', wheel_filename, dest) + return dest diff --git a/datadog_checks_downloader/datadog_checks/downloader/exceptions.py b/datadog_checks_downloader/datadog_checks/downloader/exceptions.py index bb6b75e05a156..db8764040a700 100644 --- a/datadog_checks_downloader/datadog_checks/downloader/exceptions.py +++ b/datadog_checks_downloader/datadog_checks/downloader/exceptions.py @@ -30,6 +30,10 @@ def __str__(self): return '{}'.format(self.standard_distribution_name) +class MissingVersion(CLIError): + """Raised when --version is required but absent (e.g. with --unsafe-disable-verification).""" + + # Exceptions for the download module. @@ -37,6 +41,41 @@ class TargetNotFoundError(ChecksDownloaderException): """An exception raised when a target is not found.""" +class MalformedPointerError(ChecksDownloaderException): + """Raised when a TUF-signed pointer JSON is invalid or missing fields.""" + + def __init__(self, project: str, field: str): + self.project = project + self.field = field + + def __str__(self) -> str: + return f'{self.project}: pointer field {self.field!r} is missing or malformed' + + +class DigestMismatch(ChecksDownloaderException): + """Raised when the downloaded wheel's sha256 does not match the pointer.""" + + def __init__(self, project: str, expected: str, actual: str): + self.project = project + self.expected = expected + self.actual = actual + + def __str__(self) -> str: + return f'{self.project}: expected digest {self.expected}, got {self.actual}' + + +class LengthMismatch(ChecksDownloaderException): + """Raised when the downloaded wheel's byte length does not match the pointer.""" + + def __init__(self, project: str, expected: int, actual: int): + self.project = project + self.expected = expected + self.actual = actual + + def __str__(self) -> str: + return f'{self.project}: expected length {self.expected}, got {self.actual}' + + class IncorrectRootLayoutType(ChecksDownloaderException): def __init__(self, found, expected): self.found = found diff --git a/datadog_checks_downloader/pyproject.toml b/datadog_checks_downloader/pyproject.toml index 56ecc4d80baee..b40dcb4e75d39 100644 --- a/datadog_checks_downloader/pyproject.toml +++ b/datadog_checks_downloader/pyproject.toml @@ -55,6 +55,9 @@ include = [ include = [ "/datadog_checks/downloader", ] +artifacts = [ + "/datadog_checks/downloader/data/v2/metadata/root.json", +] dev-mode-dirs = [ ".", ] diff --git a/datadog_checks_downloader/tests/test_unit.py b/datadog_checks_downloader/tests/test_unit.py index 9170abf3ee09a..160b7c584aeef 100644 --- a/datadog_checks_downloader/tests/test_unit.py +++ b/datadog_checks_downloader/tests/test_unit.py @@ -1,7 +1,28 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import urllib.error + +import pytest +from tuf.api.exceptions import DownloadError + +from datadog_checks.downloader.cli import _v2_failure_category from datadog_checks.downloader.download import TUFDownloader +from datadog_checks.downloader.exceptions import TargetNotFoundError + + +@pytest.mark.parametrize( + 'exc,expected', + [ + pytest.param(TargetNotFoundError('missing'), 'target version not found', id='target-not-found'), + pytest.param(urllib.error.URLError('timeout'), 'network error', id='network-urlerror'), + pytest.param(DownloadError('boom'), 'network error', id='network-downloaderror'), + pytest.param(TimeoutError('slow'), 'network error', id='network-timeout'), + pytest.param(ValueError('bad pointer'), 'other', id='other'), + ], +) +def test_v2_failure_category(exc, expected): + assert _v2_failure_category(exc) == expected def test_non_official_wheel_filter(mocker): diff --git a/datadog_checks_downloader/tests/test_v2_downloader.py b/datadog_checks_downloader/tests/test_v2_downloader.py new file mode 100644 index 0000000000000..db1977db2be0d --- /dev/null +++ b/datadog_checks_downloader/tests/test_v2_downloader.py @@ -0,0 +1,319 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +"""Unit tests for TUFPointerDownloader (v2 repository format) and the v2 CLI surface.""" + +import hashlib +import json +import urllib.error +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from tuf.api.exceptions import DownloadError + +from datadog_checks.downloader import cli +from datadog_checks.downloader.download_v2 import TUFPointerDownloader +from datadog_checks.downloader.exceptions import ( + DigestMismatch, + LengthMismatch, + MalformedPointerError, + MissingVersion, + NonCanonicalVersion, + NonDatadogPackage, + TargetNotFoundError, +) + +pytestmark = pytest.mark.offline + +PROJECT = 'datadog-postgres' +VERSION = '14.0.0' +WHEEL_NAME = f'datadog_postgres-{VERSION}-py3-none-any.whl' +WHEEL_CONTENT = b'fake wheel bytes for testing' +WHEEL_DIGEST = hashlib.sha256(WHEEL_CONTENT).hexdigest() +WHEEL_LENGTH = len(WHEEL_CONTENT) +REPO_URL = 'https://agent-integration-wheels-staging.s3.amazonaws.com' + +POINTER = { + 'digest': WHEEL_DIGEST, + 'length': WHEEL_LENGTH, + 'version': VERSION, + 'repository': REPO_URL, + 'wheel_path': f'/wheels/{PROJECT}/{WHEEL_NAME}', + 'attestation_path': f'/attestations/{PROJECT}/{VERSION}.sigstore.json', +} + + +def _mock_tuf_updater(pointer: dict) -> MagicMock: + pointer_bytes = json.dumps(pointer).encode() + mock_updater = MagicMock() + mock_updater.get_targetinfo.return_value = MagicMock() + + def fake_download_target(_target_info, dest_path): + Path(dest_path).parent.mkdir(parents=True, exist_ok=True) + Path(dest_path).write_bytes(pointer_bytes) + + mock_updater.download_target.side_effect = fake_download_target + return mock_updater + + +def _mock_response(content: bytes) -> MagicMock: + response = MagicMock() + response.__enter__ = lambda s: s + response.__exit__ = MagicMock(return_value=False) + response.read.return_value = content + return response + + +@pytest.fixture +def mock_urlopen(): + with patch('datadog_checks.downloader.download_v2.urllib.request.urlopen') as mock: + mock.return_value = _mock_response(WHEEL_CONTENT) + yield mock + + +@pytest.fixture +def mock_updater_cls(): + with patch('datadog_checks.downloader.download_v2.Updater') as mock: + mock.return_value = _mock_tuf_updater(POINTER) + yield mock + + +class TestTargetResolution: + @pytest.mark.parametrize( + 'version,expected_target', + [ + pytest.param(VERSION, f'{PROJECT}/{VERSION}.json', id='explicit-version'), + pytest.param(None, f'{PROJECT}/latest.json', id='missing-version'), + ], + ) + def test_get_pointer_requests_expected_target(self, mock_urlopen, mock_updater_cls, version, expected_target): + downloader = TUFPointerDownloader(repository_url=REPO_URL) + downloader.get_pointer(PROJECT, version=version) + + mock_updater = mock_updater_cls.return_value + assert mock_updater.get_targetinfo.call_args[0][0] == expected_target + + +class TestHappyPath: + def test_download_returns_wheel_path(self, mock_urlopen, mock_updater_cls, tmp_path): + downloader = TUFPointerDownloader(repository_url=REPO_URL) + wheel_path = downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + + assert wheel_path.exists() + assert wheel_path.read_bytes() == WHEEL_CONTENT + assert wheel_path.name == WHEEL_NAME + + def test_repository_flag_overrides_pointer_repository(self, mock_urlopen, mock_updater_cls, tmp_path): + prod_pointer = {**POINTER, 'repository': 'https://agent-integration-wheels-prod.s3.amazonaws.com'} + mock_updater_cls.return_value = _mock_tuf_updater(prod_pointer) + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + + mock_urlopen.assert_called_once_with( + f'{REPO_URL}/wheels/{PROJECT}/{WHEEL_NAME}', + timeout=60, + ) + + +class TestTargetNotFound: + def test_raises_when_tuf_target_absent(self, mock_urlopen, mock_updater_cls): + mock_updater = MagicMock() + mock_updater.get_targetinfo.return_value = None + mock_updater_cls.return_value = mock_updater + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + with pytest.raises(TargetNotFoundError, match=PROJECT): + downloader.get_pointer(PROJECT, version='99.99.99') + + +class TestDigestMismatch: + def test_raises_on_corrupted_wheel(self, mock_urlopen, mock_updater_cls, tmp_path): + tampered = b'tampered bytes that match the pointer length'[:WHEEL_LENGTH] + mock_urlopen.return_value = _mock_response(tampered) + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + with pytest.raises(DigestMismatch, match=PROJECT): + downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + assert not (tmp_path / WHEEL_NAME).exists() + + +class TestLengthMismatch: + def test_raises_when_pointer_length_does_not_match_wheel(self, mock_urlopen, mock_updater_cls, tmp_path): + bad_pointer = {**POINTER, 'length': WHEEL_LENGTH + 1} + mock_updater_cls.return_value = _mock_tuf_updater(bad_pointer) + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + with pytest.raises(LengthMismatch) as exc_info: + downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + assert exc_info.value.expected == WHEEL_LENGTH + 1 + assert exc_info.value.actual == WHEEL_LENGTH + assert not (tmp_path / WHEEL_NAME).exists() + + +class TestMalformedPointer: + @pytest.mark.parametrize('missing_key', ['digest', 'length', 'wheel_path']) + def test_raises_when_required_key_missing(self, mock_urlopen, mock_updater_cls, tmp_path, missing_key): + broken_pointer = {k: v for k, v in POINTER.items() if k != missing_key} + mock_updater_cls.return_value = _mock_tuf_updater(broken_pointer) + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + with pytest.raises(MalformedPointerError, match=missing_key): + downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + + def test_raises_when_wheel_path_missing_leading_slash(self, mock_urlopen, mock_updater_cls, tmp_path): + no_slash_pointer = {**POINTER, 'wheel_path': f'wheels/{PROJECT}/{WHEEL_NAME}'} + mock_updater_cls.return_value = _mock_tuf_updater(no_slash_pointer) + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + with pytest.raises(MalformedPointerError, match='wheel_path'): + downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + mock_urlopen.assert_not_called() + + +class TestNetworkErrorMidDownload: + def test_http_error_propagates(self, mock_urlopen, mock_updater_cls, tmp_path): + mock_urlopen.side_effect = urllib.error.HTTPError( + url='http://example/x.whl', code=500, msg='boom', hdrs=None, fp=None + ) + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + with pytest.raises(urllib.error.HTTPError): + downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + + def test_url_error_propagates(self, mock_urlopen, mock_updater_cls, tmp_path): + mock_urlopen.side_effect = urllib.error.URLError('unreachable') + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + with pytest.raises(urllib.error.URLError): + downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + + +class TestDisableVerification: + def test_directly_downloads_wheel_without_tuf_or_digest_checks(self, mock_urlopen, mock_updater_cls, tmp_path): + content = b'bytes not matching any signed pointer' + mock_urlopen.return_value = _mock_response(content) + + downloader = TUFPointerDownloader(repository_url=REPO_URL, disable_verification=True) + wheel_path = downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + + mock_urlopen.assert_called_once_with( + f'{REPO_URL}/wheels/{PROJECT}/{WHEEL_NAME}', + timeout=60, + ) + assert wheel_path.name == WHEEL_NAME + assert wheel_path.read_bytes() == content + mock_updater_cls.assert_not_called() + + def test_direct_download_requires_explicit_version(self, tmp_path): + downloader = TUFPointerDownloader(repository_url=REPO_URL, disable_verification=True) + with pytest.raises(MissingVersion, match='requires an explicit --version'): + downloader.download(PROJECT, dest_dir=tmp_path) + + +class TestInstantiateV2Downloader: + def test_rejects_non_datadog_package(self, monkeypatch): + monkeypatch.setattr('sys.argv', ['downloader', 'requests']) + with pytest.raises(NonDatadogPackage, match='requests'): + cli.instantiate_v2_downloader() + + def test_rejects_non_canonical_version(self, monkeypatch): + monkeypatch.setattr('sys.argv', ['downloader', 'datadog-postgres', '--version', 'banana']) + with pytest.raises(NonCanonicalVersion, match='banana'): + cli.instantiate_v2_downloader() + + def test_does_not_warn_when_v1_compat_flags_are_parsed(self, monkeypatch, capsys): + monkeypatch.setattr('sys.argv', ['downloader', 'datadog-postgres', '--type', 'core', '--ignore-python-version']) + cli.instantiate_v2_downloader() + assert capsys.readouterr().err == '' + + def test_warns_for_v1_compat_flags_in_strict_v2_mode(self, monkeypatch, capsys): + monkeypatch.setattr( + 'sys.argv', ['downloader', 'datadog-postgres', '--v2', '--type', 'core', '--ignore-python-version'] + ) + _, _, _, args = cli.instantiate_v2_downloader() + cli.warn_v2_ignored_args(args) + stderr = capsys.readouterr().err + assert 'WARNING: --type' in stderr + assert 'NOTE: --ignore-python-version' in stderr + + def test_force_flag_is_silently_ignored(self, monkeypatch, capsys): + monkeypatch.setattr('sys.argv', ['downloader', 'datadog-postgres', '--force']) + cli.instantiate_v2_downloader() + assert capsys.readouterr().err == '' + + +class TestCliDownloadFallback: + """Covers the cli.download() v2-attempt-then-v1-fallback orchestration.""" + + def test_strict_v2_raises_on_v2_failure(self, monkeypatch): + monkeypatch.setattr('sys.argv', ['downloader', 'datadog-postgres', '--v2']) + monkeypatch.setattr(cli, 'run_v2_downloader', MagicMock(side_effect=TargetNotFoundError('missing'))) + v1 = MagicMock() + monkeypatch.setattr(cli, 'run_downloader', v1) + monkeypatch.setattr(cli, 'instantiate_downloader', MagicMock(return_value=(None, None, None, None))) + + with pytest.raises(TargetNotFoundError): + cli.download() + v1.assert_not_called() + + @pytest.mark.parametrize( + 'fallback_exc', + [ + pytest.param(MissingVersion('missing'), id='missing-version'), + pytest.param(TargetNotFoundError('missing'), id='target-not-found'), + pytest.param(DownloadError('unreachable'), id='download-error'), + pytest.param(TimeoutError('slow'), id='timeout-error'), + pytest.param(urllib.error.URLError('unreachable'), id='url-error'), + ], + ) + def test_default_falls_back_to_v1_on_expected_v2_failures(self, monkeypatch, fallback_exc): + monkeypatch.setattr('sys.argv', ['downloader', 'datadog-postgres']) + monkeypatch.setattr(cli, 'run_v2_downloader', MagicMock(side_effect=fallback_exc)) + v1 = MagicMock() + monkeypatch.setattr(cli, 'run_downloader', v1) + monkeypatch.setattr(cli, 'instantiate_downloader', MagicMock(return_value=('d', 'n', 'v', False))) + + cli.download() + v1.assert_called_once_with('d', 'n', 'v', False) + + def test_default_unsafe_disable_verification_without_version_falls_back_to_v1(self, monkeypatch): + monkeypatch.setattr('sys.argv', ['downloader', 'datadog-postgres', '--unsafe-disable-verification']) + monkeypatch.setattr(cli, 'run_v2_downloader', MagicMock(side_effect=MissingVersion('missing'))) + v1 = MagicMock() + monkeypatch.setattr(cli, 'run_downloader', v1) + monkeypatch.setattr(cli, 'instantiate_downloader', MagicMock(return_value=('d', 'n', None, False))) + + cli.download() + v1.assert_called_once_with('d', 'n', None, False) + + def test_non_datadog_package_does_not_fall_back_to_v1(self, monkeypatch): + monkeypatch.setattr('sys.argv', ['downloader', 'requests']) + v1 = MagicMock() + monkeypatch.setattr(cli, 'run_downloader', v1) + monkeypatch.setattr(cli, 'instantiate_downloader', MagicMock()) + + with pytest.raises(NonDatadogPackage): + cli.download() + v1.assert_not_called() + + @pytest.mark.parametrize( + 'integrity_exc', + [ + pytest.param(DigestMismatch(PROJECT, 'a', 'b'), id='digest-mismatch'), + pytest.param(LengthMismatch(PROJECT, 1, 2), id='length-mismatch'), + pytest.param(MalformedPointerError(PROJECT, 'digest'), id='malformed-pointer'), + ], + ) + def test_integrity_errors_do_not_fall_back_to_v1(self, monkeypatch, integrity_exc): + monkeypatch.setattr('sys.argv', ['downloader', 'datadog-postgres']) + monkeypatch.setattr(cli, 'run_v2_downloader', MagicMock(side_effect=integrity_exc)) + v1 = MagicMock() + monkeypatch.setattr(cli, 'run_downloader', v1) + monkeypatch.setattr(cli, 'instantiate_downloader', MagicMock()) + + with pytest.raises(type(integrity_exc)): + cli.download() + v1.assert_not_called() From 01c70a3c0409cef7832ea4fb8908211f7eed41a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vahe=20Karamyan=20=28=D5=8E=D5=A1=D5=B0=D5=A5=20=D5=94?= =?UTF-8?q?=D5=A1=D6=80=D5=A1=D5=B4=D5=B5=D5=A1=D5=B6=29?= Date: Wed, 27 May 2026 17:33:34 +0400 Subject: [PATCH 07/44] [MOPU-312] Add related links to monitor templates (#23245) * [MOPU-288] Add related links to kubernetes monitor templates Co-Authored-By: Claude Sonnet 4.6 * [MOPU-288] Add related links to nginx monitor templates Co-Authored-By: Claude Sonnet 4.6 * [MOPU-288] Add related links to postgres and redis monitor templates Co-Authored-By: Claude Sonnet 4.6 * Fix broken Infrastructure links in monitor templates The /infrastructure?filters=... links pointed to a non-existent path with an unsupported query param, and used template variables not in each monitor's group-by. - nginx (4xx, 5xx, upstream_peer_fails): remove (upstream is not a host/pod/container resource) - k8s deployments_replicas, statefulset_replicas, pods_failed_state: remove (no host/pod template var in group-by) - k8s node_unavailable: replace with Hosts page scoped to kube_cluster_name - k8s pod_crashloopbackoff, pod_imagepullbackoff, pod_oomkilled, pods_restarting: replace with Pod Explorer scoped to pod_name Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Sonnet 4.6 --- kubernetes/assets/monitors/monitor_deployments_replicas.json | 4 ++-- kubernetes/assets/monitors/monitor_node_unavailable.json | 4 ++-- kubernetes/assets/monitors/monitor_pod_crashloopbackoff.json | 4 ++-- kubernetes/assets/monitors/monitor_pod_imagepullbackoff.json | 4 ++-- kubernetes/assets/monitors/monitor_pod_oomkilled.json | 4 ++-- kubernetes/assets/monitors/monitor_pods_failed_state.json | 4 ++-- kubernetes/assets/monitors/monitor_pods_restarting.json | 4 ++-- kubernetes/assets/monitors/monitor_statefulset_replicas.json | 4 ++-- nginx/assets/monitors/4xx.json | 4 ++-- nginx/assets/monitors/5xx.json | 4 ++-- nginx/assets/monitors/upstream_peer_fails.json | 4 ++-- postgres/assets/monitors/percent_usage_connections.json | 4 ++-- postgres/assets/monitors/replication_delay.json | 4 ++-- redisdb/assets/monitors/high_mem.json | 4 ++-- 14 files changed, 28 insertions(+), 28 deletions(-) diff --git a/kubernetes/assets/monitors/monitor_deployments_replicas.json b/kubernetes/assets/monitors/monitor_deployments_replicas.json index 39b6fb9816c5f..374a02aae8603 100644 --- a/kubernetes/assets/monitors/monitor_deployments_replicas.json +++ b/kubernetes/assets/monitors/monitor_deployments_replicas.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-07-28", - "last_updated_at": "2025-06-12", + "last_updated_at": "2026-04-09", "title": "Kubernetes Deployment Replicas are failing", "tags": [ "integration:kubernetes" ], "description": "Kubernetes replicas are clones that facilitate self-healing for pods. Each pod has a desired number of replica Pods that should be running at any given time. This monitor tracks the number of replicas that are failing per deployment.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nThere are at least 2 or more missing replicas for Deployment {{kube_namespace.name}}/{{kube_deployment.name}} over the last 15 minutes.\n\n{{/is_alert}}", + "message": "{{#is_alert}}\n\n## What's happening?\nThere are at least 2 or more missing replicas for Deployment {{kube_namespace.name}}/{{kube_deployment.name}} over the last 15 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+kube_deployment:{{kube_deployment.name}}+kube_namespace:{{kube_namespace.name}})\n- [Metrics Explorer (kubernetes_state.deployment.replicas_desired)](/metric/explorer?exp_metric=kubernetes_state.deployment.replicas_desired&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_deployment:{{kube_deployment.name}},kube_namespace:{{kube_namespace.name}}&exp_agg=avg&exp_type=line)\n- [Metrics Explorer (kubernetes_state.deployment.replicas_available)](/metric/explorer?exp_metric=kubernetes_state.deployment.replicas_available&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_deployment:{{kube_deployment.name}},kube_namespace:{{kube_namespace.name}}&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}", "name": "[Kubernetes] Monitor Kubernetes Deployments Replica Pods", "options": { "escalation_message": "", diff --git a/kubernetes/assets/monitors/monitor_node_unavailable.json b/kubernetes/assets/monitors/monitor_node_unavailable.json index 37ff9c574dcec..cc57835121156 100644 --- a/kubernetes/assets/monitors/monitor_node_unavailable.json +++ b/kubernetes/assets/monitors/monitor_node_unavailable.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-07-28", - "last_updated_at": "2025-06-12", + "last_updated_at": "2026-04-09", "title": "Nodes are unavailable", "tags": [ "integration:kubernetes" ], "description": "Kubernetes nodes can either be schedulable or unschedulable. When unschedulable, the node prevents the scheduler from placing new pods onto that node. This monitor tracks the percentage of schedulable nodes.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nThe percentage of schedulable nodes is below 80% for status:schedulable on ({{kube_cluster_name.name}} cluster over the last 15 minutes.\n\n{{/is_alert}}\n\n Keep in mind that this might be expected based on your infrastructure.", + "message": "{{#is_alert}}\n\n## What's happening?\nThe percentage of schedulable nodes is below 80% for status:schedulable on ({{kube_cluster_name.name}} cluster over the last 15 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+status:schedulable)\n- [Hosts](/infrastructure/hosts?scope=kube_cluster_name:{{kube_cluster_name.name}})\n- [Metrics Explorer (kubernetes_state.node.status)](/metric/explorer?exp_metric=kubernetes_state.node.status&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},status:schedulable&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}\n\n Keep in mind that this might be expected based on your infrastructure.", "name": "[Kubernetes] Monitor Unschedulable Kubernetes Nodes", "options": { "escalation_message": "", diff --git a/kubernetes/assets/monitors/monitor_pod_crashloopbackoff.json b/kubernetes/assets/monitors/monitor_pod_crashloopbackoff.json index 1b14f874c716a..317eec3fd0032 100644 --- a/kubernetes/assets/monitors/monitor_pod_crashloopbackoff.json +++ b/kubernetes/assets/monitors/monitor_pod_crashloopbackoff.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-07-28", - "last_updated_at": "2025-06-12", + "last_updated_at": "2026-04-09", "title": "Pod is in a CrashloopBackOff state", "tags": [ "integration:kubernetes" ], "description": "The status CrashloopBackOff means that a container in the Pod is started, crashes, and is restarted, over and over again. This monitor tracks when a pod is in a CrashloopBackOff state for your Kubernetes integration.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nAt least one container in pod {{pod_name.name}} on {{kube_namespace.name}} is in a waiting state due to reason crashloopbackoff in the last 10 minutes.\n\n{{/is_alert}}\n\n This alert could generate several alerts for a bad deployment. Adjust the thresholds of the query to suit your infrastructure.", + "message": "{{#is_alert}}\n\n## What's happening?\nAt least one container in pod {{pod_name.name}} on {{kube_namespace.name}} is in a waiting state due to reason crashloopbackoff in the last 10 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+kube_namespace:{{kube_namespace.name}}+pod_name:{{pod_name.name}}+reason:crashloopbackoff)\n- [Pod Explorer](/orchestration/explorer/pod?query={{pod_name.name}})\n- [Metrics Explorer (kubernetes_state.container.status_report.count.waiting)](/metric/explorer?exp_metric=kubernetes_state.container.status_report.count.waiting&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_namespace:{{kube_namespace.name}},pod_name:{{pod_name.name}},reason:crashloopbackoff&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}\n\n This alert could generate several alerts for a bad deployment. Adjust the thresholds of the query to suit your infrastructure.", "name": "[Kubernetes] Pod {{pod_name.name}} is CrashloopBackOff on namespace {{kube_namespace.name}}", "options": { "escalation_message": "", diff --git a/kubernetes/assets/monitors/monitor_pod_imagepullbackoff.json b/kubernetes/assets/monitors/monitor_pod_imagepullbackoff.json index 07f30a6eb7b44..a42c9be57e11d 100644 --- a/kubernetes/assets/monitors/monitor_pod_imagepullbackoff.json +++ b/kubernetes/assets/monitors/monitor_pod_imagepullbackoff.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-09-15", - "last_updated_at": "2025-06-12", + "last_updated_at": "2026-04-09", "title": "Pod is in an ImagePullBackOff state", "tags": [ "integration:kubernetes" ], "description": "The status ImagePullBackOff means that a container could not start because Kubernetes could not pull a container image. This monitor tracks when a pod is in an ImagePullBackOff state for your Kubernetes integration.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nAt least one container in pod {{pod_name.name}} on namespace {{kube_namespace.name}} is in a waiting state due to an ImagePullBackOff error in the last 10 minutes.\n\n{{/is_alert}}\n\n This could happen for several reasons, for example a bad image path or tag or if the credentials for pulling images are not configured properly.", + "message": "{{#is_alert}}\n\n## What's happening?\nAt least one container in pod {{pod_name.name}} on namespace {{kube_namespace.name}} is in a waiting state due to an ImagePullBackOff error in the last 10 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+kube_namespace:{{kube_namespace.name}}+pod_name:{{pod_name.name}}+reason:imagepullbackoff)\n- [Pod Explorer](/orchestration/explorer/pod?query={{pod_name.name}})\n- [Metrics Explorer (kubernetes_state.container.status_report.count.waiting)](/metric/explorer?exp_metric=kubernetes_state.container.status_report.count.waiting&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_namespace:{{kube_namespace.name}},pod_name:{{pod_name.name}},reason:imagepullbackoff&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}\n\n This could happen for several reasons, for example a bad image path or tag or if the credentials for pulling images are not configured properly.", "name": "[Kubernetes] Pod {{pod_name.name}} is ImagePullBackOff on namespace {{kube_namespace.name}}", "options": { "escalation_message": "", diff --git a/kubernetes/assets/monitors/monitor_pod_oomkilled.json b/kubernetes/assets/monitors/monitor_pod_oomkilled.json index 3eece4a5d9e41..e4f7ad7aa755c 100644 --- a/kubernetes/assets/monitors/monitor_pod_oomkilled.json +++ b/kubernetes/assets/monitors/monitor_pod_oomkilled.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2025-09-15", - "last_updated_at": "2025-09-15", + "last_updated_at": "2026-04-09", "title": "Pod is in an OOMKilled state", "tags": [ "integration:kubernetes" ], "description": "The status OOMKilled means that a container was killed because it exceeded memory limits or the node ran out of available memory. This monitor tracks when a pod is in an OOMKilled state for your Kubernetes integration.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nThere has been at least one container terminated in pod {{pod_name.name}} on namespace {{kube_namespace.name}} with reason oomkilled in the last 10 minutes.\n\n{{/is_alert}}\n\n This could happen for several reasons, for example insufficient memory limits, memory leaks in the application, or the node running out of available memory.", + "message": "{{#is_alert}}\n\n## What's happening?\nThere has been at least one container terminated in pod {{pod_name.name}} on namespace {{kube_namespace.name}} with reason oomkilled in the last 10 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+kube_namespace:{{kube_namespace.name}}+pod_name:{{pod_name.name}}+reason:oomkilled)\n- [Pod Explorer](/orchestration/explorer/pod?query={{pod_name.name}})\n- [Metrics Explorer (kubernetes.containers.state.terminated)](/metric/explorer?exp_metric=kubernetes.containers.state.terminated&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_namespace:{{kube_namespace.name}},pod_name:{{pod_name.name}},reason:oomkilled&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}\n\n This could happen for several reasons, for example insufficient memory limits, memory leaks in the application, or the node running out of available memory.", "name": "[Kubernetes] Pod {{pod_name.name}} is OOMKilled on namespace {{kube_namespace.name}}", "options": { "escalation_message": "", diff --git a/kubernetes/assets/monitors/monitor_pods_failed_state.json b/kubernetes/assets/monitors/monitor_pods_failed_state.json index 708a41da74ee4..33ee1b348348e 100644 --- a/kubernetes/assets/monitors/monitor_pods_failed_state.json +++ b/kubernetes/assets/monitors/monitor_pods_failed_state.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-07-28", - "last_updated_at": "2025-06-12", + "last_updated_at": "2026-04-09", "title": "Pods are failing", "tags": [ "integration:kubernetes" ], "description": "When a pod is failing it means the container either exited with non-zero status or was terminated by the system. This monitor tracks when more than 10 pods are failing for a given Kubernetes cluster.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nThe number of failed pods has increased by more than 10 in ({{kube_cluster_name.name}} cluster in the last 5 minutes.\n\n{{/is_alert}}\n\n The threshold of ten pods varies depending on your infrastructure. Change the threshold to suit your needs.", + "message": "{{#is_alert}}\n\n## What's happening?\nThe number of failed pods has increased by more than 10 in ({{kube_cluster_name.name}} cluster in the last 5 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+kube_namespace:{{kube_namespace.name}}+pod_phase:failed)\n- [Metrics Explorer (kubernetes_state.pod.status_phase)](/metric/explorer?exp_metric=kubernetes_state.pod.status_phase&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_namespace:{{kube_namespace.name}},pod_phase:failed&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}\n\n The threshold of ten pods varies depending on your infrastructure. Change the threshold to suit your needs.", "name": "[Kubernetes] Monitor Kubernetes Failed Pods in Namespaces", "options": { "escalation_message": "", diff --git a/kubernetes/assets/monitors/monitor_pods_restarting.json b/kubernetes/assets/monitors/monitor_pods_restarting.json index f35d90c629c09..c7cccced75755 100644 --- a/kubernetes/assets/monitors/monitor_pods_restarting.json +++ b/kubernetes/assets/monitors/monitor_pods_restarting.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-07-28", - "last_updated_at": "2025-06-12", + "last_updated_at": "2026-04-09", "title": "Pods are restarting", "tags": [ "integration:kubernetes" ], "description": "Kubernetes pods restart according to the restart policy. A restarting container can indicate problems with memory, CPU usage, or an application exiting prematurely. This monitor tracks when pods are restarting multiple times.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nThere has been an increase of more than 5 container restarts in the pod {{pod_name.name}} in the last 5 minutes.\n\n{{/is_alert}}", + "message": "{{#is_alert}}\n\n## What's happening?\nThere has been an increase of more than 5 container restarts in the pod {{pod_name.name}} in the last 5 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+pod_name:{{pod_name.name}})\n- [Pod Explorer](/orchestration/explorer/pod?query={{pod_name.name}})\n- [Metrics Explorer (kubernetes.containers.restarts)](/metric/explorer?exp_metric=kubernetes.containers.restarts&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},pod_name:{{pod_name.name}}&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}", "name": "[Kubernetes] Monitor Kubernetes Pods Restarting", "options": { "escalation_message": "", diff --git a/kubernetes/assets/monitors/monitor_statefulset_replicas.json b/kubernetes/assets/monitors/monitor_statefulset_replicas.json index b0954fe2785bb..ef5fbc979d832 100644 --- a/kubernetes/assets/monitors/monitor_statefulset_replicas.json +++ b/kubernetes/assets/monitors/monitor_statefulset_replicas.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-07-28", - "last_updated_at": "2025-06-12", + "last_updated_at": "2026-04-09", "title": "Kubernetes Statefulset Replicas are failing", "tags": [ "integration:kubernetes" ], "description": "Kubernetes replicas are clones that facilitate self-healing for pods. Each pod has a desired number of replica Pods that should be running at any given time. This monitor tracks when the number of replicas per statefulset is falling.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nThere are at least 2 desired replicas that are not ready for {{kube_namespace.name}}/{{kube_stateful_set.name}} StatefulSet over the last 15 minutes.\n\n{{/is_alert}}\n\n This might present an unsafe situation for any further manual operations, such as killing other pods.", + "message": "{{#is_alert}}\n\n## What's happening?\nThere are at least 2 desired replicas that are not ready for {{kube_namespace.name}}/{{kube_stateful_set.name}} StatefulSet over the last 15 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+kube_namespace:{{kube_namespace.name}}+kube_stateful_set:{{kube_stateful_set.name}})\n- [Metrics Explorer (kubernetes_state.statefulset.replicas_desired)](/metric/explorer?exp_metric=kubernetes_state.statefulset.replicas_desired&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_namespace:{{kube_namespace.name}},kube_stateful_set:{{kube_stateful_set.name}}&exp_agg=avg&exp_type=line)\n- [Metrics Explorer (kubernetes_state.statefulset.replicas_ready)](/metric/explorer?exp_metric=kubernetes_state.statefulset.replicas_ready&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_namespace:{{kube_namespace.name}},kube_stateful_set:{{kube_stateful_set.name}}&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}\n\n This might present an unsafe situation for any further manual operations, such as killing other pods.", "name": "[Kubernetes] Monitor Kubernetes Statefulset Replicas", "options": { "escalation_message": "", diff --git a/nginx/assets/monitors/4xx.json b/nginx/assets/monitors/4xx.json index 17fbd04888321..e38ee3436c7e3 100644 --- a/nginx/assets/monitors/4xx.json +++ b/nginx/assets/monitors/4xx.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-09-16", - "last_updated_at": "2026-03-09", + "last_updated_at": "2026-04-09", "title": "Upstream 4xx errors are high", "tags": [ "integration:nginx" ], "description": "NGINX sends requests to upstream peers that can fail eventually. This monitor tracks the count of 4xx HTTP responses to identify issues in the communication between NGINX and the backend servers.", "definition": { - "message": "{{#is_alert}}\n## 🚨 What's happening\n\nAn anomaly has been detected in the number of 4xx HTTP responses from NGINX upstream **{{upstream.name}}** (anomaly score: `{{value}}`, threshold: `{{threshold}}`). The 4xx response rate is significantly higher than normal, indicating that a notable portion of incoming requests are being rejected with client-side error codes.\n\nFirst triggered at **{{first_triggered_at}}**, active for **{{triggered_duration_sec}}** seconds.\n{{/is_alert}}{{#is_recovery}}\n## ✅ Recovered\n\nThe 4xx anomaly for upstream **{{upstream.name}}** has resolved. Current value: `{{value}}`.\n{{/is_recovery}}\n{{^is_recovery}}\n***\n\n## 📈 Impact\n\nElevated 4xx error rates can result in failed requests for end users and may expose misconfigurations or broken routes. Services and clients relying on this NGINX upstream may experience partial or complete degradation of functionality.\n\n***\n\n## Runbook\n\n### Initial Troubleshooting Steps\n\n1. **Identify the affected upstream** from the alert (`{{upstream.name}}`).\n2. Open [**Metrics Explorer**](/metric/explorer) and inspect `nginx.upstream.peers.responses.4xx` broken down by `upstream`.\n3. Review NGINX access logs for specific endpoints and status codes:\n ```bash\n tail -f /var/log/nginx/access.log | grep \" 4[0-9][0-9] \"\n ```\n4. Correlate the spike with recent configuration changes, upstream deployments, or traffic shifts.\n\n### Cause and Resolution\n\n| Cause | Resolution |\n| ----- | ---------- |\n| Invalid or removed request paths (404) | Verify routes in NGINX configuration; update upstream routing rules to reflect the current backend state. |\n| Authentication or authorization failures (401/403) | Review auth configuration; check if credentials or access tokens have expired or been revoked. |\n| Malformed client requests (400) | Inspect incoming request headers and payloads; check client-side request construction. |\n| Rate limiting triggered (429) | Review rate limit thresholds; consider scaling upstream services or relaxing limits. |\n| Upstream endpoints renamed or removed | Update NGINX upstream configuration to reflect the current backend service endpoints. |\n\n### Related links\n\n* [Documentation](https://docs.datadoghq.com/integrations/nginx/)\n* [Metrics Explorer](/metric/explorer)\n* [Log Explorer](/logs?query=source%3Anginx)\n\n### Who should be notified?\n\nAssign the appropriate notification handle for this alert (e.g., `@slack-infra`, `@pagerduty-nginx`):\n`@your-team-handle`\n{{/is_recovery}}", + "message": "{{#is_alert}}\n## 🚨 What's happening\n\nAn anomaly has been detected in the number of 4xx HTTP responses from NGINX upstream **{{upstream.name}}** (anomaly score: `{{value}}`, threshold: `{{threshold}}`). The 4xx response rate is significantly higher than normal, indicating that a notable portion of incoming requests are being rejected with client-side error codes.\n\nFirst triggered at **{{first_triggered_at}}**, active for **{{triggered_duration_sec}}** seconds.\n{{/is_alert}}{{#is_recovery}}\n## ✅ Recovered\n\nThe 4xx anomaly for upstream **{{upstream.name}}** has resolved. Current value: `{{value}}`.\n{{/is_recovery}}\n{{^is_recovery}}\n***\n\n## 📈 Impact\n\nElevated 4xx error rates can result in failed requests for end users and may expose misconfigurations or broken routes. Services and clients relying on this NGINX upstream may experience partial or complete degradation of functionality.\n\n***\n\n## Runbook\n\n### Initial Troubleshooting Steps\n\n1. **Identify the affected upstream** from the alert (`{{upstream.name}}`).\n2. Open [**Metrics Explorer**](/metric/explorer) and inspect `nginx.upstream.peers.responses.4xx` broken down by `upstream`.\n3. Review NGINX access logs for specific endpoints and status codes:\n ```bash\n tail -f /var/log/nginx/access.log | grep \" 4[0-9][0-9] \"\n ```\n4. Correlate the spike with recent configuration changes, upstream deployments, or traffic shifts.\n\n### Cause and Resolution\n\n| Cause | Resolution |\n| ----- | ---------- |\n| Invalid or removed request paths (404) | Verify routes in NGINX configuration; update upstream routing rules to reflect the current backend state. |\n| Authentication or authorization failures (401/403) | Review auth configuration; check if credentials or access tokens have expired or been revoked. |\n| Malformed client requests (400) | Inspect incoming request headers and payloads; check client-side request construction. |\n| Rate limiting triggered (429) | Review rate limit thresholds; consider scaling upstream services or relaxing limits. |\n| Upstream endpoints renamed or removed | Update NGINX upstream configuration to reflect the current backend service endpoints. |\n\n### Related links\n\n* [Documentation](https://docs.datadoghq.com/integrations/nginx/)\n* [Logs](/logs?query=upstream:{{upstream.name}})\n* [Metrics Explorer (nginx.upstream.peers.responses.4xx)](/metric/explorer?exp_metric=nginx.upstream.peers.responses.4xx&exp_scope=upstream:{{upstream.name}}&exp_agg=avg&exp_type=line)\n\n### Who should be notified?\n\nAssign the appropriate notification handle for this alert (e.g., `@slack-infra`, `@pagerduty-nginx`):\n`@your-team-handle`\n{{/is_recovery}}", "name": "[NGINX] 4xx Errors higher than usual", "options": { "escalation_message": "", diff --git a/nginx/assets/monitors/5xx.json b/nginx/assets/monitors/5xx.json index c7b9ef7201dbc..b98d0bf985336 100644 --- a/nginx/assets/monitors/5xx.json +++ b/nginx/assets/monitors/5xx.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-09-16", - "last_updated_at": "2026-03-09", + "last_updated_at": "2026-04-09", "title": "Upstream 5xx errors are high", "tags": [ "integration:nginx" ], "description": "“5xx upstream request errors” are indicating server issues from backend servers. This monitor tracks the count of 5xx responses from NGINX's upstream peers to identify server-related issues in your web or application infrastructure.", "definition": { - "message": "{{#is_alert}}\n## 🚨 What's happening\n\nAn anomaly has been detected in the number of 5xx HTTP responses from NGINX upstream **{{upstream.name}}** (anomaly score: `{{value}}`, threshold: `{{threshold}}`). The 5xx error rate is significantly higher than normal, indicating that backend servers are failing to handle a notable portion of requests.\n\nFirst triggered at **{{first_triggered_at}}**, active for **{{triggered_duration_sec}}** seconds.\n{{/is_alert}}{{#is_recovery}}\n## ✅ Recovered\n\nThe 5xx anomaly for upstream **{{upstream.name}}** has resolved. Current value: `{{value}}`.\n{{/is_recovery}}\n{{^is_recovery}}\n***\n\n## 📈 Impact\n\n5xx errors indicate server-side failures that cause direct service disruptions for users. Dependent services that rely on successful responses from this NGINX upstream may experience cascading failures or degraded functionality.\n\n***\n\n## Runbook\n\n### Initial Troubleshooting Steps\n\n1. **Identify the affected upstream** from the alert (`{{upstream.name}}`).\n2. Open [**Metrics Explorer**](/metric/explorer) and inspect `nginx.upstream.peers.responses.5xx` broken down by `upstream`.\n3. Review NGINX error logs for connection failures or backend errors:\n ```bash\n tail -f /var/log/nginx/error.log\n ```\n4. Check upstream backend service health and application logs.\n5. Correlate the spike with recent deployments or infrastructure changes.\n\n### Cause and Resolution\n\n| Cause | Resolution |\n| ----- | ---------- |\n| Backend server is down or crashed (502) | Verify the upstream service is running; restart the service if needed and check its logs. |\n| Gateway timeout due to slow upstream (504) | Check upstream response times; increase `proxy_read_timeout` if the upstream is legitimately slow. |\n| Application-level errors (500) | Inspect upstream application logs for unhandled exceptions or crashes; roll back recent deployments if correlated. |\n| Service unavailable due to overload (503) | Check upstream server resource utilization; scale out or enable load balancing across more peers. |\n| Resource exhaustion on upstream servers | Review CPU, memory, and connection pool usage on the backend; tune resource limits and autoscaling. |\n\n### Related links\n\n* [Documentation](https://docs.datadoghq.com/integrations/nginx/)\n* [Metrics Explorer](/metric/explorer)\n* [Log Explorer](/logs?query=source%3Anginx)\n\n### Who should be notified?\n\nAssign the appropriate notification handle for this alert (e.g., `@slack-infra`, `@pagerduty-nginx`):\n`@your-team-handle`\n{{/is_recovery}}", + "message": "{{#is_alert}}\n## 🚨 What's happening\n\nAn anomaly has been detected in the number of 5xx HTTP responses from NGINX upstream **{{upstream.name}}** (anomaly score: `{{value}}`, threshold: `{{threshold}}`). The 5xx error rate is significantly higher than normal, indicating that backend servers are failing to handle a notable portion of requests.\n\nFirst triggered at **{{first_triggered_at}}**, active for **{{triggered_duration_sec}}** seconds.\n{{/is_alert}}{{#is_recovery}}\n## ✅ Recovered\n\nThe 5xx anomaly for upstream **{{upstream.name}}** has resolved. Current value: `{{value}}`.\n{{/is_recovery}}\n{{^is_recovery}}\n***\n\n## 📈 Impact\n\n5xx errors indicate server-side failures that cause direct service disruptions for users. Dependent services that rely on successful responses from this NGINX upstream may experience cascading failures or degraded functionality.\n\n***\n\n## Runbook\n\n### Initial Troubleshooting Steps\n\n1. **Identify the affected upstream** from the alert (`{{upstream.name}}`).\n2. Open [**Metrics Explorer**](/metric/explorer) and inspect `nginx.upstream.peers.responses.5xx` broken down by `upstream`.\n3. Review NGINX error logs for connection failures or backend errors:\n ```bash\n tail -f /var/log/nginx/error.log\n ```\n4. Check upstream backend service health and application logs.\n5. Correlate the spike with recent deployments or infrastructure changes.\n\n### Cause and Resolution\n\n| Cause | Resolution |\n| ----- | ---------- |\n| Backend server is down or crashed (502) | Verify the upstream service is running; restart the service if needed and check its logs. |\n| Gateway timeout due to slow upstream (504) | Check upstream response times; increase `proxy_read_timeout` if the upstream is legitimately slow. |\n| Application-level errors (500) | Inspect upstream application logs for unhandled exceptions or crashes; roll back recent deployments if correlated. |\n| Service unavailable due to overload (503) | Check upstream server resource utilization; scale out or enable load balancing across more peers. |\n| Resource exhaustion on upstream servers | Review CPU, memory, and connection pool usage on the backend; tune resource limits and autoscaling. |\n\n### Related links\n\n* [Documentation](https://docs.datadoghq.com/integrations/nginx/)\n* [Logs](/logs?query=upstream:{{upstream.name}})\n* [Metrics Explorer (nginx.upstream.peers.responses.5xx)](/metric/explorer?exp_metric=nginx.upstream.peers.responses.5xx&exp_scope=upstream:{{upstream.name}}&exp_agg=avg&exp_type=line)\n\n### Who should be notified?\n\nAssign the appropriate notification handle for this alert (e.g., `@slack-infra`, `@pagerduty-nginx`):\n`@your-team-handle`\n{{/is_recovery}}", "name": "[NGINX] 5xx Errors higher than usual", "options": { "escalation_message": "", diff --git a/nginx/assets/monitors/upstream_peer_fails.json b/nginx/assets/monitors/upstream_peer_fails.json index 08cef0b5647dd..d8f4b0fb003d1 100644 --- a/nginx/assets/monitors/upstream_peer_fails.json +++ b/nginx/assets/monitors/upstream_peer_fails.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-09-16", - "last_updated_at": "2026-03-09", + "last_updated_at": "2026-04-09", "title": "Upstream peers are failing", "tags": [ "integration:nginx" ], "description": "NGINX can be configured to distribute incoming client requests to multiple upstream peers (individual web servers, application servers, or other backend services). This monitor tracks anomalies in the number of failed upstream peers to identify issues.", "definition": { - "message": "{{#is_alert}}\n## 🚨 What's happening\n\nAn anomaly has been detected in the number of upstream peer communication failures for **{{upstream.name}}** (anomaly score: `{{value}}`, threshold: `{{threshold}}`). NGINX is experiencing an unusual number of unsuccessful attempts to connect to or communicate with one or more backend servers.\n\nFirst triggered at **{{first_triggered_at}}**, active for **{{triggered_duration_sec}}** seconds.\n{{/is_alert}}{{#is_recovery}}\n## ✅ Recovered\n\nUpstream peer failures for **{{upstream.name}}** have resolved. Current value: `{{value}}`.\n{{/is_recovery}}\n{{^is_recovery}}\n***\n\n## 📈 Impact\n\nUpstream peer failures reduce the pool of available backend servers, increasing load on healthy peers. Users may experience intermittent errors or increased response times as NGINX retries or routes traffic around failed peers.\n\n***\n\n## Runbook\n\n### Initial Troubleshooting Steps\n\n1. **Identify the affected upstream** from the alert (`{{upstream.name}}`).\n2. Open [**Metrics Explorer**](/metric/explorer) and inspect `nginx.stream.upstream.peers.fails` broken down by `upstream` to identify which specific peers are failing.\n3. Review NGINX error logs for connection-level failures:\n ```bash\n tail -f /var/log/nginx/error.log | grep \"upstream\"\n ```\n4. Test connectivity from the NGINX host to the failing upstream servers:\n ```bash\n curl -v http://:/health\n ```\n5. Correlate with recent configuration changes or upstream service deployments.\n\n### Cause and Resolution\n\n| Cause | Resolution |\n| ----- | ---------- |\n| Upstream server is down or crashed | Verify the upstream service is running and listening on the expected port; restart if needed. |\n| Network connectivity issues | Test connectivity from the NGINX host to the upstream; check firewall rules and network routing. |\n| Upstream not responding within timeout | Review `proxy_connect_timeout` and `proxy_read_timeout` in NGINX config; increase if the upstream is legitimately slow. |\n| Misconfigured upstream address or port | Verify the upstream block in NGINX configuration has the correct server addresses and ports. |\n| Firewall or security group blocking traffic | Check security group rules and host-based firewall (iptables/nftables) on the upstream servers. |\n\n### Related links\n\n* [Documentation](https://docs.datadoghq.com/integrations/nginx/)\n* [Metrics Explorer](/metric/explorer)\n* [Log Explorer](/logs?query=source%3Anginx)\n\n### Who should be notified?\n\nAssign the appropriate notification handle for this alert (e.g., `@slack-infra`, `@pagerduty-nginx`):\n`@your-team-handle`\n{{/is_recovery}}", + "message": "{{#is_alert}}\n## 🚨 What's happening\n\nAn anomaly has been detected in the number of upstream peer communication failures for **{{upstream.name}}** (anomaly score: `{{value}}`, threshold: `{{threshold}}`). NGINX is experiencing an unusual number of unsuccessful attempts to connect to or communicate with one or more backend servers.\n\nFirst triggered at **{{first_triggered_at}}**, active for **{{triggered_duration_sec}}** seconds.\n{{/is_alert}}{{#is_recovery}}\n## ✅ Recovered\n\nUpstream peer failures for **{{upstream.name}}** have resolved. Current value: `{{value}}`.\n{{/is_recovery}}\n{{^is_recovery}}\n***\n\n## 📈 Impact\n\nUpstream peer failures reduce the pool of available backend servers, increasing load on healthy peers. Users may experience intermittent errors or increased response times as NGINX retries or routes traffic around failed peers.\n\n***\n\n## Runbook\n\n### Initial Troubleshooting Steps\n\n1. **Identify the affected upstream** from the alert (`{{upstream.name}}`).\n2. Open [**Metrics Explorer**](/metric/explorer) and inspect `nginx.stream.upstream.peers.fails` broken down by `upstream` to identify which specific peers are failing.\n3. Review NGINX error logs for connection-level failures:\n ```bash\n tail -f /var/log/nginx/error.log | grep \"upstream\"\n ```\n4. Test connectivity from the NGINX host to the failing upstream servers:\n ```bash\n curl -v http://:/health\n ```\n5. Correlate with recent configuration changes or upstream service deployments.\n\n### Cause and Resolution\n\n| Cause | Resolution |\n| ----- | ---------- |\n| Upstream server is down or crashed | Verify the upstream service is running and listening on the expected port; restart if needed. |\n| Network connectivity issues | Test connectivity from the NGINX host to the upstream; check firewall rules and network routing. |\n| Upstream not responding within timeout | Review `proxy_connect_timeout` and `proxy_read_timeout` in NGINX config; increase if the upstream is legitimately slow. |\n| Misconfigured upstream address or port | Verify the upstream block in NGINX configuration has the correct server addresses and ports. |\n| Firewall or security group blocking traffic | Check security group rules and host-based firewall (iptables/nftables) on the upstream servers. |\n\n### Related links\n\n* [Documentation](https://docs.datadoghq.com/integrations/nginx/)\n* [Logs](/logs?query=upstream:{{upstream.name}})\n* [Metrics Explorer (nginx.stream.upstream.peers.fails)](/metric/explorer?exp_metric=nginx.stream.upstream.peers.fails&exp_scope=upstream:{{upstream.name}}&exp_agg=avg&exp_type=line)\n\n### Who should be notified?\n\nAssign the appropriate notification handle for this alert (e.g., `@slack-infra`, `@pagerduty-nginx`):\n`@your-team-handle`\n{{/is_recovery}}", "name": "[NGINX] Upstream peers fails", "options": { "escalation_message": "", diff --git a/postgres/assets/monitors/percent_usage_connections.json b/postgres/assets/monitors/percent_usage_connections.json index bfa477bbe4e8a..01608f00c12bb 100644 --- a/postgres/assets/monitors/percent_usage_connections.json +++ b/postgres/assets/monitors/percent_usage_connections.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2021-03-17", - "last_updated_at": "2023-07-24", + "last_updated_at": "2026-04-09", "title": "Connection pool is reaching saturation point", "tags": [ "integration:postgres" ], "description": "In PostgreSQL, there is a limit of concurrent connections that can be increased. When this limit is exceeded, new users cannot establish a connection with the database. This monitor tracks the total number of connections.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nPostgreSQL connection usage on host {{host.name}} has exceeded 90% of the maximum allowed connections over the last 15 minutes.\n\n{{/is_alert}}", + "message": "{{#is_alert}}\n\n## What's happening?\nPostgreSQL connection usage on host {{host.name}} has exceeded 90% of the maximum allowed connections over the last 15 minutes.\n\n## Related Links\n\n- [Metrics Explorer (postgresql.percent_usage_connections)](/metric/explorer?exp_metric=postgresql.percent_usage_connections&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}", "name": "[Postgres] Number of connections is approaching connection limit on {{host.name}}", "options": { "escalation_message": "", diff --git a/postgres/assets/monitors/replication_delay.json b/postgres/assets/monitors/replication_delay.json index 889700af13e39..6ec45e4efe1c5 100644 --- a/postgres/assets/monitors/replication_delay.json +++ b/postgres/assets/monitors/replication_delay.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2021-02-16", - "last_updated_at": "2021-03-17", + "last_updated_at": "2026-04-09", "title": "Replication delay is high", "tags": [ "integration:postgres" ], "description": "Replication lag is the delay between the time when data is written to the primary database and the time when it is replicated to the standby databases. This monitor tracks the replication lag of the postgres database.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nAnomalies in replication delay on host {{host.name}} for PostgreSQL have been detected above the expected range within the past 15 minutes, over the last hour.\n\n{{/is_alert}}", + "message": "{{#is_alert}}\n\n## What's happening?\nAnomalies in replication delay on host {{host.name}} for PostgreSQL have been detected above the expected range within the past 15 minutes, over the last hour.\n\n## Related Links\n\n- [Metrics Explorer (postgresql.replication_delay)](/metric/explorer?exp_metric=postgresql.replication_delay&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}", "name": "[Postgres] Replication delay is abnormally high on {{host.name}}", "options": { "escalation_message": "", diff --git a/redisdb/assets/monitors/high_mem.json b/redisdb/assets/monitors/high_mem.json index b230aa497a55d..a360246d126a6 100644 --- a/redisdb/assets/monitors/high_mem.json +++ b/redisdb/assets/monitors/high_mem.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2021-02-08", - "last_updated_at": "2021-02-08", + "last_updated_at": "2026-04-09", "title": "Memory consumption is high", "tags": [ "integration:redis" ], "description": "Redis servers use RAM to store data and memory is a critical resource for its performance. This monitor tracks the percentage of used memory to avoid the risk of running out of memory, which can lead to performance issues.", "definition": { - "message": "## What's happening?\n{{#is_alert}}\nRedis memory usage has exceeded 90% of its allocated limit in the last 5 minutes with current value of {{value}}.\n{{/is_alert}} \n\n{{#is_warning}}\nRedis memory usage has exceeded 70% of its allocated limit in the last 5 minutes with current value of {{value}}.\n{{/is_warning}}", + "message": "## What's happening?\n{{#is_alert}}\nRedis memory usage has exceeded 90% of its allocated limit in the last 5 minutes with current value of {{value}}.\n{{/is_alert}} \n\n{{#is_warning}}\nRedis memory usage has exceeded 70% of its allocated limit in the last 5 minutes with current value of {{value}}.\n{{/is_warning}}\n\n## Related Links\n\n- [Metrics Explorer (redis.mem.used)](/metric/explorer?exp_metric=redis.mem.used&exp_agg=avg&exp_type=line)\n- [Metrics Explorer (redis.mem.maxmemory)](/metric/explorer?exp_metric=redis.mem.maxmemory&exp_agg=avg&exp_type=line)", "name": "[Redis] High memory consumption", "options": { "escalation_message": "", From 8b1901e670f94eb3544eb1c3b624c37a0854aec2 Mon Sep 17 00:00:00 2001 From: Piotr WOLSKI Date: Wed, 27 May 2026 11:24:45 -0600 Subject: [PATCH 08/44] Remove message reading from kafka_consumer (#23842) * Remove live messages reading from kafka_consumer This functionality moved to the kafka_actions integration. Co-Authored-By: Claude Opus 4.7 (1M context) * Add changelog entry Co-Authored-By: Claude Opus 4.7 (1M context) * Fix lint Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- kafka_consumer/changelog.d/23842.removed | 1 + .../datadog_checks/kafka_consumer/client.py | 16 - .../datadog_checks/kafka_consumer/config.py | 67 -- .../kafka_consumer/kafka_consumer.py | 433 --------- kafka_consumer/pyproject.toml | 2 - kafka_consumer/tests/test_integration.py | 77 -- kafka_consumer/tests/test_unit.py | 865 +----------------- 7 files changed, 2 insertions(+), 1459 deletions(-) create mode 100644 kafka_consumer/changelog.d/23842.removed diff --git a/kafka_consumer/changelog.d/23842.removed b/kafka_consumer/changelog.d/23842.removed new file mode 100644 index 0000000000000..9969ddeefc6a1 --- /dev/null +++ b/kafka_consumer/changelog.d/23842.removed @@ -0,0 +1 @@ +Remove the Data Streams live messages reading feature, which has moved to the kafka_actions integration. diff --git a/kafka_consumer/datadog_checks/kafka_consumer/client.py b/kafka_consumer/datadog_checks/kafka_consumer/client.py index 2beb6d8ef2b47..d26c81beed385 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/client.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/client.py @@ -270,22 +270,6 @@ def list_consumer_group_offsets(self, groups): offsets.append((response_offset_info.group_id, tpo)) return offsets - def start_collecting_messages(self, start_offsets, consumer_group): - self.open_consumer(consumer_group) - self._consumer.assign(start_offsets) - - def get_next_message(self): - return self._consumer.poll(timeout=1) - - def delete_consumer_group(self, consumer_group): - """Delete a consumer group using the AdminClient.""" - try: - future = self.kafka_client.delete_consumer_groups([consumer_group]) - future[consumer_group].result(timeout=self.config._request_timeout) - self.log.debug("Successfully deleted consumer group: %s", consumer_group) - except Exception as e: - self.log.warning("Failed to delete consumer group %s: %s", consumer_group, e) - def describe_consumer_group(self, consumer_group): desc = self.kafka_client.describe_consumer_groups([consumer_group])[consumer_group].result() return desc.state.name diff --git a/kafka_consumer/datadog_checks/kafka_consumer/config.py b/kafka_consumer/datadog_checks/kafka_consumer/config.py index 041d8be414f59..d296d30405850 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/config.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/config.py @@ -84,9 +84,6 @@ def __init__(self, init_config, instance, log) -> None: self._sasl_oauth_token_provider.get("tls_ca_cert") if self._sasl_oauth_token_provider else None ) - # Data Streams live messages - self.live_messages_configs = instance.get('live_messages_configs', []) - self._kafka_cluster_id_override = instance.get('kafka_cluster_id_override') self._auto_detected_cluster_id = "" @@ -216,70 +213,6 @@ def validate_config(self): ) self._validate_consumer_groups() - self._validate_live_messages_configs() - - def _validate_live_messages_configs(self): - live_messages_configs = [] - for config in self.live_messages_configs: - if 'id' not in config: - self.log.debug('Data Streams live messages configuration has no ID') - continue - kafka = config.get('kafka', None) - if not kafka: - self.log.debug('Data Streams live messages configuration has no kafka configuration') - continue - if not ( - 'cluster' in kafka - and 'topic' in kafka - and 'partition' in kafka - and 'start_offset' in kafka - and 'n_messages' in kafka - ): - self.log.debug('Data Streams live messages configuration missing required kafka parameters.', kafka) - continue - - # Validate value format - if kafka.get('value_format', '') == '': - kafka['value_format'] = 'json' - value_format = kafka['value_format'] - if value_format not in ['json', 'avro', 'protobuf']: - self.log.debug( - 'Unsupported value format for Data Streams live messages, got %s. ' - 'Supported formats: json, avro, protobuf', - value_format, - ) - continue - - # Validate key format - if kafka.get('key_format', '') == '': - kafka['key_format'] = 'json' - key_format = kafka['key_format'] - if key_format not in ['json', 'avro', 'protobuf']: - self.log.debug( - 'Unsupported key format for Data Streams live messages, got %s. ' - 'Supported formats: json, avro, protobuf', - key_format, - ) - continue - - # Validate schemas for non-JSON formats - if value_format in ['avro', 'protobuf']: - if 'value_schema' not in kafka or not kafka['value_schema']: - self.log.debug( - 'Value schema is required for %s format in Data Streams live messages configuration', - value_format, - ) - continue - - if key_format in ['avro', 'protobuf']: - if 'key_schema' not in kafka or not kafka['key_schema']: - self.log.debug( - 'Key schema is required for %s format in Data Streams live messages configuration', key_format - ) - continue - - live_messages_configs.append(config) - self.live_messages_configs = live_messages_configs def _compile_regex(self, consumer_groups_regex, consumer_groups): # Turn the dict of regex dicts into a single string and compile diff --git a/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py b/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py index 8f0fe0effbd0a..3fe6a81bab830 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py @@ -1,18 +1,10 @@ # (C) Datadog, Inc. 2019-present # All rights reserved # Licensed under Simplified BSD License (see LICENSE) -import base64 import json from collections import defaultdict -from io import BytesIO from time import time -from confluent_kafka import TopicPartition -from fastavro import schemaless_reader -from google.protobuf import descriptor_pb2, descriptor_pool, message_factory -from google.protobuf.json_format import MessageToJson -from google.protobuf.message import DecodeError, EncodeError - from datadog_checks.base import AgentCheck from datadog_checks.kafka_consumer.client import KafkaClient from datadog_checks.kafka_consumer.cluster_metadata import ClusterMetadataCollector @@ -24,8 +16,6 @@ ) MAX_TIMESTAMPS = 1000 -SCHEMA_REGISTRY_MAGIC_BYTE = 0x00 -DATA_STREAMS_MESSAGES_CACHE_KEY = 'get_messages_cache' class KafkaCheck(AgentCheck): @@ -126,7 +116,6 @@ def check(self, _): broker_timestamps, cluster_id, ) - self.data_streams_live_message(highwater_offsets or {}, cluster_id) # Collect cluster metadata if enabled if self.config._cluster_monitoring_enabled: @@ -233,30 +222,6 @@ def _load_broker_timestamps(self, persistent_cache_key): self.log.warning('Could not read broker timestamps from cache: %s', str(e)) return broker_timestamps - def _messages_have_been_retrieved(self, config_id): - """Check if messages have been retrieved for the given config ID.""" - try: - content = self.read_persistent_cache(DATA_STREAMS_MESSAGES_CACHE_KEY) - if content: - config_ids = set(content.split(",")) - return config_id in config_ids - except Exception as e: - self.log.warning('Could not read persistent cache: %s', str(e)) - return False - - def _mark_messages_retrieved(self, config_id): - """Mark that messages have been retrieved for the given config ID.""" - try: - content = self.read_persistent_cache(DATA_STREAMS_MESSAGES_CACHE_KEY) - if content: - config_ids = set(content.split(",")) - else: - config_ids = set() - config_ids.add(config_id) - self.write_persistent_cache(DATA_STREAMS_MESSAGES_CACHE_KEY, ",".join(config_ids)) - except Exception as e: - self.log.warning('Could not write to persistent cache: %s', str(e)) - def _add_broker_timestamps(self, broker_timestamps, highwater_offsets): for (topic, partition), highwater_offset in highwater_offsets.items(): timestamps = broker_timestamps["{}_{}".format(topic, partition)] @@ -458,128 +423,6 @@ def send_event(self, title, text, tags, event_type, aggregation_key, severity='i } self.event(event_dict) - def data_streams_live_message(self, highwater_offsets, cluster_id): - monitored_topics = None - for cfg in self.config.live_messages_configs: - monitored_topics = monitored_topics or {topic.lower() for (topic, _) in highwater_offsets.keys()} - kafka = cfg['kafka'] - topic = kafka["topic"] - partition = kafka["partition"] - start_offset = kafka["start_offset"] - n_messages = kafka["n_messages"] - cluster = kafka["cluster"] - config_id = cfg["id"] - value_format = kafka["value_format"] - value_schema_str = kafka.get("value_schema", "") - value_uses_schema_registry = kafka.get("value_uses_schema_registry", False) - key_format = kafka["key_format"] - key_schema_str = kafka.get("key_schema", "") - key_uses_schema_registry = kafka.get("key_uses_schema_registry", False) - if self._messages_have_been_retrieved(config_id): - continue - if not cluster or not cluster_id or cluster.lower() != cluster_id.lower(): - continue - if topic.lower() not in monitored_topics: - self.log.debug('Skipping live messages for topic %s because it is not monitored by this check', topic) - continue - start_offsets = resolve_start_offsets(highwater_offsets, topic, partition, start_offset, n_messages) - - if not start_offsets: - self.log.warning('Unable to get a list of partitions to read from for live messages') - self.send_log( - { - 'timestamp': int(time()), - 'config_id': config_id, - 'technology': 'kafka', - 'cluster': str(cluster), - 'topic': str(topic), - 'live_messages_error': 'Unable to list partitions to read from', - 'message': "Unable to list partitions to read from", - 'feature': 'data_streams_messages', - } - ) - continue - - try: - value_schema, key_schema = ( - build_schema(value_format, value_schema_str), - build_schema(key_format, key_schema_str), - ) - except ( - ValueError, - json.JSONDecodeError, - base64.binascii.Error, - IndexError, - KeyError, - TypeError, - DecodeError, - EncodeError, - ) as e: - self.log.error( - "Failed to build schemas for config_id: %s, topic: %s, partition: %s. Error: %s", - config_id, - topic, - partition, - e, - ) - continue - - consumer_group = f"datadog_messages_{config_id}" - self.client.start_collecting_messages(start_offsets, consumer_group) - try: - for _ in range(n_messages): - message = self.client.get_next_message() - if message is None: - self.log.debug('Live messages: no message to retrieve') - self.send_log( - { - 'timestamp': int(time()), - 'config_id': config_id, - 'technology': 'kafka', - 'cluster': str(cluster), - 'topic': str(topic), - 'live_messages_error': 'No more messages to retrieve', - 'message': "No more messages to retrieve", - 'feature': 'data_streams_messages', - } - ) - break - data = { - 'timestamp': int(time()), - 'technology': 'kafka', - 'cluster': str(cluster), - 'config_id': config_id, - 'topic': str(topic), - 'partition': str(message.partition()), - 'offset': str(message.offset()), - 'feature': 'data_streams_messages', - } - decoded_value, value_schema_id, decoded_key, key_schema_id = deserialize_message( - message, - value_format, - value_schema, - value_uses_schema_registry, - key_format, - key_schema, - key_uses_schema_registry, - ) - if decoded_value: - data['message_value'] = decoded_value - else: - data['message'] = "Message format not supported" - data['live_messages_error'] = 'Message format not supported' - if value_schema_id: - data['value_schema_id'] = str(value_schema_id) - if decoded_key: - data['message_key'] = decoded_key - if key_schema_id: - data['key_schema_id'] = str(key_schema_id) - self.send_log(data) - finally: - self.client.close_consumer() - self.client.delete_consumer_group(consumer_group) - self._mark_messages_retrieved(config_id) - def _get_interpolated_timestamp(timestamps, offset): if offset in timestamps: @@ -608,279 +451,3 @@ def _get_interpolated_timestamp(timestamps, offset): slope = (timestamp_after - timestamp_before) / float(offset_after - offset_before) timestamp = slope * (offset - offset_after) + timestamp_after return timestamp - - -def resolve_start_offsets(highwater_offsets, target_topic, target_partition, start_offset, n_messages): - if int(target_partition) == -1: - # in this case, we get n_messages, starting at offset latest - n_messages on each partition. - # this doesn't match exactly to the latest messages, but if we don't do that, we could run into - # edge cases when some partitions don't get any traffic. - start_offsets = [] - for topic, partition in highwater_offsets: - if topic == target_topic and highwater_offsets[(topic, partition)] >= 0: - start_offsets.append( - TopicPartition(topic, partition, max(0, highwater_offsets[(topic, partition)] - n_messages + 1)) - ) - if len(start_offsets) >= n_messages: - break - return start_offsets - if int(start_offset) == -1: - end_offset = highwater_offsets.get((target_topic, target_partition), -1) - return ( - [] - if end_offset < 0 - else [TopicPartition(target_topic, target_partition, max(0, end_offset - n_messages + 1))] - ) - return [TopicPartition(target_topic, target_partition, start_offset)] - - -def deserialize_message( - message, - value_format, - value_schema, - value_uses_schema_registry, - key_format, - key_schema, - key_uses_schema_registry, -): - try: - decoded_value, value_schema_id = _deserialize_bytes_maybe_schema_registry( - message.value(), value_format, value_schema, value_uses_schema_registry - ) - except (UnicodeDecodeError, json.JSONDecodeError, ValueError): - return None, None, None, None - try: - decoded_key, key_schema_id = _deserialize_bytes_maybe_schema_registry( - message.key(), key_format, key_schema, key_uses_schema_registry - ) - return decoded_value, value_schema_id, decoded_key, key_schema_id - except (UnicodeDecodeError, json.JSONDecodeError, ValueError): - return decoded_value, value_schema_id, None, None - - -def _read_varint(data): - shift = 0 - result = 0 - bytes_read = 0 - - for byte in data: - bytes_read += 1 - result |= (byte & 0x7F) << shift - if (byte & 0x80) == 0: - return result, bytes_read - shift += 7 - - raise ValueError("Incomplete varint") - - -def _read_protobuf_message_indices(payload): - """ - Read the Confluent Protobuf message indices array. - - The Confluent Protobuf wire format includes message indices after the schema ID: - [message_indices_length:varint][message_indices:varint...] - - The indices indicate which message type to use from the .proto schema. - For example, [0] = first message, [1] = second message, [0, 0] = nested message. - - Args: - payload: bytes after the schema ID - - Returns: - tuple: (message_indices list, remaining payload bytes) - """ - array_len, bytes_read = _read_varint(payload) - payload = payload[bytes_read:] - - indices = [] - for _ in range(array_len): - index, bytes_read = _read_varint(payload) - indices.append(index) - payload = payload[bytes_read:] - - return indices, payload - - -def _deserialize_bytes_maybe_schema_registry(message, message_format, schema, uses_schema_registry): - if not message: - return "", None - if uses_schema_registry: - return _deserialize_bytes(message, message_format, schema, True) - else: - # Fallback behavior: try without schema registry format first, then with it - try: - return _deserialize_bytes(message, message_format, schema, False) - except (UnicodeDecodeError, json.JSONDecodeError, ValueError): - return _deserialize_bytes(message, message_format, schema, True) - - -def _deserialize_bytes(message, message_format, schema, uses_schema_registry): - """Deserialize a message from Kafka. - Args: - message: Raw message bytes from Kafka - message_format: Format of the message (protobuf, avro, json, etc.) - schema: Schema object (type depends on message_format) - uses_schema_registry: Whether message uses schema registry format - Returns: - Tuple of (decoded_message, schema_id) where schema_id is None if not using schema registry - """ - if not message: - return "", None - - schema_id = None - if uses_schema_registry: - if len(message) < 5 or message[0] != SCHEMA_REGISTRY_MAGIC_BYTE: - msg_hex = message[:5].hex() if len(message) >= 5 else message.hex() - raise ValueError( - f"Expected schema registry format (magic byte 0x00 + 4-byte schema ID), " - f"but message is too short or has wrong magic byte: {msg_hex}" - ) - schema_id = int.from_bytes(message[1:5], 'big') - message = message[5:] - - if message_format == 'protobuf': - return _deserialize_protobuf(message, schema, uses_schema_registry), schema_id - elif message_format == 'avro': - return _deserialize_avro(message, schema), schema_id - else: - return _deserialize_json(message), schema_id - - -def _deserialize_json(message): - decoded = message.decode('utf-8') - json.loads(decoded) - return decoded - - -def _get_protobuf_message_class(schema_info, message_indices): - """Get the protobuf message class based on schema info and message indices. - - Args: - schema_info: Tuple of (descriptor_pool, file_descriptor_set) - message_indices: List of indices (e.g., [0], [1], [2, 0] for nested) - - Returns: - Message class for the specified type - """ - pool, descriptor_set = schema_info - - # First index is the message type in the file - file_descriptor = descriptor_set.file[0] - message_descriptor_proto = file_descriptor.message_type[message_indices[0]] - - package = file_descriptor.package - name_parts = [message_descriptor_proto.name] - - # Handle nested messages if there are more indices - current_proto = message_descriptor_proto - for idx in message_indices[1:]: - current_proto = current_proto.nested_type[idx] - name_parts.append(current_proto.name) - - if package: - full_name = f"{package}.{'.'.join(name_parts)}" - else: - full_name = '.'.join(name_parts) - - message_descriptor = pool.FindMessageTypeByName(full_name) - return message_factory.GetMessageClass(message_descriptor) - - -def _deserialize_protobuf(message, schema_info, uses_schema_registry): - """Deserialize a Protobuf message using google.protobuf with strict validation. - - Args: - message: Raw protobuf bytes - schema_info: Tuple of (descriptor_pool, file_descriptor_set) from build_protobuf_schema - uses_schema_registry: Whether to extract Confluent message indices from the message - """ - try: - if uses_schema_registry: - message_indices, message = _read_protobuf_message_indices(message) - # Empty indices array means use the first message type (index 0) - if not message_indices: - message_indices = [0] - else: - message_indices = [0] - - message_class = _get_protobuf_message_class(schema_info, message_indices) - schema_instance = message_class() - - bytes_consumed = schema_instance.ParseFromString(message) - - # Check if all bytes were consumed (strict validation) - if bytes_consumed != len(message): - raise ValueError( - f"Not all bytes were consumed during Protobuf decoding! " - f"Read {bytes_consumed} bytes, but message has {len(message)} bytes. " - ) - - return MessageToJson(schema_instance) - except Exception as e: - raise ValueError(f"Failed to deserialize Protobuf message: {e}") - - -def _deserialize_avro(message, schema): - """Deserialize an Avro message using fastavro with strict validation.""" - try: - bio = BytesIO(message) - initial_position = bio.tell() - data = schemaless_reader(bio, schema) - final_position = bio.tell() - - # Check if all bytes were consumed (strict validation) - bytes_read = final_position - initial_position - total_bytes = len(message) - - if bytes_read != total_bytes: - raise ValueError( - f"Not all bytes were consumed during Avro decoding! " - f"Read {bytes_read} bytes, but message has {total_bytes} bytes. " - ) - - return json.dumps(data) - except Exception as e: - raise ValueError(f"Failed to deserialize Avro message: {e}") - - -def build_schema(message_format, schema_str): - if message_format == 'protobuf': - return build_protobuf_schema(schema_str) - elif message_format == 'avro': - return build_avro_schema(schema_str) - return None - - -def build_avro_schema(schema_str): - """Build an Avro schema from a JSON string.""" - schema = json.loads(schema_str) - - if schema is None: - raise ValueError("Avro schema cannot be None") - - return schema - - -def build_protobuf_schema(schema_str): - """Build a Protobuf schema from a base64-encoded FileDescriptorSet. - - Returns a tuple of (descriptor_pool, file_descriptor_set) that can be used - to dynamically select and instantiate message types based on message indices. - - Args: - schema_str: Base64-encoded FileDescriptorSet - - Returns: - tuple: (DescriptorPool, FileDescriptorSet) - """ - # schema is encoded in base64, decode it before passing it to ParseFromString - schema_str = base64.b64decode(schema_str) - descriptor_set = descriptor_pb2.FileDescriptorSet() - descriptor_set.ParseFromString(schema_str) - - # Register all the file descriptors in a descriptor pool - pool = descriptor_pool.DescriptorPool() - for fd_proto in descriptor_set.file: - pool.Add(fd_proto) - - return (pool, descriptor_set) diff --git a/kafka_consumer/pyproject.toml b/kafka_consumer/pyproject.toml index f8c83b21fce9a..05f1b6dbb0f69 100644 --- a/kafka_consumer/pyproject.toml +++ b/kafka_consumer/pyproject.toml @@ -39,8 +39,6 @@ deps = [ "aws-msk-iam-sasl-signer-python==1.0.2", "boto3==1.42.72", "confluent-kafka==2.13.2", - "fastavro==1.12.1", - "protobuf==7.34.0", ] [project.urls] diff --git a/kafka_consumer/tests/test_integration.py b/kafka_consumer/tests/test_integration.py index 152eb61025e1b..7f4e3e45753a0 100644 --- a/kafka_consumer/tests/test_integration.py +++ b/kafka_consumer/tests/test_integration.py @@ -8,7 +8,6 @@ import mock import pytest -from confluent_kafka.admin import AdminClient from datadog_checks.dev.utils import get_metadata_metrics @@ -35,27 +34,6 @@ def mocked_time(): return 400 -def get_all_consumer_groups(kafka_instance): - """Get all consumer groups from Kafka cluster.""" - config = { - "bootstrap.servers": kafka_instance['kafka_connect_str'], - "socket.timeout.ms": 1000, - "topic.metadata.refresh.interval.ms": 2000, - } - config.update(common.get_authentication_configuration(kafka_instance)) - admin_client = AdminClient(config) - - final_groups = set() - try: - groups_result = admin_client.list_consumer_groups().result() - for valid_group in groups_result.valid: - final_groups.add(valid_group.group_id) - except Exception as e: - print(f"Error getting final consumer groups: {e}") - - return final_groups - - def test_check_kafka(aggregator, check, kafka_instance, dd_run_check): """ Testing Kafka_consumer check. @@ -488,58 +466,3 @@ def test_regex_consumer_groups( aggregator.assert_metric("kafka.estimated_consumer_lag", count=consumer_lag_seconds_count) assert expected_warning in caplog.text - - -@mock.patch('datadog_checks.kafka_consumer.kafka_consumer.time', mocked_time) -def test_data_streams_live_messages(dd_run_check, check, kafka_instance, datadog_agent): - cluster_id = common.get_cluster_id() - kafka_instance['live_messages_configs'] = [ - { - 'kafka': { - 'cluster': cluster_id, - 'topic': 'marvel', - 'partition': 0, - 'start_offset': 0, - 'n_messages': 2, - 'value_format': 'json', - }, - 'id': 'config_1_id', - } - ] - kafka_check = check(kafka_instance) - dd_run_check(kafka_check) - - # Verify that live messages is not leaving behind any new consumer groups - final_groups = get_all_consumer_groups(kafka_instance) - assert final_groups == {'my_consumer'} - - expected_logs = [ - { - 'timestamp': 400 * 1000, - 'technology': 'kafka', - 'cluster': str(cluster_id), - 'config_id': 'config_1_id', - 'topic': 'marvel', - 'partition': '0', - 'offset': '0', - 'feature': 'data_streams_messages', - 'message_value': '{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"}', - 'ddtags': 'optional:tag1', - }, - { - 'timestamp': 400 * 1000, - 'technology': 'kafka', - 'cluster': str(cluster_id), - 'config_id': 'config_1_id', - 'topic': 'marvel', - 'partition': '0', - 'offset': '1', - 'feature': 'data_streams_messages', - 'message_value': '{"name": "Bruce Banner", "age": 45,\ - "transaction_amount": 456, "currency": "dollar"}', - 'value_schema_id': '350', - 'message_key': '{"name": "Bruce Banner"}', - 'ddtags': 'optional:tag1', - }, - ] - datadog_agent.assert_logs(kafka_check.check_id, expected_logs) diff --git a/kafka_consumer/tests/test_unit.py b/kafka_consumer/tests/test_unit.py index bb1d7a20bd0d3..35bbcacaf1488 100644 --- a/kafka_consumer/tests/test_unit.py +++ b/kafka_consumer/tests/test_unit.py @@ -1,29 +1,15 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import base64 -import json import logging from contextlib import nullcontext as does_not_raise import mock import pytest -from confluent_kafka import TopicPartition -from google.protobuf import descriptor_pb2 -from google.protobuf.message import DecodeError from datadog_checks.kafka_consumer import KafkaCheck from datadog_checks.kafka_consumer.client import KafkaClient -from datadog_checks.kafka_consumer.kafka_consumer import ( - DATA_STREAMS_MESSAGES_CACHE_KEY, - _get_interpolated_timestamp, - _get_protobuf_message_class, - build_avro_schema, - build_protobuf_schema, - build_schema, - deserialize_message, - resolve_start_offsets, -) +from datadog_checks.kafka_consumer.kafka_consumer import _get_interpolated_timestamp pytestmark = [pytest.mark.unit] @@ -543,855 +529,6 @@ def test_add_broker_timestamps_evicts_by_oldest_timestamp(kafka_instance, check) assert 600 in timestamps -def test_resolve_start_offsets(): - highwater_offsets = { - ("topic1", 0): 100, - ("topic1", 1): 200, - ("topic2", 0): 150, - } - assert resolve_start_offsets(highwater_offsets, "topic1", 0, 80, 10) == [TopicPartition("topic1", 0, 80)] - assert resolve_start_offsets(highwater_offsets, "topic2", 0, -1, 10) == [TopicPartition("topic2", 0, 141)] - assert sorted(resolve_start_offsets(highwater_offsets, "topic1", -1, -1, 10)) == [ - TopicPartition("topic1", 0, 81), - TopicPartition("topic1", 1, 191), - ] - - -class MockedMessage: - def __init__(self, value, key=None, offset=0): - self.v = value - self.k = key - self.o = offset - - def value(self): - return self.v - - def key(self): - return self.k - - def partition(self): - return 0 - - def offset(self): - return self.o - - -def test_deserialize_message(): - message = b'{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"}' - # schema ID is 350, which is 0x015E in hex. - # A magic byte (0x00) is added and the schema ID (4-byte big-endian integer). - message_with_schema = ( - b'\x00\x00\x00\x01\x5e{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"}' - ) - key = b'{"name": "Peter Parker"}' - assert deserialize_message(MockedMessage(message, key), 'json', '', False, 'json', '', False) == ( - '{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"}', - None, - '{"name": "Peter Parker"}', - None, - ) - assert deserialize_message(MockedMessage(message_with_schema), 'json', '', False, 'json', '', False) == ( - '{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"}', - 350, - '', - None, - ) - invalid_json = b'{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"' - assert deserialize_message(MockedMessage(invalid_json, key), 'json', '', False, 'json', '', False) == ( - None, - None, - None, - None, - ) - - invalid_utf8 = b'{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"\xff' - assert deserialize_message(MockedMessage(invalid_utf8, key), 'json', '', False, 'json', '', False) == ( - None, - None, - None, - None, - ) - - # Test Avro deserialization - avro_schema = ( - '{"type": "record", "name": "Book", "namespace": "com.book", ' - '"fields": [{"name": "isbn", "type": "long"}, {"name": "title", "type": "string"}, ' - '{"name": "author", "type": "string"}]}' - ) - avro_message = b'\xd0\xf5\xe4\xd6\xa3\xb9\x046The Go Programming Language\x18Alan Donovan' - parsed_avro_schema = build_schema('avro', avro_schema) - assert deserialize_message( - MockedMessage(avro_message, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) == ( - '{"isbn": 9780134190440, "title": "The Go Programming Language", "author": "Alan Donovan"}', - None, - '{"name": "Peter Parker"}', - None, - ) - - # Test Protobuf deserialization - protobuf_schema = ( - 'CmoKDHNjaGVtYS5wcm90bxIIY29tLmJvb2siSAoEQm9vaxISCgRpc2JuGAEgASgDUgRpc2Ju' - 'EhQKBXRpdGxlGAIgASgJUgV0aXRsZRIWCgZhdXRob3IYAyABKAlSBmF1dGhvcmIGcHJvdG8z' - ) - protobuf_message = ( - b'\x08\xe8\xba\xb2\xeb\xd1\x9c\x02\x12\x1b\x54\x68\x65\x20\x47\x6f\x20\x50\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x4c\x61\x6e\x67\x75\x61\x67\x65' - b'\x1a\x0c\x41\x6c\x61\x6e\x20\x44\x6f\x6e\x6f\x76\x61\x6e' - ) - parsed_protobuf_schema = build_schema('protobuf', protobuf_schema) - assert deserialize_message( - MockedMessage(protobuf_message, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) == ( - '{\n "isbn": "9780134190440",\n "title": "The Go Programming Language",\n "author": "Alan Donovan"\n}', - None, - '{"name": "Peter Parker"}', - None, - ) - - # Test invalid Avro messages - # Empty message (returns empty string, not None) - assert deserialize_message(MockedMessage(b'', key), 'avro', parsed_avro_schema, False, 'json', '', False) == ( - '', - None, - '{"name": "Peter Parker"}', - None, - ) - - # Corrupted message (truncated) - corrupted_avro = b'\xd0\xf5\xe4\xd6\xa3\xb9\x046The Go Programming Language' # Missing author field - assert deserialize_message( - MockedMessage(corrupted_avro, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) == ( - None, - None, - None, - None, - ) - - # Wrong data type (string instead of long for isbn) - wrong_type_avro = b'\x02\x12\x1bThe Go Programming Language\x18Alan Donovan' # Wrong encoding for isbn - assert deserialize_message( - MockedMessage(wrong_type_avro, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) == ( - None, - None, - None, - None, - ) - - # Random bytes - random_avro = b'\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0' - assert deserialize_message( - MockedMessage(random_avro, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) == ( - None, - None, - None, - None, - ) - - # Completely invalid Avro message (random bytes) - invalid_avro = b'\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0' - assert deserialize_message( - MockedMessage(invalid_avro, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) == ( - None, - None, - None, - None, - ) - - # Avro message with wrong data types (string where long expected) - wrong_type_avro = b'\x02\x12\x1bThe Go Programming Language\x18Alan Donovan' # Wrong encoding for isbn - assert deserialize_message( - MockedMessage(wrong_type_avro, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) == ( - None, - None, - None, - None, - ) - - # Test invalid Protobuf messages - # Empty message (returns empty string, not None) - assert deserialize_message( - MockedMessage(b'', key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) == ( - '', - None, - '{"name": "Peter Parker"}', - None, - ) - - # Random bytes - random_protobuf = b'\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0' - assert deserialize_message( - MockedMessage(random_protobuf, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) == ( - None, - None, - None, - None, - ) - - # Completely invalid Protobuf message (random bytes) - invalid_protobuf = b'\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0' - assert deserialize_message( - MockedMessage(invalid_protobuf, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) == (None, None, None, None) - - # Protobuf message with wrong field number (field 99 instead of 1) - wrong_field_protobuf = ( - b'\x99\x01\xe8\xba\xb2\xeb\xd1\x9c\x02\x12\x1bThe Go Programming Language\x1a\x0cAlan Donovan' - ) - assert deserialize_message( - MockedMessage(wrong_field_protobuf, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) == (None, None, None, None) - - # Protobuf message with truncated varint - truncated_varint_protobuf = b'\x08\xff\xff\xff\xff\xff\xff\xff\xff\xff' # Incomplete varint - assert deserialize_message( - MockedMessage(truncated_varint_protobuf, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) == (None, None, None, None) - - -def test_strict_avro_validation(): - """Test that Avro deserialization fails when not all bytes are consumed.""" - key = b'{"name": "Peter Parker"}' - - # Test case 1: Simple primitive string schema with extra bytes - # A primitive string in Avro is encoded as: varint length + UTF-8 bytes - # An empty string is just: 0x00 (zero length) - # If we have 0x00 followed by extra bytes (e.g., magic byte + 4 bytes + stuff), - # the string decoder will read the empty string but leave bytes unconsumed - string_schema = '"string"' - parsed_string_schema = build_schema('avro', string_schema) - - # Message: 0x00 (empty string) + 0x00 (magic byte) + 4 bytes + some random data - # The Avro string decoder will only consume the first 0x00, leaving the rest - message_with_extra_bytes = b'\x00\x00\x00\x00\x01\x5e\x12\x34\x56\x78' - - # This should now fail because not all bytes are consumed - result = deserialize_message( - MockedMessage(message_with_extra_bytes, key), 'avro', parsed_string_schema, False, 'json', '', False - ) - assert result == (None, None, None, None), "Expected deserialization to fail due to unconsumed bytes" - - # Test case 2: Avro message with trailing garbage bytes after valid data - avro_schema = ( - '{"type": "record", "name": "Book", "namespace": "com.book", ' - '"fields": [{"name": "isbn", "type": "long"}, {"name": "title", "type": "string"}, ' - '{"name": "author", "type": "string"}]}' - ) - parsed_avro_schema = build_schema('avro', avro_schema) - - # Valid Avro message + trailing garbage - valid_avro_message = b'\xd0\xf5\xe4\xd6\xa3\xb9\x046The Go Programming Language\x18Alan Donovan' - message_with_trailing_bytes = valid_avro_message + b'\xff\xfe\xfd\xfc' - - # This should now fail because of the trailing bytes - result = deserialize_message( - MockedMessage(message_with_trailing_bytes, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) - assert result == (None, None, None, None), "Expected deserialization to fail due to trailing bytes" - - # Test case 3: Simple int schema with extra bytes - int_schema = '"int"' - parsed_int_schema = build_schema('avro', int_schema) - - # Message: 0x02 (int value 1) + extra bytes - message_int_with_extra = b'\x02\xde\xad\xbe\xef' - - result = deserialize_message( - MockedMessage(message_int_with_extra, key), 'avro', parsed_int_schema, False, 'json', '', False - ) - assert result == (None, None, None, None), "Expected deserialization to fail due to unconsumed bytes" - - # Test case 4: Verify that valid messages still work - valid_string_message = b'\x0aHello' # Length 5 (encoded as 0x0a = 10/2 = 5) + "Hello" - result = deserialize_message( - MockedMessage(valid_string_message, key), 'avro', parsed_string_schema, False, 'json', '', False - ) - assert result[0] == '"Hello"', "Expected valid string message to deserialize correctly" - assert result[1] is None - - valid_int_message = b'\x02' # int value 1 - result = deserialize_message( - MockedMessage(valid_int_message, key), 'avro', parsed_int_schema, False, 'json', '', False - ) - assert result[0] == '1', "Expected valid int message to deserialize correctly" - - -def test_strict_protobuf_validation(): - """Test that Protobuf deserialization fails when not all bytes are consumed.""" - key = b'{"name": "Peter Parker"}' - - # Build the same Book schema used in other tests - protobuf_schema = ( - 'CmoKDHNjaGVtYS5wcm90bxIIY29tLmJvb2siSAoEQm9vaxISCgRpc2JuGAEgASgDUgRpc2Ju' - 'EhQKBXRpdGxlGAIgASgJUgV0aXRsZRIWCgZhdXRob3IYAyABKAlSBmF1dGhvcmIGcHJvdG8z' - ) - parsed_protobuf_schema = build_schema('protobuf', protobuf_schema) - - # Test case 1: Valid Protobuf message with trailing garbage bytes - valid_protobuf_message = ( - b'\x08\xe8\xba\xb2\xeb\xd1\x9c\x02\x12\x1b\x54\x68\x65\x20\x47\x6f\x20\x50\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x4c\x61\x6e\x67\x75\x61\x67\x65' - b'\x1a\x0c\x41\x6c\x61\x6e\x20\x44\x6f\x6e\x6f\x76\x61\x6e' - ) - message_with_trailing_bytes = valid_protobuf_message + b'\xff\xfe\xfd\xfc' - - # This should now fail because of the trailing bytes - result = deserialize_message( - MockedMessage(message_with_trailing_bytes, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) - assert result == (None, None, None, None), "Expected deserialization to fail due to trailing bytes" - - # Test case 2: Message with extra fields that aren't in the schema - # Protobuf will parse this but leave bytes unconsumed if there are truly extra bytes beyond valid fields - # Adding a completely invalid trailing byte sequence - message_with_invalid_trailer = valid_protobuf_message + b'\x00\x00\x00\x01\x5e' - - result = deserialize_message( - MockedMessage(message_with_invalid_trailer, key), - 'protobuf', - parsed_protobuf_schema, - False, - 'json', - '', - False, - ) - assert result == (None, None, None, None), "Expected deserialization to fail due to unconsumed bytes" - - # Test case 3: Verify that valid messages still work - result = deserialize_message( - MockedMessage(valid_protobuf_message, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) - assert result[0] is not None, "Expected valid protobuf message to deserialize correctly" - assert 'The Go Programming Language' in result[0] - - -def test_schema_registry_explicit_configuration(): - """Test that explicit schema registry configuration is enforced.""" - key = b'{"name": "Peter Parker"}' - - # Test Avro with value_uses_schema_registry=True - avro_schema = ( - '{"type": "record", "name": "Book", "namespace": "com.book", ' - '"fields": [{"name": "isbn", "type": "long"}, {"name": "title", "type": "string"}, ' - '{"name": "author", "type": "string"}]}' - ) - parsed_avro_schema = build_schema('avro', avro_schema) - - # Valid Avro message WITHOUT schema registry format - avro_message_no_sr = b'\xd0\xf5\xe4\xd6\xa3\xb9\x046The Go Programming Language\x18Alan Donovan' - - # When uses_schema_registry=False, this should work - result = deserialize_message( - MockedMessage(avro_message_no_sr, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) - assert result[0] is not None, "Should succeed when uses_schema_registry=False" - assert result[1] is None, "Should have no schema ID" - - # When uses_schema_registry=True, this should fail (missing magic byte and schema ID) - result = deserialize_message( - MockedMessage(avro_message_no_sr, key), 'avro', parsed_avro_schema, True, 'json', '', False - ) - assert result == (None, None, None, None), "Should fail when uses_schema_registry=True" - - # Valid Avro message WITH schema registry format (schema ID 350 = 0x015E) - avro_message_with_sr = ( - b'\x00\x00\x00\x01\x5e\xd0\xf5\xe4\xd6\xa3\xb9\x046The Go Programming Language\x18Alan Donovan' - ) - - # When uses_schema_registry=True, this should work - result = deserialize_message( - MockedMessage(avro_message_with_sr, key), 'avro', parsed_avro_schema, True, 'json', '', False - ) - assert result[0] is not None, "Should succeed when uses_schema_registry=True" - assert result[1] == 350, "Should extract schema ID 350" - assert 'The Go Programming Language' in result[0] - - # Test with wrong magic byte - wrong_magic_byte = b'\x01\x00\x00\x01\x5e\xd0\xf5\xe4\xd6\xa3\xb9\x046The Go Programming Language\x18Alan Donovan' - result = deserialize_message( - MockedMessage(wrong_magic_byte, key), 'avro', parsed_avro_schema, True, 'json', '', False - ) - assert result == (None, None, None, None), "Should fail with wrong magic byte" - - # Test with message too short (less than 5 bytes) - too_short = b'\x00\x00\x01' - result = deserialize_message(MockedMessage(too_short, key), 'avro', parsed_avro_schema, True, 'json', '', False) - assert result == (None, None, None, None), "Should fail when message too short for SR format" - - # Test Protobuf with value_uses_schema_registry=True - protobuf_schema = ( - 'CmoKDHNjaGVtYS5wcm90bxIIY29tLmJvb2siSAoEQm9vaxISCgRpc2JuGAEgASgDUgRpc2Ju' - 'EhQKBXRpdGxlGAIgASgJUgV0aXRsZRIWCgZhdXRob3IYAyABKAlSBmF1dGhvcmIGcHJvdG8z' - ) - parsed_protobuf_schema = build_schema('protobuf', protobuf_schema) - - # Valid Protobuf message WITHOUT schema registry format - protobuf_message_no_sr = ( - b'\x08\xe8\xba\xb2\xeb\xd1\x9c\x02\x12\x1b\x54\x68\x65\x20\x47\x6f\x20\x50\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x4c\x61\x6e\x67\x75\x61\x67\x65' - b'\x1a\x0c\x41\x6c\x61\x6e\x20\x44\x6f\x6e\x6f\x76\x61\x6e' - ) - - # When uses_schema_registry=False, this should work - result = deserialize_message( - MockedMessage(protobuf_message_no_sr, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) - assert result[0] is not None, "Protobuf should succeed when uses_schema_registry=False" - assert result[1] is None, "Should have no schema ID" - - # When uses_schema_registry=True, this should fail - result = deserialize_message( - MockedMessage(protobuf_message_no_sr, key), 'protobuf', parsed_protobuf_schema, True, 'json', '', False - ) - assert result == (None, None, None, None), "Protobuf should fail when uses_schema_registry=True but no SR format" - - # Valid Protobuf message WITH schema registry format - # Confluent Protobuf wire format: - # [magic_byte][schema_id:4bytes][array_length:varint][index:varint][protobuf_payload] - protobuf_message_with_sr = ( - b'\x00\x00\x00\x01\x5e' # magic byte (0x00) + schema ID 350 (0x0000015e) - b'\x01' # message indices array length = 1 - b'\x00' # message index = 0 - b'\x08\xe8\xba\xb2\xeb\xd1\x9c\x02\x12\x1b\x54\x68\x65\x20\x47\x6f\x20\x50\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x4c\x61\x6e\x67\x75\x61\x67\x65' - b'\x1a\x0c\x41\x6c\x61\x6e\x20\x44\x6f\x6e\x6f\x76\x61\x6e' - ) - - # When uses_schema_registry=True, this should work - result = deserialize_message( - MockedMessage(protobuf_message_with_sr, key), - 'protobuf', - parsed_protobuf_schema, - True, - 'json', - '', - False, - ) - assert result[0] is not None, "Protobuf should succeed when uses_schema_registry=True with SR format" - assert result[1] == 350, "Should extract schema ID 350" - assert 'The Go Programming Language' in result[0] - - # Test key_uses_schema_registry=True - # When key has no schema registry format but key_uses_schema_registry=True, key decoding should fail - # but value should still succeed - result = deserialize_message( - MockedMessage(avro_message_no_sr, key), 'avro', parsed_avro_schema, False, 'json', '', True - ) - # Value should succeed, but key should fail (returning None for key fields) - assert result[0] is not None, "Value should succeed" - assert result[2] is None, "Key should fail when key_uses_schema_registry=True but no SR format" - assert result[3] is None, "Key schema ID should be None when key fails" - - -def test_protobuf_message_indices_with_schema_registry(): - """Test Confluent Protobuf wire format with different message indices.""" - key = b'{"test": "key"}' - - # Schema with multiple message types and nested type - # message Book { int64 isbn = 1; string title = 2; } - # message Author { string name = 1; int32 age = 2; } - # message Library { message Section { string name = 1; } string name = 1; } - protobuf_schema = ( - 'CpMBCgxzY2hlbWEucHJvdG8SC2NvbS5leGFtcGxlIh8KBEJvb2sSCgoEaXNibhgBKAMSCwoFdGl0bGUY' - 'AigJIh8KBkF1dGhvchIKCgRuYW1lGAEoCRIJCgNhZ2UYAigFIiwKB0xpYnJhcnkSCgoEbmFtZRgBKAka' - 'FQoHU2VjdGlvbhIKCgRuYW1lGAEoCWIGcHJvdG8z' - ) - parsed_schema = build_schema('protobuf', protobuf_schema) - - # Test index [0] - Book message - book_payload = bytes.fromhex('08e80712095465737420426f6f6b') - book_msg = b'\x00\x00\x00\x01\x5e\x01\x00' + book_payload - result = deserialize_message(MockedMessage(book_msg, key), 'protobuf', parsed_schema, True, 'json', '', False) - assert result[0] and 'Test Book' in result[0] - - # Test index [1] - Author message - author_payload = bytes.fromhex('0a0a4a616e6520536d697468101e') - author_msg = b'\x00\x00\x00\x01\x5e\x01\x01' + author_payload - result = deserialize_message(MockedMessage(author_msg, key), 'protobuf', parsed_schema, True, 'json', '', False) - assert result[0] and 'Jane Smith' in result[0] and '30' in result[0] - - # Test nested [2, 0] - Library.Section message - section_payload = bytes.fromhex('0a0746696374696f6e') - section_msg = b'\x00\x00\x00\x01\x5e\x02\x02\x00' + section_payload - result = deserialize_message(MockedMessage(section_msg, key), 'protobuf', parsed_schema, True, 'json', '', False) - assert result[0] and 'Fiction' in result[0] - - -def test_protobuf_empty_message_indices_with_schema_registry(): - """Test Confluent Protobuf wire format with empty message indices array. - - When message indices array is empty (encoded as varint 0x00), it should - default to using the first message type (index 0). - - This test uses real message bytes from a Kafka topic to ensure the - deserialization handles the Confluent wire format correctly. - """ - key = b'null' - - # Schema from real Kafka topic - Purchase message - # message Purchase { string order_id = 1; string customer_id = 2; int64 order_date = 3; - # string city = 6; string country = 7; } - protobuf_schema = ( - 'CrkDCgxzY2hlbWEucHJvdG8SCHB1cmNoYXNlIpMBCghQdXJjaGFzZRIZCghvcmRlcl9pZBgBIAEoCVIH' - 'b3JkZXJJZBIfCgtjdXN0b21lcl9pZBgCIAEoCVIKY3VzdG9tZXJJZBIdCgpvcmRlcl9kYXRlGAMgASgD' - 'UglvcmRlckRhdGUSEgoEY2l0eRgGIAEoCVIEY2l0eRIYCgdjb3VudHJ5GAcgASgJUgdjb3VudHJ5ItIB' - 'CgpQdXJjaGFzZVYyEiUKDnRyYW5zYWN0aW9uX2lkGAEgASgJUg10cmFuc2FjdGlvbklkEhcKB3VzZXJf' - 'aWQYAiABKAlSBnVzZXJJZBIcCgl0aW1lc3RhbXAYAyABKANSCXRpbWVzdGFtcBIaCghsb2NhdGlvbhgE' - 'IAEoCVIIbG9jYXRpb24SFgoGcmVnaW9uGAUgASgJUgZyZWdpb24SFgoGYW1vdW50GAYgASgBUgZhbW91' - 'bnQSGgoIY3VycmVuY3kYByABKAlSCGN1cnJlbmN5QiwKG2RhdGFkb2cua2Fma2EuZXhhbXBsZS5wcm90' - 'b0INUHVyY2hhc2VQcm90b2IGcHJvdG8z' - ) - parsed_schema = build_schema('protobuf', protobuf_schema) - - # Real message from Kafka topic "human-orders" - # Hex breakdown: - # 00 00 00 00 01 - Schema Registry header (magic byte + schema ID 1) - # 00 - Empty message indices array (varint 0 = 0 elements) - # 0a 05 31 32 33 34 35 ... - Protobuf payload (Purchase message) - message_hex = '0000000001000a0531323334351205363738393018f4eae0c4b8333a064d657869636f' - message_bytes = bytes.fromhex(message_hex) - - # Test with uses_schema_registry=True (explicit) - result = deserialize_message(MockedMessage(message_bytes, key), 'protobuf', parsed_schema, True, 'json', '', False) - assert result[0], "Deserialization should succeed" - assert '12345' in result[0], "Should contain order_id" - assert '67890' in result[0], "Should contain customer_id" - assert 'Mexico' in result[0], "Should contain country" - assert result[1] == 1, "Should detect schema ID 1" - - # Test with uses_schema_registry=False (fallback mode) - result_fallback = deserialize_message( - MockedMessage(message_bytes, key), 'protobuf', parsed_schema, False, 'json', '', False - ) - assert result_fallback[0], "Fallback mode should also succeed" - assert '12345' in result_fallback[0], "Fallback should contain order_id" - assert result_fallback[1] == 1, "Fallback should detect schema ID 1" - - -def mocked_time(): - return 400 - - -@mock.patch('datadog_checks.kafka_consumer.kafka_consumer.time', mocked_time) -@pytest.mark.parametrize( - 'messages, value_format, value_schema, persistent_cache_read_content, ' - 'expected_persistent_cache_writes, expected_logs', - [ - pytest.param( - [ - MockedMessage( - b'{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"}', - b'{"name": "Peter Parker"}', - 12, - ), - MockedMessage( - b'{"name": "Bruce Banner", "age": 45, "transaction_amount": 456, "currency": "dollar"}', - b'', - 13, - ), - None, - ], - 'json', - '', - "config_1_id,config_id_2", - [], - [], - id='Does not retrieve messages a second time', - ), - pytest.param( - [ - MockedMessage( - b'{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"}', - b'{"name": "Peter Parker"}', - 12, - ), - MockedMessage( - b'{"name": "Bruce Banner", "age": 45, "transaction_amount": 456, "currency": "dollar"}', - b'', - 13, - ), - None, - ], - 'json', - '', - "", - ["config_1_id"], - [ - { - 'timestamp': 400, - 'technology': 'kafka', - 'cluster': 'cluster_id', - 'config_id': 'config_1_id', - 'topic': 'topic1', - 'partition': '0', - 'offset': '12', - 'feature': 'data_streams_messages', - 'message_value': '{"name": "Peter Parker", "age": 18, \ -"transaction_amount": 123, "currency": "dollar"}', - 'message_key': '{"name": "Peter Parker"}', - }, - { - 'timestamp': 400, - 'technology': 'kafka', - 'cluster': 'cluster_id', - 'config_id': 'config_1_id', - 'topic': 'topic1', - 'partition': '0', - 'offset': '13', - 'feature': 'data_streams_messages', - 'message_value': '{"name": "Bruce Banner", "age": 45, \ -"transaction_amount": 456, "currency": "dollar"}', - }, - { - 'timestamp': 400, - 'technology': 'kafka', - 'cluster': 'cluster_id', - 'config_id': 'config_1_id', - 'topic': 'topic1', - 'message': 'No more messages to retrieve', - 'live_messages_error': 'No more messages to retrieve', - 'feature': 'data_streams_messages', - }, - ], - id='Retrieves messages from Kafka', - ), - # This is the serialized Protobuf representing: - # syntax = "proto3"; - # package com.book; - # message Book { - # int64 isbn = 1; - # string title = 2; - # string author = 3; - # } - pytest.param( - [ - MockedMessage( - b'\x08\xe8\xba\xb2\xeb\xd1\x9c\x02\x12\x1b\x54\x68\x65\x20\x47\x6f\x20\x50\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x4c\x61\x6e\x67\x75\x61\x67\x65\x1a\x0c\x41\x6c\x61\x6e\x20\x44\x6f\x6e\x6f\x76\x61\x6e', - b'{"name": "Peter Parker"}', - 12, - ), - None, - ], - 'protobuf', - 'CmoKDHNjaGVtYS5wcm90bxIIY29tLmJvb2siSAoEQm9vaxISCgRpc2JuGAEgASgDUgRpc2JuEhQKBXRpdGxlGAIgASgJUgV0aXRsZRIWCgZhdXRob3IYAyABKAlSBmF1dGhvcmIGcHJvdG8z', - "", - ["config_1_id"], - [ - { - 'timestamp': 400, - 'technology': 'kafka', - 'cluster': 'cluster_id', - 'config_id': 'config_1_id', - 'topic': 'topic1', - 'partition': '0', - 'offset': '12', - 'feature': 'data_streams_messages', - 'message_value': ( - '{\n "isbn": "9780134190440",\n "title": "The Go Programming Language",\n ' - '"author": "Alan Donovan"\n}' - ), - 'message_key': '{"name": "Peter Parker"}', - }, - { - 'timestamp': 400, - 'technology': 'kafka', - 'cluster': 'cluster_id', - 'config_id': 'config_1_id', - 'topic': 'topic1', - 'message': 'No more messages to retrieve', - 'live_messages_error': 'No more messages to retrieve', - 'feature': 'data_streams_messages', - }, - ], - id='Retrieves Protobuf messages from Kafka', - ), - pytest.param( - [ - MockedMessage( - b'\xd0\xf5\xe4\xd6\xa3\xb9\x046The Go Programming Language\x18Alan Donovan', - b'{"name": "Peter Parker"}', - 12, - ), - None, - ], - 'avro', - ( - '{"type": "record", "name": "Book", "namespace": "com.book", ' - '"fields": [{"name": "isbn", "type": "long"}, {"name": "title", "type": "string"}, ' - '{"name": "author", "type": "string"}]}' - ), - "", - ["config_1_id"], - [ - { - 'timestamp': 400, - 'technology': 'kafka', - 'cluster': 'cluster_id', - 'config_id': 'config_1_id', - 'topic': 'topic1', - 'partition': '0', - 'offset': '12', - 'feature': 'data_streams_messages', - 'message_value': ( - '{"isbn": 9780134190440, "title": "The Go Programming Language", "author": "Alan Donovan"}' - ), - 'message_key': '{"name": "Peter Parker"}', - }, - { - 'timestamp': 400, - 'technology': 'kafka', - 'cluster': 'cluster_id', - 'config_id': 'config_1_id', - 'topic': 'topic1', - 'message': 'No more messages to retrieve', - 'live_messages_error': 'No more messages to retrieve', - 'feature': 'data_streams_messages', - }, - ], - id='Retrieves Avro messages from Kafka', - ), - ], -) -def test_data_streams_messages( - messages, - value_format, - value_schema, - persistent_cache_read_content, - expected_persistent_cache_writes, - expected_logs, - kafka_instance, - dd_run_check, - check, -): - ( - kafka_instance.update( - { - 'consumer_groups': {}, - 'monitor_unlisted_consumer_groups': True, - 'live_messages_configs': [ - { - 'kafka': { - 'cluster': 'cluster_id', - 'topic': 'topic1', - 'partition': 0, - 'start_offset': 0, - 'n_messages': 3, - 'value_format': value_format, - 'value_schema': value_schema, - 'key_format': 'json', - 'key_schema': '', - }, - 'id': 'config_1_id', - } - ], - } - ), - ) - mock_client = seed_mock_client(cluster_id="Cluster_id") - mock_client.get_next_message.side_effect = messages - check = check(kafka_instance) - check.client = mock_client - - def mocked_read_persistent_cache(key): - if key == DATA_STREAMS_MESSAGES_CACHE_KEY: - return persistent_cache_read_content - return "" - - check.read_persistent_cache = mock.Mock(side_effect=mocked_read_persistent_cache) - check.write_persistent_cache = mock.Mock() - check.send_log = mock.Mock() - - dd_run_check(check) - - for content in expected_persistent_cache_writes: - assert mock.call(DATA_STREAMS_MESSAGES_CACHE_KEY, content) in check.write_persistent_cache.mock_calls - assert [mock.call(log) for log in expected_logs] == check.send_log.mock_calls - - -def test_build_schema(): - """Test build_schema function with various valid and invalid schemas.""" - - # Test JSON format (should return None) - assert build_schema('json', '') is None - assert build_schema('json', '{"some": "json"}') is None - assert build_schema('json', None) is None - - # Test valid Avro schema - valid_avro_schema = ( - '{"type": "record", "name": "Book", "namespace": "com.book", ' - '"fields": [{"name": "isbn", "type": "long"}, {"name": "title", "type": "string"}, ' - '{"name": "author", "type": "string"}]}' - ) - avro_result = build_schema('avro', valid_avro_schema) - assert avro_result is not None - assert avro_result['type'] == 'record' - assert avro_result['name'] == 'Book' - assert avro_result['namespace'] == 'com.book' - - # Test valid Protobuf schema - valid_protobuf_schema = ( - 'CmoKDHNjaGVtYS5wcm90bxIIY29tLmJvb2siSAoEQm9vaxISCgRpc2JuGAEgASgDUgRpc2Ju' - 'EhQKBXRpdGxlGAIgASgJUgV0aXRsZRIWCgZhdXRob3IYAyABKAlSBmF1dGhvcmIGcHJvdG8z' - ) - protobuf_result = build_schema('protobuf', valid_protobuf_schema) - message_class = _get_protobuf_message_class(protobuf_result, [0]) - message_instance = message_class() - assert hasattr(message_instance, 'isbn') - assert hasattr(message_instance, 'title') - assert hasattr(message_instance, 'author') - - # Test unknown format - assert build_schema('unknown_format', 'some_schema') is None - - -def test_build_schema_error_cases(): - """Test build_schema with various error cases and edge cases.""" - - # Test Avro error cases - # Invalid JSON syntax - with pytest.raises(json.JSONDecodeError): - build_schema('avro', '{"invalid": json}') - - # Valid JSON but incomplete schema (fastavro is permissive) - result = build_schema('avro', '{"type": "record"}') # Missing name and fields - assert result is not None - - # Test Protobuf error cases - # Invalid base64 encoding - with pytest.raises(base64.binascii.Error): - build_schema('protobuf', 'invalid-base64!') - - # Valid base64 but invalid protobuf schema - # This is a valid base64 string that doesn't represent a valid FileDescriptorSet - with pytest.raises(DecodeError): # Will be a protobuf DecodeError - build_schema('protobuf', 'SGVsbG8gV29ybGQ=') # "Hello World" in base64 - - # Valid base64 but empty schema - should fail when trying to access message types - # Create a minimal but empty FileDescriptorSet - empty_descriptor = descriptor_pb2.FileDescriptorSet() - empty_descriptor_bytes = empty_descriptor.SerializeToString() - empty_descriptor_b64 = base64.b64encode(empty_descriptor_bytes).decode('utf-8') - - result = build_schema('protobuf', empty_descriptor_b64) - with pytest.raises(IndexError): # Should fail when trying to access file[0] - _get_protobuf_message_class(result, [0]) - - -def test_build_schema_none_handling(): - """Test that build_schema functions properly handle None values.""" - - # Test Avro schema with None - should raise TypeError - with pytest.raises(TypeError): - build_avro_schema(None) - - # Test Protobuf schema with None - should raise TypeError or base64.binascii.Error - with pytest.raises((TypeError, base64.binascii.Error)): - build_protobuf_schema(None) - - def test_count_consumer_contexts(check, kafka_instance): kafka_consumer_check = check(kafka_instance) consumer_offsets = { From e2f996e5d89dcca99441e1cf27d7bcec3f673446 Mon Sep 17 00:00:00 2001 From: Arnav Sehgal <143423884+sehgal23@users.noreply.github.com> Date: Wed, 27 May 2026 13:50:27 -0400 Subject: [PATCH 09/44] Added the pod level resources to the metadata.csv (#23847) --- kubelet/metadata.csv | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kubelet/metadata.csv b/kubelet/metadata.csv index 7b1460a0bcaf4..f82a55aee1c9f 100644 --- a/kubelet/metadata.csv +++ b/kubelet/metadata.csv @@ -15,6 +15,8 @@ kubernetes.cpu.cfs.throttled.seconds,gauge,,,,Total time duration the container kubernetes.cpu.capacity,gauge,,core,,The number of cores in this machine (available until kubernetes v1.18),0,kubernetes,k8s.cpu.capacity, kubernetes.cpu.usage.total,gauge,,nanocore,,The number of cores used,-1,kubernetes,k8s.cpu, kubernetes.cpu.limits,gauge,,core,,The limit of cpu cores set,0,kubernetes,k8s.cpu.limits, +kubernetes.pod.cpu.request,gauge,,core,,The pod-level requested CPU cores,0,kubernetes,k8s.pod.cpu.request, +kubernetes.pod.cpu.limit,gauge,,core,,The pod-level CPU core limit,0,kubernetes,k8s.pod.cpu.limit, kubernetes.cpu.requests,gauge,,core,,The requested cpu cores,0,kubernetes,k8s.cpu.requests, kubernetes.filesystem.usage,gauge,,byte,,The amount of disk used,-1,kubernetes,k8s.disk.usage, kubernetes.filesystem.usage_pct,gauge,,fraction,,The percentage of disk used,-1,kubernetes,k8s.disk.used_pct, @@ -24,6 +26,8 @@ kubernetes.memory.capacity,gauge,,byte,,The amount of memory (in bytes) in this kubernetes.memory.limits,gauge,,byte,,The limit of memory set,0,kubernetes,k8s.mem.limits, kubernetes.memory.sw_limit,gauge,,byte,,The limit of swap space set,0,kubernetes,k8s.mem.sw_limit, kubernetes.memory.requests,gauge,,byte,,The requested memory,0,kubernetes,k8s.mem.requests, +kubernetes.pod.memory.request,gauge,,byte,,The pod-level requested memory in bytes,0,kubernetes,k8s.pod.mem.request, +kubernetes.pod.memory.limit,gauge,,byte,,The pod-level memory limit in bytes,0,kubernetes,k8s.pod.mem.limit, kubernetes.memory.usage,gauge,,byte,,Current memory usage in bytes including all memory regardless of when it was accessed,-1,kubernetes,k8s.mem, kubernetes.memory.working_set,gauge,,byte,,Current working set in bytes - this is what the OOM killer is watching for,-1,kubernetes,k8s.mem.ws, kubernetes.memory.cache,gauge,,byte,,The amount of memory that is being used to cache data from disk (e.g. memory contents that can be associated precisely with a block on a block device),-1,kubernetes,k8s.mem.cache, From faacce572d9952acfa6b44dc051418d1341b7e1b Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Wed, 27 May 2026 15:50:03 -0400 Subject: [PATCH 10/44] [anthropic_usage_and_costs] Update README to include information about setting up analytics keys (#23852) * README changes to add analytics information * more readme updates * more readme * ascii * Apply suggestions from code review Co-authored-by: Eva Parish * Apply suggestion from @evazorro --------- Co-authored-by: Eva Parish --- anthropic_usage_and_costs/README.md | 40 +++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/anthropic_usage_and_costs/README.md b/anthropic_usage_and_costs/README.md index 0b8a57878c0e7..240b76fd9c356 100644 --- a/anthropic_usage_and_costs/README.md +++ b/anthropic_usage_and_costs/README.md @@ -2,37 +2,54 @@ ## Overview -Datadog's Anthropic Usage and Costs integration allows you to get visibility into your Anthropic usage and associated costs. By ingesting data from Anthropic's newly released Admin usage and cost API, this integration enables your teams to: +Datadog's Anthropic Usage and Costs integration allows you to get visibility into your Anthropic usage and associated costs. By ingesting data from Anthropic's Admin and Analytics usage and cost APIs, this integration enables your teams to: - **Monitor LLM token consumption** (input, output, cache usage) in near real-time - **Track costs** by model, workspace, and service tier, supporting accurate attribution and budgeting +- **Attribute usage and spend to individual users** (Enterprise plans): break down token consumption and dollar cost by user, product (API, Claude Code, Claude.ai), context window, inference geography, and speed - **Understand usage trends** across teams, API keys, or users to optimize model usage - **Set up alerting and dashboards** that highlight anomalies in usage or unexpected cost spikes This integration is especially valuable for teams using Anthropic at scale who want to manage spend, understand product adoption, and ensure efficient use of AI resources-all within Datadog. With this data you will be able to introduce and validate optimization strategies to get the best out of Anthropic. -You can also see your Anthropic costs in Datadog [Cloud Cost Management][6], allowing you to answer key questions: Which models or workspaces are generating the most cost? Are workloads using the right service tier (Standard, Batch, or Priority)? Are teams effectively using caching or ephemeral sessions? What's the cost breakdown between Claude Opus and Claude Sonnet? +You can also see your Anthropic costs in Datadog [Cloud Cost Management][6], allowing you to answer key questions: Which models or workspaces are generating the most cost? Which users or teams are driving the most spend? Are workloads using the right service tier (Standard, Batch, or Priority)? Are teams effectively using caching or ephemeral sessions? What's the cost breakdown between Claude Opus and Claude Sonnet? **Minimum Agent version:** 7.69.0 ## Setup -To get started with the Anthropic Admin API integration in Datadog, follow the steps below: +To get started with the Anthropic integration in Datadog, follow the steps below: -### 1. Generate an Admin API Key +### 1. Identify your Anthropic plan and the key type you'll use -You will need an [Admin API key][5] from Anthropic. This key allows access to usage and cost reports across your organization. +Your Anthropic organization is on either a **Platform** plan or an **Enterprise** plan, and the plan determines which API key type you use to authenticate this integration: -1. Navigate to your organization's settings or reach out to your Anthropic account admin to create a new Admin API key. -2. Copy the API key to a secure location. +| Anthropic plan | API key type | Data ingested | +| --- | --- | --- | +| **Platform** | **Admin API key** | Organization-wide usage and cost, broken down by model, workspace, API key, and service tier. | +| **Enterprise** | **Analytics API key** | Per-user usage and cost, broken down by user, product (API, Claude Code, Claude.ai), model, context window, inference geography, and speed. Usage metrics are emitted in **1-hour buckets** after the hour closes on Anthropic's side, with a typical end-to-end latency of **1-3 hours**. Per-user cost data covers usage from **January 1, 2026** onward. | -### 2. Configure the Datadog Integration +Each Anthropic organization issues only one of these key types based on its plan, so most customers will configure exactly one key. If your company operates more than one Anthropic organization (for example, a Platform org and an Enterprise org), configure each one as a separate Datadog account. + +All metrics and Cloud Cost Management line items emitted from an Enterprise (Analytics) account carry the `org_type:enterprise` tag. Platform (Admin) data is emitted without an `org_type` tag, so you can separate the two data sources in dashboards, monitors, and Cloud Cost Management filters using the presence (or absence) of `org_type:enterprise`. + +### 2. Generate your Anthropic API key + +Follow the path that matches your Anthropic plan: + +- **Platform plan**: Generate an [Admin API key][5] from your Anthropic organization's settings, or ask your Anthropic account admin to create one for you. +- **Enterprise plan**: Generate an Analytics API key (with the `read:analytics` scope) at [claude.ai/analytics/api-keys][7]. Analytics API keys must be provisioned by your organization's **Primary Owner**. + +Copy the API key to a secure location after you generate it. + +### 3. Configure the Datadog integration 1. In Datadog, go to [**Integrations > Anthropic Usage and Costs**](https://app.datadoghq.com/integrations?integrationId=anthropic-usage-and-costs). -2. In the configuration panel, provide the **Admin API Key** by pasting the key you generated from Anthropic. -3. Click **Save Configuration**. +2. In the configuration panel, paste your **Admin API Key** or **Analytics API Key** into the API key field. Datadog automatically detects the key type and ingests the appropriate usage and cost data. +3. (Optional) Enable **Cost data ingestion** to send cost data to [Cloud Cost Management][6]. This requires Cloud Cost Management to be enabled on your Datadog account. +4. Click **Save Configuration**. -Once saved, Datadog will begin polling Anthropic usage and cost endpoints using this key and populate metrics in your environment. +After you save the configuration, Datadog begins polling the appropriate Anthropic usage and cost endpoints and populates metrics in your environment. Usage metrics typically appear within 10 minutes, and cost data appears in Cloud Cost Management within 24 hours. ## Data Collected @@ -58,3 +75,4 @@ Need help? Contact [Datadog support][3]. [4]: https://github.com/DataDog/integrations-core/blob/master/anthropic_usage_and_costs/metadata.csv [5]: https://docs.anthropic.com/en/api/administration-api [6]: /cost +[7]: https://claude.ai/analytics/api-keys From f2ec2d2014020985e33b16a8f5953dff0e10600d Mon Sep 17 00:00:00 2001 From: Gouri Yerra Date: Wed, 27 May 2026 13:57:51 -0600 Subject: [PATCH 11/44] gpu: document required system-probe.yaml flag and gpu.enabled for non-containerized Linux hosts (#23853) * Update GPU README to document required system-probe.yaml flag and gpu.enabled for non-containerized hosts Co-Authored-By: Claude Sonnet 4.6 * Address review feedback: style and consistency fixes in GPU README Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 --- gpu/README.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/gpu/README.md b/gpu/README.md index ca83e18b5aaa8..31650d2207bcb 100644 --- a/gpu/README.md +++ b/gpu/README.md @@ -33,21 +33,32 @@ The check also uses eBPF probes to assign GPU usage and performance metrics to p #### Host -The agent needs to be configured to enable GPU-related features. Add the following parameters to the `/etc/datadog-agent/datadog.yaml` configuration file and then restart the Agent: +GPU monitoring requires configuration in both `/etc/datadog-agent/datadog.yaml` and `/etc/datadog-agent/system-probe.yaml`. Configuring only one of these files results in incomplete metrics collection. + +1. Add the following parameters to `/etc/datadog-agent/datadog.yaml`: ```yaml +gpu: + enabled: true collect_gpu_tags: true enable_nvml_detection: true ``` -Enabling the `gpu` integration requires `system-probe` to have the configuration option enabled for collecting per-process metrics. Inside the `/etc/datadog-agent/system-probe.yaml` configuration file, the following parameters must be set: +2. Add the following parameter to `/etc/datadog-agent/system-probe.yaml`. This flag loads the eBPF module responsible for per-process GPU metrics and is required even for non-containerized hosts: ```yaml gpu_monitoring: enabled: true ``` -The check in the Agent configuration file is enabled by default whenever NVIDIA GPUs and their drivers are detected in the system, as long as the `enable_nvml_detection` parameter is set to `true`. However, it can also be configured manually following these steps: +3. Restart both the Agent and system-probe: + +```shell +sudo systemctl restart datadog-agent +sudo systemctl restart datadog-agent-sysprobe +``` + +The check in the Agent configuration file is enabled by default whenever NVIDIA GPUs and their drivers are detected in the system, as long as the `enable_nvml_detection` parameter is set to `true`. The check can also be configured manually following these steps: 1. Edit the `gpu.d/conf.yaml` file, in the `conf.d/` folder at the root of your Agent's configuration directory, to start collecting your GPU performance data. From d0a239fdbc5a56cd58ab41de173fbc2dfc076420 Mon Sep 17 00:00:00 2001 From: Juanpe Araque Date: Thu, 28 May 2026 09:43:10 +0100 Subject: [PATCH 12/44] Wait for gamesim_primary index online before running couchbase tests (#23850) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Wait for the gamesim_primary index to be online before running tests test_index_stats_metrics asserts metrics tagged with index_name:gamesim_primary, which the Couchbase indexer only reports once the bundled GSI index for gamesim-sample finishes building. Add a WaitFor against /api/v1/stats on port 9102 (Couchbase 7+) so the environment fixture blocks until the indexer is actually reporting that keyspace. Also fix the inverted exit condition in load_sample_bucket(): the loop was breaking the moment the task appeared in /pools/default/tasks instead of when it disappeared, so it returned while the sample load was still running. * Confirm gamesim_primary build completes before yielding environment Three follow-ups from review: - Check `initial_build_progress == 100` on the keyspace; the indexer also publishes stats while an index is still building. - Use `raise_for_status()` so 401/404/5xx surface immediately instead of collapsing into a 60s WaitFor timeout. - Drop the >=7 conditional around the new WaitFor; the test matrix only covers 7.x and the test that needs the index is itself gated on 7+. - Bail out of load_sample_bucket if the install response carries no matching task entry, instead of spinning on `None == None`. * Require every gamesim_primary keyspace to report build progress 100 * Condense gamesim_primary_index_ready docstring to a one-liner * Clarify gamesim_primary_index_ready and log keyspaces on the negative path - Refactor to a list comprehension that separates 'no keyspace yet' from 'still building' so each retry path is self-documenting. - Print the keyspaces seen on the negative path so a future WaitFor timeout doesn't leave a debugging dead end. - Document why load_sample_bucket bails out when the install response carries no matching task — the next gamesim_primary_index_ready WaitFor is the real readiness gate. --- couchbase/tests/conftest.py | 39 ++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/couchbase/tests/conftest.py b/couchbase/tests/conftest.py index 5d5136982e14c..002ee072a145d 100644 --- a/couchbase/tests/conftest.py +++ b/couchbase/tests/conftest.py @@ -75,6 +75,7 @@ def dd_environment(): WaitFor(bucket_stats), WaitFor(load_sample_bucket), WaitFor(create_syncgw_database), + WaitFor(gamesim_primary_index_ready), ] with docker_run( compose_file=os.path.join(HERE, 'compose', 'docker-compose.yaml'), @@ -215,10 +216,15 @@ def load_sample_bucket(): if task["sample"] == "gamesim-sample": task_id = task["taskId"] + # No matching task in the install response — the bucket is likely loading + # under a task we can't observe. WaitFor will retry; on the retry the install + # POST takes the already-loaded shortcut, and gamesim_primary_index_ready is + # the authoritative gate that blocks until the bundled GSI is online. + if task_id is None: + return False + while True: # Loop until the task ID is gone, meaning the task is done. - task_is_done = False - r = requests.get( '{}/pools/default/tasks'.format(URL), auth=(USER, PASSWORD), @@ -226,11 +232,8 @@ def load_sample_bucket(): r.raise_for_status() result = r.json() - for task in result: - if task.get("task_id", "") == task_id: - task_is_done = True - - if task_is_done: + task_still_running = any(task.get("task_id") == task_id for task in result) + if not task_still_running: break time.sleep(1) @@ -238,6 +241,28 @@ def load_sample_bucket(): return True +def gamesim_primary_index_ready(): + """Wait until every gamesim_primary keyspace reports initial_build_progress == 100.""" + r = requests.get( + '{}/api/v1/stats'.format(INDEX_STATS_URL), + auth=(USER, PASSWORD), + ) + r.raise_for_status() + + data = r.json() + matches = [ + stats + for keyspace, stats in data.items() + if keyspace != "indexer" + and keyspace.split(":")[0] == "gamesim-sample" + and keyspace.split(":")[-1] == "gamesim_primary" + ] + if not matches: + print("gamesim_primary not yet visible; keyspaces seen: {}".format(list(data.keys()))) + return False + return all(s.get("initial_build_progress") == 100 for s in matches) + + def create_syncgw_database(): """ Create sample database From 00dcdbf54e9098c13de480def8d29257b8a85741 Mon Sep 17 00:00:00 2001 From: dkirov-dd <166512750+dkirov-dd@users.noreply.github.com> Date: Thu, 28 May 2026 10:55:49 +0200 Subject: [PATCH 13/44] Keep release tooling separate from source refs (#23837) * Honor source ref in release dispatch Keep release workflow tooling checked out separately from the source tree being released. This lets the workflow use current setup actions and release scripts while ddev tags and validates the requested source-repo-ref. Add source-repo-branch to manual release dispatches so stable versus pre-release validation follows the branch that contains source-repo-ref instead of the branch used to launch the workflow. * align source-repo-branch description with release-trigger.yml --- .github/workflows/release-dispatch.yml | 65 +++++++++++++++++--------- .github/workflows/release-trigger.yml | 15 ++++-- 2 files changed, 55 insertions(+), 25 deletions(-) diff --git a/.github/workflows/release-dispatch.yml b/.github/workflows/release-dispatch.yml index ed5b1051575fa..42e8fcb02e954 100644 --- a/.github/workflows/release-dispatch.yml +++ b/.github/workflows/release-dispatch.yml @@ -20,6 +20,10 @@ on: description: "Commit SHA or ref to build from" required: false type: string + source-repo-branch: + description: "Branch that contains source-repo-ref, used to determine stable vs pre-release behavior" + required: false + type: string dry-run: description: >- When true, print what would be released and where without pushing tags @@ -56,20 +60,42 @@ jobs: batches: ${{ steps.release-dispatch.outputs.batches }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # Workflow tooling (composite actions + release scripts) always comes from + # integrations-core at the workflow's own commit. When called from another + # repo we don't have that commit available locally, so we fall back to + # master. This is decoupled from inputs.source-repo-ref on purpose so the + # release pipeline can build older refs without losing recent tooling. + - name: Checkout workflow tooling + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - ref: ${{ inputs.source-repo-ref || github.sha }} - fetch-depth: 0 # ddev needs full tag history + repository: DataDog/integrations-core + ref: ${{ github.repository == 'DataDog/integrations-core' && github.sha || 'master' }} + fetch-depth: 1 + sparse-checkout: .github + path: tooling - - name: Checkout integrations-core actions - if: github.repository != 'DataDog/integrations-core' + # Source tree to tag and validate. ddev release tag operates on HEAD of + # this checkout, so it must be at the ref the caller asked to release. + - name: Checkout source repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - repository: "DataDog/integrations-core" - ref: master - fetch-depth: 1 - sparse-checkout: .github/actions - path: core-temp + ref: ${{ inputs.source-repo-ref || github.sha }} + fetch-depth: 0 # ddev needs full tag history + path: source + + - name: Verify source ref is on release branch + if: inputs.source-repo-branch != '' + working-directory: source + env: + SOURCE_REF: ${{ inputs.source-repo-ref || github.sha }} + SOURCE_BRANCH: ${{ inputs.source-repo-branch }} + run: | + branch="${SOURCE_BRANCH#refs/heads/}" + git fetch --no-tags origin "+refs/heads/${branch}:refs/remotes/origin/${branch}" + if ! git merge-base --is-ancestor HEAD "refs/remotes/origin/${branch}"; then + echo "::error::source-repo-ref '${SOURCE_REF}' is not contained in source-repo-branch '${SOURCE_BRANCH}'" + exit 1 + fi - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -77,47 +103,42 @@ jobs: python-version: "${{ env.PYTHON_VERSION }}" - name: Install ddev - if: github.repository == 'DataDog/integrations-core' - uses: ./.github/actions/setup-ddev - with: - install-mode: pypi - ddev-version: "==${{ inputs.ddev-version || env.DEFAULT_DDEV_VERSION }}" - - - name: Install ddev - if: github.repository != 'DataDog/integrations-core' - uses: ./core-temp/.github/actions/setup-ddev + uses: ./tooling/.github/actions/setup-ddev with: install-mode: pypi ddev-version: "==${{ inputs.ddev-version || env.DEFAULT_DDEV_VERSION }}" - name: Configure ddev + working-directory: source env: SOURCE_REPO: ${{ inputs.source-repo || 'integrations-core' }} run: | REPO_SHORT="${SOURCE_REPO#integrations-}" ddev config set upgrade_check false - ddev config set repos.${REPO_SHORT} . + ddev config set repos.${REPO_SHORT} "$PWD" ddev config set repo ${REPO_SHORT} - name: Prepare dispatch id: prepare + working-directory: source env: DRY_RUN: ${{ inputs.dry-run }} SELECTED_PACKAGES: ${{ inputs.packages }} SOURCE_REPO: ${{ inputs.source-repo || 'integrations-core' }} REF: ${{ inputs.source-repo-ref || github.sha }} IS_STABLE_RELEASE: ${{ inputs.is-stable-release }} - run: python .github/workflows/scripts/release_prepare.py + run: python "$GITHUB_WORKSPACE/tooling/.github/workflows/scripts/release_prepare.py" - name: Build dispatch batches id: release-dispatch if: steps.prepare.outputs.has_packages == 'true' + working-directory: source env: PACKAGES: ${{ steps.prepare.outputs.packages }} SOURCE_REPO: ${{ inputs.source-repo || 'integrations-core' }} REF: ${{ inputs.source-repo-ref || github.sha }} DRY_RUN: ${{ inputs.dry-run }} - run: python .github/workflows/scripts/release_dispatch.py + run: python "$GITHUB_WORKSPACE/tooling/.github/workflows/scripts/release_dispatch.py" dispatch: name: Dispatch wheel builds (batch ${{ strategy.job-index }}) diff --git a/.github/workflows/release-trigger.yml b/.github/workflows/release-trigger.yml index d5d1c542d3c2c..060b3068fdf95 100644 --- a/.github/workflows/release-trigger.yml +++ b/.github/workflows/release-trigger.yml @@ -27,6 +27,10 @@ on: description: "Commit SHA or ref to build from" required: true type: string + source-repo-branch: + description: "Branch that contains source-repo-ref, used to determine stable vs pre-release behavior" + required: true + type: string dry-run: description: "Print what would be released without pushing tags or starting builds" required: false @@ -45,10 +49,14 @@ jobs: is-stable-release: ${{ steps.detect.outputs.is-stable-release }} steps: - id: detect - # Sets is-stable-release based on the branch: true for master/X.Y.x, false otherwise. - # Manual runs on master get "true", which blocks pre-release packages — conservative and intentional. + # Stable for master/X.Y.x, pre-release for alpha/beta/rc branches. + # Manual dispatches use source-repo-branch instead of GITHUB_REF so the + # release behavior follows the branch that contains source-repo-ref. + env: + RELEASE_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.source-repo-branch || github.ref_name }} run: | - if [[ "$GITHUB_REF" =~ ^refs/heads/(master|[0-9]+\.[0-9]+\.x)$ ]]; then + branch="${RELEASE_BRANCH#refs/heads/}" + if [[ "$branch" =~ ^(master|[0-9]+\.[0-9]+\.x)$ ]]; then echo "is-stable-release=true" >> "$GITHUB_OUTPUT" else echo "is-stable-release=false" >> "$GITHUB_OUTPUT" @@ -70,6 +78,7 @@ jobs: source-repo: integrations-core packages: ${{ inputs.packages || '' }} source-repo-ref: ${{ github.event_name == 'workflow_dispatch' && inputs.source-repo-ref || github.sha }} + source-repo-branch: ${{ github.event_name == 'workflow_dispatch' && inputs.source-repo-branch || github.ref_name }} dry-run: ${{ inputs.dry-run || false }} ddev-version: ${{ inputs.ddev-version || '' }} is-stable-release: ${{ needs.context.outputs.is-stable-release }} From d34ad02914cd7024b1fa65c01a9dd9f2fd708069 Mon Sep 17 00:00:00 2001 From: Samy YACEF Date: Thu, 28 May 2026 12:02:38 +0200 Subject: [PATCH 14/44] [LOGSP-29] Add bypass annotations for linter checks (#23787) * [LOGSP-29] Add bypass annotations for linter checks A linter rule is being added or changed and some integrations are expected to remain non-compliant. Add permanent bypass annotations so CI stays green and does not fail for new PRs. * fix(logs-linter): bypass date-remapper-parse-failure-checks for known false positives These integrations either use log samples that intentionally lack timestamp source fields, or rely on date formats the test harness cannot parse. The date-remapper is functioning correctly in production for all affected integrations. Co-Authored-By: Claude Sonnet 4.6 (1M context) * Trigger CI --------- Co-authored-by: Claude Sonnet 4.6 (1M context) --- apache/assets/logs/apache.yaml | 1 + apache/assets/logs/apache_tests.yaml | 1 + .../assets/logs/arctic-wolf-aurora-endpoint-security_tests.yaml | 1 + argocd/assets/logs/argocd_tests.yaml | 1 + .../assets/logs/barracuda_secure_edge_tests.yaml | 1 + .../assets/logs/checkpoint-quantum-firewall_tests.yaml | 1 + cloudgen_firewall/assets/logs/cloudgen_firewall_tests.yaml | 1 + consul/assets/logs/consul_tests.yaml | 1 + coredns/assets/logs/coredns.yaml | 1 + coredns/assets/logs/coredns_tests.yaml | 1 + .../assets/logs/delinea-privilege-manager_tests.yaml | 1 + druid/assets/logs/druid_tests.yaml | 1 + gitlab/assets/logs/gitlab_tests.yaml | 1 + gitlab_runner/assets/logs/gitlab-runner_tests.yaml | 1 + haproxy/assets/logs/haproxy_tests.yaml | 1 + harbor/assets/logs/harbor_tests.yaml | 1 + kafka/assets/logs/kafka_tests.yaml | 1 + kuma/assets/logs/kuma_tests.yaml | 1 + microsoft_copilot/assets/logs/microsoft-copilot_tests.yaml | 1 + mysql/assets/logs/mysql_tests.yaml | 1 + nginx/assets/logs/nginx.yaml | 1 + nginx/assets/logs/nginx_tests.yaml | 1 + pan_firewall/assets/logs/pan.firewall_tests.yaml | 1 + plaid/assets/logs/plaid_tests.yaml | 1 + plivo/assets/logs/plivo_tests.yaml | 1 + riak/assets/logs/riak_tests.yaml | 1 + sonicwall_firewall/assets/logs/sonicwall-firewall_tests.yaml | 1 + tenable/assets/logs/tenable_tests.yaml | 1 + tomcat/assets/logs/tomcat_tests.yaml | 1 + traefik_mesh/assets/logs/traefik_tests.yaml | 1 + vault/assets/logs/vault_tests.yaml | 1 + vonage/assets/logs/vonage_tests.yaml | 1 + zk/assets/logs/zookeeper_tests.yaml | 1 + .../assets/logs/zscaler-private-access_tests.yaml | 1 + 34 files changed, 34 insertions(+) diff --git a/apache/assets/logs/apache.yaml b/apache/assets/logs/apache.yaml index b572eb435b2b9..9c77406452236 100644 --- a/apache/assets/logs/apache.yaml +++ b/apache/assets/logs/apache.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: apache metric_id: apache backend_only: false diff --git a/apache/assets/logs/apache_tests.yaml b/apache/assets/logs/apache_tests.yaml index c51d2753cac14..7c3e9ed25ae39 100644 --- a/apache/assets/logs/apache_tests.yaml +++ b/apache/assets/logs/apache_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "apache" tests: - diff --git a/arctic_wolf_aurora_endpoint_security/assets/logs/arctic-wolf-aurora-endpoint-security_tests.yaml b/arctic_wolf_aurora_endpoint_security/assets/logs/arctic-wolf-aurora-endpoint-security_tests.yaml index 058935fd711c8..bbc06821640a4 100644 --- a/arctic_wolf_aurora_endpoint_security/assets/logs/arctic-wolf-aurora-endpoint-security_tests.yaml +++ b/arctic_wolf_aurora_endpoint_security/assets/logs/arctic-wolf-aurora-endpoint-security_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "arctic-wolf-aurora-endpoint-security" tests: - diff --git a/argocd/assets/logs/argocd_tests.yaml b/argocd/assets/logs/argocd_tests.yaml index ce2e5cebc9798..fb0ca3bc756bd 100644 --- a/argocd/assets/logs/argocd_tests.yaml +++ b/argocd/assets/logs/argocd_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "argocd" tests: diff --git a/barracuda_secure_edge/assets/logs/barracuda_secure_edge_tests.yaml b/barracuda_secure_edge/assets/logs/barracuda_secure_edge_tests.yaml index 42c70d4625c66..10b793e9900fd 100644 --- a/barracuda_secure_edge/assets/logs/barracuda_secure_edge_tests.yaml +++ b/barracuda_secure_edge/assets/logs/barracuda_secure_edge_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: barracuda_secure_edge tests: - diff --git a/checkpoint_quantum_firewall/assets/logs/checkpoint-quantum-firewall_tests.yaml b/checkpoint_quantum_firewall/assets/logs/checkpoint-quantum-firewall_tests.yaml index 45c057af1c9b5..1620ca125123a 100644 --- a/checkpoint_quantum_firewall/assets/logs/checkpoint-quantum-firewall_tests.yaml +++ b/checkpoint_quantum_firewall/assets/logs/checkpoint-quantum-firewall_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "checkpoint-quantum-firewall" tests: diff --git a/cloudgen_firewall/assets/logs/cloudgen_firewall_tests.yaml b/cloudgen_firewall/assets/logs/cloudgen_firewall_tests.yaml index 879809ecc2a37..d18eea4e1bc88 100644 --- a/cloudgen_firewall/assets/logs/cloudgen_firewall_tests.yaml +++ b/cloudgen_firewall/assets/logs/cloudgen_firewall_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: cloudgen_firewall tests: - diff --git a/consul/assets/logs/consul_tests.yaml b/consul/assets/logs/consul_tests.yaml index 2ca9f74998753..1299069dd1ac7 100644 --- a/consul/assets/logs/consul_tests.yaml +++ b/consul/assets/logs/consul_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "consul" tests: diff --git a/coredns/assets/logs/coredns.yaml b/coredns/assets/logs/coredns.yaml index 57011c74cbd64..4bb9c1ed7d0bc 100644 --- a/coredns/assets/logs/coredns.yaml +++ b/coredns/assets/logs/coredns.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: coredns metric_id: coredns backend_only: false diff --git a/coredns/assets/logs/coredns_tests.yaml b/coredns/assets/logs/coredns_tests.yaml index 9eb3610f85326..7ed86e3c86139 100644 --- a/coredns/assets/logs/coredns_tests.yaml +++ b/coredns/assets/logs/coredns_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "coredns" tests: - diff --git a/delinea_privilege_manager/assets/logs/delinea-privilege-manager_tests.yaml b/delinea_privilege_manager/assets/logs/delinea-privilege-manager_tests.yaml index 4cceff815c45d..1f5df7aac64fe 100644 --- a/delinea_privilege_manager/assets/logs/delinea-privilege-manager_tests.yaml +++ b/delinea_privilege_manager/assets/logs/delinea-privilege-manager_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: delinea-privilege-manager tests: - diff --git a/druid/assets/logs/druid_tests.yaml b/druid/assets/logs/druid_tests.yaml index b37d6fab11911..501d496eef1f6 100644 --- a/druid/assets/logs/druid_tests.yaml +++ b/druid/assets/logs/druid_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "druid" tests: diff --git a/gitlab/assets/logs/gitlab_tests.yaml b/gitlab/assets/logs/gitlab_tests.yaml index 76f9dc6b5288b..2fc43f6b89e99 100644 --- a/gitlab/assets/logs/gitlab_tests.yaml +++ b/gitlab/assets/logs/gitlab_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "gitlab" tests: diff --git a/gitlab_runner/assets/logs/gitlab-runner_tests.yaml b/gitlab_runner/assets/logs/gitlab-runner_tests.yaml index e6f4a2c02689c..1bb0d2c32953f 100644 --- a/gitlab_runner/assets/logs/gitlab-runner_tests.yaml +++ b/gitlab_runner/assets/logs/gitlab-runner_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "gitlab-runner" tests: diff --git a/haproxy/assets/logs/haproxy_tests.yaml b/haproxy/assets/logs/haproxy_tests.yaml index 3dd1fb2e9a623..12e5d67e5ed40 100644 --- a/haproxy/assets/logs/haproxy_tests.yaml +++ b/haproxy/assets/logs/haproxy_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: haproxy tests: - sample: 10.0.1.2:33317 [06/Feb/2009:12:14:14.655] http-in static/srv1 10/0/30/69/109 200 2750 - - ---- 1/1/1/1/0 0/0 {1wt.eu} {} "GET /index.html HTTP/1.1" diff --git a/harbor/assets/logs/harbor_tests.yaml b/harbor/assets/logs/harbor_tests.yaml index c979bb1c452c2..5eddcfc77a643 100644 --- a/harbor/assets/logs/harbor_tests.yaml +++ b/harbor/assets/logs/harbor_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "harbor" tests: diff --git a/kafka/assets/logs/kafka_tests.yaml b/kafka/assets/logs/kafka_tests.yaml index 5a36845ba0c9e..00343fbbc39be 100644 --- a/kafka/assets/logs/kafka_tests.yaml +++ b/kafka/assets/logs/kafka_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "kafka" tests: diff --git a/kuma/assets/logs/kuma_tests.yaml b/kuma/assets/logs/kuma_tests.yaml index 876b406bd9f59..fa40b17145659 100644 --- a/kuma/assets/logs/kuma_tests.yaml +++ b/kuma/assets/logs/kuma_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: kuma tests: diff --git a/microsoft_copilot/assets/logs/microsoft-copilot_tests.yaml b/microsoft_copilot/assets/logs/microsoft-copilot_tests.yaml index 537acd9ad6686..1d1ded03dac5e 100644 --- a/microsoft_copilot/assets/logs/microsoft-copilot_tests.yaml +++ b/microsoft_copilot/assets/logs/microsoft-copilot_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: microsoft-copilot tests: - sample: >- diff --git a/mysql/assets/logs/mysql_tests.yaml b/mysql/assets/logs/mysql_tests.yaml index 144ed6c0edac5..005eccb18972d 100644 --- a/mysql/assets/logs/mysql_tests.yaml +++ b/mysql/assets/logs/mysql_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "mysql" tests: diff --git a/nginx/assets/logs/nginx.yaml b/nginx/assets/logs/nginx.yaml index 21508a86b2e0a..e36c1fa983bbf 100755 --- a/nginx/assets/logs/nginx.yaml +++ b/nginx/assets/logs/nginx.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: nginx metric_id: nginx backend_only: false diff --git a/nginx/assets/logs/nginx_tests.yaml b/nginx/assets/logs/nginx_tests.yaml index b3392b1576202..cc9b568b5bc02 100755 --- a/nginx/assets/logs/nginx_tests.yaml +++ b/nginx/assets/logs/nginx_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "nginx" tests: - sample: '127.0.0.1 - frank [13/Jul/2016:10:55:36 +0000] "GET /apache_pb.gif HTTP/1.0" 200 2326' diff --git a/pan_firewall/assets/logs/pan.firewall_tests.yaml b/pan_firewall/assets/logs/pan.firewall_tests.yaml index 6640136dc610e..620b0727552f6 100644 --- a/pan_firewall/assets/logs/pan.firewall_tests.yaml +++ b/pan_firewall/assets/logs/pan.firewall_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "pan.firewall" tests: - diff --git a/plaid/assets/logs/plaid_tests.yaml b/plaid/assets/logs/plaid_tests.yaml index d0e37ee789668..dfeaff3a76290 100644 --- a/plaid/assets/logs/plaid_tests.yaml +++ b/plaid/assets/logs/plaid_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "plaid" tests: - diff --git a/plivo/assets/logs/plivo_tests.yaml b/plivo/assets/logs/plivo_tests.yaml index f54636a02f8b7..0fc83b49751ad 100644 --- a/plivo/assets/logs/plivo_tests.yaml +++ b/plivo/assets/logs/plivo_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: plivo tests: - sample: >- diff --git a/riak/assets/logs/riak_tests.yaml b/riak/assets/logs/riak_tests.yaml index ff1a1f0f82cb3..c4fdc632805f1 100644 --- a/riak/assets/logs/riak_tests.yaml +++ b/riak/assets/logs/riak_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "riak" tests: - diff --git a/sonicwall_firewall/assets/logs/sonicwall-firewall_tests.yaml b/sonicwall_firewall/assets/logs/sonicwall-firewall_tests.yaml index e71e99993bfab..27dfb21a379a7 100644 --- a/sonicwall_firewall/assets/logs/sonicwall-firewall_tests.yaml +++ b/sonicwall_firewall/assets/logs/sonicwall-firewall_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "sonicwall-firewall" tests: - diff --git a/tenable/assets/logs/tenable_tests.yaml b/tenable/assets/logs/tenable_tests.yaml index e6716565e3055..6fd087762b73f 100644 --- a/tenable/assets/logs/tenable_tests.yaml +++ b/tenable/assets/logs/tenable_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "tenable" tests: - diff --git a/tomcat/assets/logs/tomcat_tests.yaml b/tomcat/assets/logs/tomcat_tests.yaml index 0d5f325f90ae5..d2314773a7726 100644 --- a/tomcat/assets/logs/tomcat_tests.yaml +++ b/tomcat/assets/logs/tomcat_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: tomcat tests: - sample: 2005-09-07 14:07:41,508 [main] INFO MyApp - Entering application. diff --git a/traefik_mesh/assets/logs/traefik_tests.yaml b/traefik_mesh/assets/logs/traefik_tests.yaml index 9e4e30e944404..2e90e7f32ffac 100644 --- a/traefik_mesh/assets/logs/traefik_tests.yaml +++ b/traefik_mesh/assets/logs/traefik_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "traefik" tests: - diff --git a/vault/assets/logs/vault_tests.yaml b/vault/assets/logs/vault_tests.yaml index 70f47c423a285..6295905cba662 100644 --- a/vault/assets/logs/vault_tests.yaml +++ b/vault/assets/logs/vault_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "vault" tests: - diff --git a/vonage/assets/logs/vonage_tests.yaml b/vonage/assets/logs/vonage_tests.yaml index b65d23eebcb34..1040b50afb5a2 100644 --- a/vonage/assets/logs/vonage_tests.yaml +++ b/vonage/assets/logs/vonage_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks Expected sample output: id: "vonage" tests: diff --git a/zk/assets/logs/zookeeper_tests.yaml b/zk/assets/logs/zookeeper_tests.yaml index 0acc153a3ce49..443a6e6b6cced 100644 --- a/zk/assets/logs/zookeeper_tests.yaml +++ b/zk/assets/logs/zookeeper_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "zookeeper" tests: diff --git a/zscaler_private_access/assets/logs/zscaler-private-access_tests.yaml b/zscaler_private_access/assets/logs/zscaler-private-access_tests.yaml index fe46f6bcf8419..18f0de70a98a9 100644 --- a/zscaler_private_access/assets/logs/zscaler-private-access_tests.yaml +++ b/zscaler_private_access/assets/logs/zscaler-private-access_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: zscaler-private-access tests: - From d2280ed9f914f8c3a32fdafa9a2d3490748faf02 Mon Sep 17 00:00:00 2001 From: Lucia Date: Thu, 28 May 2026 12:39:31 +0200 Subject: [PATCH 15/44] Remove Codecov following migration to Datadog Code Coverage (#23360) * WIP * Change orchestrator message * Add changelog and fix tests * Adjust Datadog code coverage gates * Sync Datadog code coverage services * Add integrations that don't pass threshold * Validate stale Datadog coverage services Detect services present in the Datadog Code Coverage config but no longer expected by ddev validate ci, and remove them when running with --sync. * Lower cilium threshold * Validate code coverage gates * Validate duplicate coverage services * Remove tests from coverage config * Exclude tests from coverage services * Lower coverage gates for migration exceptions --- .codecov.yml | 1895 ----------------- .github/workflows/config/labeler.yml | 2 +- .github/workflows/master-windows.yml | 9 - .github/workflows/master.yml | 9 - .../nightly-base-package-windows.yml | 2 - .github/workflows/nightly-base-package.yml | 2 - .github/workflows/pr-all-windows.yml | 9 - .github/workflows/pr-all.yml | 9 - .github/workflows/pr-test.yml | 9 - .github/workflows/pr.yml | 2 - .github/workflows/test-agent-target.yml | 2 - .github/workflows/test-agent-windows.yml | 2 - .github/workflows/test-agent.yml | 2 - .github/workflows/test-fips-e2e.yml | 12 +- .github/workflows/weekly-latest-windows.yml | 2 - .github/workflows/weekly-latest.yml | 2 - README.md | 4 +- code-coverage.datadog.yml | 715 +++++++ ddev/changelog.d/23360.changed | 1 + .../src/ddev/cli/validate/all/orchestrator.py | 2 +- ddev/src/ddev/cli/validate/ci.py | 266 +-- ddev/tests/cli/validate/all/test_github.py | 10 +- ddev/tests/cli/validate/test_ci.py | 213 +- docs/developer/.snippets/links.txt | 1 - docs/developer/index.md | 1 - docs/developer/meta/ci/labels.md | 2 +- docs/developer/meta/ci/validation.md | 2 +- 27 files changed, 934 insertions(+), 2253 deletions(-) delete mode 100644 .codecov.yml create mode 100644 ddev/changelog.d/23360.changed diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index b54d570280199..0000000000000 --- a/.codecov.yml +++ /dev/null @@ -1,1895 +0,0 @@ -comment: - layout: flags - behavior: default - branches: null -coverage: - range: 50..100 - round: down - precision: 2 - status: - default_rules: - flag_coverage_not_uploaded_behavior: exclude - project: - .NET_CLR: - target: 75 - flags: - - dotnetclr - ASP.NET: - target: 75 - flags: - - aspdotnet - AWS_Neuron: - target: 75 - flags: - - aws_neuron - ActiveMQ_XML: - target: 75 - flags: - - activemq_xml - Active_Directory: - target: 75 - flags: - - active_directory - Aerospike: - target: 75 - flags: - - aerospike - Airflow: - target: 75 - flags: - - airflow - Amazon_ECS_Fargate: - target: 75 - flags: - - ecs_fargate - Amazon_Kafka: - target: 75 - flags: - - amazon_msk - Ambari: - target: 75 - flags: - - ambari - Apache: - target: 75 - flags: - - apache - Apache_NiFi: - target: 75 - flags: - - nifi - Appgate_SDP: - target: 75 - flags: - - appgate_sdp - ArangoDB: - target: 75 - flags: - - arangodb - ArgoCD: - target: 75 - flags: - - argocd - Argo_Rollouts: - target: 75 - flags: - - argo_rollouts - Argo_Workflows: - target: 75 - flags: - - argo_workflows - Avi_Vantage: - target: 75 - flags: - - avi_vantage - Azure_IoT_Edge: - target: 75 - flags: - - azure_iot_edge - BentoML: - target: 75 - flags: - - bentoml - Boundary: - target: 75 - flags: - - boundary - Btrfs: - target: 75 - flags: - - btrfs - CRI-O: - target: 75 - flags: - - crio - Cacti: - target: 75 - flags: - - cacti - Calico: - target: 75 - flags: - - calico - Cassandra_Nodetool: - target: 75 - flags: - - cassandra_nodetool - Celery: - target: 75 - flags: - - celery - Ceph: - target: 75 - flags: - - ceph - Cilium: - target: 75 - flags: - - cilium - Cisco_ACI: - target: 75 - flags: - - cisco_aci - Citrix_Hypervisor: - target: 75 - flags: - - citrix_hypervisor - ClickHouse: - target: 75 - flags: - - clickhouse - Cloud_Foundry_API: - target: 75 - flags: - - cloud_foundry_api - Cloudera: - target: 75 - flags: - - cloudera - CockroachDB: - target: 75 - flags: - - cockroachdb - Consul: - target: 75 - flags: - - consul - Control-M: - target: 75 - flags: - - control_m - CoreDNS: - target: 75 - flags: - - coredns - CouchDB: - target: 75 - flags: - - couch - Couchbase: - target: 75 - flags: - - couchbase - DNS: - target: 75 - flags: - - dns_check - DO_Query_Actions: - target: 75 - flags: - - do_query_actions - Datadog_Checks_Base: - target: 75 - flags: - - datadog_checks_base - Datadog_Checks_Dev: - target: 75 - flags: - - datadog_checks_dev - Datadog_Checks_Downloader: - target: 75 - flags: - - datadog_checks_downloader - Datadog_Cluster_Agent: - target: 75 - flags: - - datadog_cluster_agent - Directory: - target: 75 - flags: - - directory - Disk: - target: 75 - flags: - - disk - Druid: - target: 75 - flags: - - druid - DuckDB: - target: 75 - flags: - - duckdb - EKS_Fargate: - target: 75 - flags: - - eks_fargate - ESXi: - target: 75 - flags: - - esxi - Elasticsearch: - target: 75 - flags: - - elastic - Envoy: - target: 75 - flags: - - envoy - Exchange_Server: - target: 75 - flags: - - exchange_server - External_DNS: - target: 75 - flags: - - external_dns - Falco: - target: 75 - flags: - - falco - Fluentd: - target: 75 - flags: - - fluentd - Fly.io: - target: 75 - flags: - - fly_io - FoundationDB: - target: 75 - flags: - - foundationdb - Gearman: - target: 75 - flags: - - gearmand - Gitlab: - target: 75 - flags: - - gitlab - Gitlab_Runner: - target: 75 - flags: - - gitlab_runner - GlusterFS: - target: 75 - flags: - - glusterfs - Go_Expvar: - target: 75 - flags: - - go_expvar - GuardDog: - target: 75 - flags: - - guarddog - Gunicorn: - target: 75 - flags: - - gunicorn - HAProxy: - target: 75 - flags: - - haproxy - HDFS_Datanode: - target: 75 - flags: - - hdfs_datanode - HDFS_Namenode: - target: 75 - flags: - - hdfs_namenode - HTTP: - target: 75 - flags: - - http_check - Harbor: - target: 75 - flags: - - harbor - Hazelcast: - target: 75 - flags: - - hazelcast - Hugging_Face_TGI: - target: 75 - flags: - - hugging_face_tgi - IBM_ACE: - target: 75 - flags: - - ibm_ace - IBM_Db2: - target: 75 - flags: - - ibm_db2 - IBM_MQ: - target: 75 - flags: - - ibm_mq - IBM_Spectrum_LSF: - target: 75 - flags: - - ibm_spectrum_lsf - IBM_WAS: - target: 75 - flags: - - ibm_was - IBM_i: - target: 75 - flags: - - ibm_i - IIS: - target: 75 - flags: - - iis - Impala: - target: 75 - flags: - - impala - Infiniband: - target: 75 - flags: - - infiniband - Istio: - target: 75 - flags: - - istio - Kafka_Actions: - target: 75 - flags: - - kafka_actions - Kafka_Consumer: - target: 75 - flags: - - kafka_consumer - Karpenter: - target: 75 - flags: - - karpenter - Keda: - target: 75 - flags: - - keda - Kong: - target: 75 - flags: - - kong - KrakenD: - target: 75 - flags: - - krakend - KubeVirt_API: - target: 75 - flags: - - kubevirt_api - KubeVirt_Controller: - target: 75 - flags: - - kubevirt_controller - KubeVirt_Handler: - target: 75 - flags: - - kubevirt_handler - Kube_DNS: - target: 75 - flags: - - kube_dns - Kube_Proxy: - target: 75 - flags: - - kube_proxy - Kube_metrics_server: - target: 75 - flags: - - kube_metrics_server - Kubeflow: - target: 75 - flags: - - kubeflow - Kubelet: - target: 75 - flags: - - kubelet - Kubernetes_API_server_metrics: - target: 75 - flags: - - kube_apiserver_metrics - Kubernetes_Cluster_Autoscaler: - target: 75 - flags: - - kubernetes_cluster_autoscaler - Kubernetes_Controller_Manager: - target: 75 - flags: - - kube_controller_manager - Kubernetes_Scheduler: - target: 75 - flags: - - kube_scheduler - Kubernetes_State: - target: 75 - flags: - - kubernetes_state - Kuma: - target: 75 - flags: - - kuma - Kyoto_Tycoon: - target: 75 - flags: - - kyototycoon - LPARStats: - target: 75 - flags: - - lparstats - Lighttpd: - target: 75 - flags: - - lighttpd - Linkerd: - target: 75 - flags: - - linkerd - Linux_proc_extras: - target: 75 - flags: - - linux_proc_extras - LiteLLM: - target: 75 - flags: - - litellm - Lustre: - target: 75 - flags: - - lustre - Mac_Audit_Logs: - target: 75 - flags: - - mac_audit_logs - MapR: - target: 75 - flags: - - mapr - MapReduce: - target: 75 - flags: - - mapreduce - Marathon: - target: 75 - flags: - - marathon - MarkLogic: - target: 75 - flags: - - marklogic - Memcached: - target: 75 - flags: - - mcache - Mesos: - target: 75 - flags: - - mesos_slave - Mesos_Master: - target: 75 - flags: - - mesos_master - Milvus: - target: 75 - flags: - - milvus - MongoDB: - target: 75 - flags: - - mongo - MySQL: - target: 75 - flags: - - mysql - NFSstat: - target: 75 - flags: - - nfsstat - NGINX: - target: 75 - flags: - - nginx - NGINX_Ingress_Controller: - target: 75 - flags: - - nginx_ingress_controller - Nagios: - target: 75 - flags: - - nagios - Network: - target: 75 - flags: - - network - Nutanix: - target: 75 - flags: - - nutanix - Nvidia_Triton: - target: 75 - flags: - - nvidia_triton - Octopus_Deploy: - target: 75 - flags: - - octopus_deploy - OpenLDAP: - target: 75 - flags: - - openldap - OpenMetrics: - target: 75 - flags: - - openmetrics - OpenStack: - target: 50 - flags: - - openstack - OpenStack_Controller: - target: 75 - flags: - - openstack_controller - PDH: - target: 75 - flags: - - pdh_check - PGBouncer: - target: 75 - flags: - - pgbouncer - PHP-FPM: - target: 75 - flags: - - php_fpm - Postfix: - target: 75 - flags: - - postfix - Postgres: - target: 75 - flags: - - postgres - PowerDNS_Recursor: - target: 75 - flags: - - powerdns_recursor - Prefect: - target: 75 - flags: - - prefect - Process: - target: 75 - flags: - - process - Prometheus: - target: 75 - flags: - - prometheus - Proxmox: - target: 75 - flags: - - proxmox - ProxySQL: - target: 75 - flags: - - proxysql - Pulsar: - target: 75 - flags: - - pulsar - Quarkus: - target: 75 - flags: - - quarkus - RabbitMQ: - target: 75 - flags: - - rabbitmq - Ray: - target: 75 - flags: - - ray - Redis: - target: 75 - flags: - - redisdb - RethinkDB: - target: 75 - flags: - - rethinkdb - Riak: - target: 75 - flags: - - riak - RiakCS: - target: 75 - flags: - - riakcs - SAP_HANA: - target: 75 - flags: - - sap_hana - SNMP: - target: 30 - flags: - - snmp - SQL_Server: - target: 75 - flags: - - sqlserver - SSH: - target: 75 - flags: - - ssh_check - Scylla: - target: 75 - flags: - - scylla - Silk: - target: 75 - flags: - - silk - Silverstripe_CMS: - target: 75 - flags: - - silverstripe_cms - SingleStore: - target: 75 - flags: - - singlestore - Slurm: - target: 75 - flags: - - slurm - SonarQube: - target: 75 - flags: - - sonarqube - Spark: - target: 75 - flags: - - spark - Squid: - target: 75 - flags: - - squid - StatsD: - target: 75 - flags: - - statsd - Strimzi: - target: 75 - flags: - - strimzi - Supabase: - target: 75 - flags: - - supabase - Supervisord: - target: 75 - flags: - - supervisord - System_Core: - target: 75 - flags: - - system_core - System_Swap: - target: 75 - flags: - - system_swap - TCP: - target: 75 - flags: - - tcp_check - TLS: - target: 75 - flags: - - tls - TeamCity: - target: 75 - flags: - - teamcity - Tekton: - target: 75 - flags: - - tekton - Teleport: - target: 75 - flags: - - teleport - Temporal: - target: 75 - flags: - - temporal - Teradata: - target: 75 - flags: - - teradata - TokuMX: - target: 50 - flags: - - tokumx - TorchServe: - target: 75 - flags: - - torchserve - Traefik_Mesh: - target: 75 - flags: - - traefik_mesh - Traffic_Server: - target: 75 - flags: - - traffic_server - Twemproxy: - target: 75 - flags: - - twemproxy - Twistlock: - target: 75 - flags: - - twistlock - Varnish: - target: 75 - flags: - - varnish - Vault: - target: 75 - flags: - - vault - Velero: - target: 75 - flags: - - velero - Vertica: - target: 75 - flags: - - vertica - VoltDB: - target: 75 - flags: - - voltdb - WMI: - target: 75 - flags: - - wmi_check - Weaviate: - target: 75 - flags: - - weaviate - Windows_Event_Log: - target: 75 - flags: - - win32_event_log - Windows_Service: - target: 75 - flags: - - windows_service - Windows_performance_counters: - target: 75 - flags: - - windows_performance_counters - Yarn: - target: 75 - flags: - - yarn - ZooKeeper: - target: 75 - flags: - - zk - cert-manager: - target: 75 - flags: - - cert_manager - checkpoint_harmony_endpoint: - target: 75 - flags: - - checkpoint_harmony_endpoint - dcgm: - target: 75 - flags: - - dcgm - ddev: - target: 75 - flags: - - ddev - etcd: - target: 75 - flags: - - etcd - fluxcd: - target: 75 - flags: - - fluxcd - kyverno: - target: 75 - flags: - - kyverno - n8n: - target: 75 - flags: - - n8n - nvidia_nim: - target: 75 - flags: - - nvidia_nim - sonatype_nexus: - target: 75 - flags: - - sonatype_nexus - tibco_ems: - target: 75 - flags: - - tibco_ems - vLLM: - target: 75 - flags: - - vllm - vSphere: - target: 75 - flags: - - vsphere - patch: false -flags: - active_directory: - carryforward: true - paths: - - active_directory/datadog_checks/active_directory - - active_directory/tests - activemq_xml: - carryforward: true - paths: - - activemq_xml/datadog_checks/activemq_xml - - activemq_xml/tests - aerospike: - carryforward: true - paths: - - aerospike/datadog_checks/aerospike - - aerospike/tests - airflow: - carryforward: true - paths: - - airflow/datadog_checks/airflow - - airflow/tests - amazon_msk: - carryforward: true - paths: - - amazon_msk/datadog_checks/amazon_msk - - amazon_msk/tests - ambari: - carryforward: true - paths: - - ambari/datadog_checks/ambari - - ambari/tests - apache: - carryforward: true - paths: - - apache/datadog_checks/apache - - apache/tests - appgate_sdp: - carryforward: true - paths: - - appgate_sdp/datadog_checks/appgate_sdp - - appgate_sdp/tests - arangodb: - carryforward: true - paths: - - arangodb/datadog_checks/arangodb - - arangodb/tests - argo_rollouts: - carryforward: true - paths: - - argo_rollouts/datadog_checks/argo_rollouts - - argo_rollouts/tests - argo_workflows: - carryforward: true - paths: - - argo_workflows/datadog_checks/argo_workflows - - argo_workflows/tests - argocd: - carryforward: true - paths: - - argocd/datadog_checks/argocd - - argocd/tests - aspdotnet: - carryforward: true - paths: - - aspdotnet/datadog_checks/aspdotnet - - aspdotnet/tests - avi_vantage: - carryforward: true - paths: - - avi_vantage/datadog_checks/avi_vantage - - avi_vantage/tests - aws_neuron: - carryforward: true - paths: - - aws_neuron/datadog_checks/aws_neuron - - aws_neuron/tests - azure_iot_edge: - carryforward: true - paths: - - azure_iot_edge/datadog_checks/azure_iot_edge - - azure_iot_edge/tests - bentoml: - carryforward: true - paths: - - bentoml/datadog_checks/bentoml - - bentoml/tests - boundary: - carryforward: true - paths: - - boundary/datadog_checks/boundary - - boundary/tests - btrfs: - carryforward: true - paths: - - btrfs/datadog_checks/btrfs - - btrfs/tests - cacti: - carryforward: true - paths: - - cacti/datadog_checks/cacti - - cacti/tests - calico: - carryforward: true - paths: - - calico/datadog_checks/calico - - calico/tests - cassandra_nodetool: - carryforward: true - paths: - - cassandra_nodetool/datadog_checks/cassandra_nodetool - - cassandra_nodetool/tests - celery: - carryforward: true - paths: - - celery/datadog_checks/celery - - celery/tests - ceph: - carryforward: true - paths: - - ceph/datadog_checks/ceph - - ceph/tests - cert_manager: - carryforward: true - paths: - - cert_manager/datadog_checks/cert_manager - - cert_manager/tests - checkpoint_harmony_endpoint: - carryforward: true - paths: - - checkpoint_harmony_endpoint/datadog_checks/checkpoint_harmony_endpoint - - checkpoint_harmony_endpoint/tests - cilium: - carryforward: true - paths: - - cilium/datadog_checks/cilium - - cilium/tests - cisco_aci: - carryforward: true - paths: - - cisco_aci/datadog_checks/cisco_aci - - cisco_aci/tests - citrix_hypervisor: - carryforward: true - paths: - - citrix_hypervisor/datadog_checks/citrix_hypervisor - - citrix_hypervisor/tests - clickhouse: - carryforward: true - paths: - - clickhouse/datadog_checks/clickhouse - - clickhouse/tests - cloud_foundry_api: - carryforward: true - paths: - - cloud_foundry_api/datadog_checks/cloud_foundry_api - - cloud_foundry_api/tests - cloudera: - carryforward: true - paths: - - cloudera/datadog_checks/cloudera - - cloudera/tests - cockroachdb: - carryforward: true - paths: - - cockroachdb/datadog_checks/cockroachdb - - cockroachdb/tests - consul: - carryforward: true - paths: - - consul/datadog_checks/consul - - consul/tests - control_m: - carryforward: true - paths: - - control_m/datadog_checks/control_m - - control_m/tests - coredns: - carryforward: true - paths: - - coredns/datadog_checks/coredns - - coredns/tests - couch: - carryforward: true - paths: - - couch/datadog_checks/couch - - couch/tests - couchbase: - carryforward: true - paths: - - couchbase/datadog_checks/couchbase - - couchbase/tests - crio: - carryforward: true - paths: - - crio/datadog_checks/crio - - crio/tests - datadog_checks_base: - carryforward: true - paths: - - datadog_checks_base/datadog_checks/base - - datadog_checks_base/tests - datadog_checks_dev: - carryforward: true - paths: - - datadog_checks_dev/datadog_checks/dev - - datadog_checks_dev/tests - datadog_checks_downloader: - carryforward: true - paths: - - datadog_checks_downloader/datadog_checks/downloader - - datadog_checks_downloader/tests - datadog_cluster_agent: - carryforward: true - paths: - - datadog_cluster_agent/datadog_checks/datadog_cluster_agent - - datadog_cluster_agent/tests - dcgm: - carryforward: true - paths: - - dcgm/datadog_checks/dcgm - - dcgm/tests - ddev: - carryforward: true - paths: - - ddev/src/ddev - - ddev/tests - directory: - carryforward: true - paths: - - directory/datadog_checks/directory - - directory/tests - disk: - carryforward: true - paths: - - disk/datadog_checks/disk - - disk/tests - dns_check: - carryforward: true - paths: - - dns_check/datadog_checks/dns_check - - dns_check/tests - do_query_actions: - carryforward: true - paths: - - do_query_actions/datadog_checks/do_query_actions - - do_query_actions/tests - dotnetclr: - carryforward: true - paths: - - dotnetclr/datadog_checks/dotnetclr - - dotnetclr/tests - druid: - carryforward: true - paths: - - druid/datadog_checks/druid - - druid/tests - duckdb: - carryforward: true - paths: - - duckdb/datadog_checks/duckdb - - duckdb/tests - ecs_fargate: - carryforward: true - paths: - - ecs_fargate/datadog_checks/ecs_fargate - - ecs_fargate/tests - eks_fargate: - carryforward: true - paths: - - eks_fargate/datadog_checks/eks_fargate - - eks_fargate/tests - elastic: - carryforward: true - paths: - - elastic/datadog_checks/elastic - - elastic/tests - envoy: - carryforward: true - paths: - - envoy/datadog_checks/envoy - - envoy/tests - esxi: - carryforward: true - paths: - - esxi/datadog_checks/esxi - - esxi/tests - etcd: - carryforward: true - paths: - - etcd/datadog_checks/etcd - - etcd/tests - exchange_server: - carryforward: true - paths: - - exchange_server/datadog_checks/exchange_server - - exchange_server/tests - external_dns: - carryforward: true - paths: - - external_dns/datadog_checks/external_dns - - external_dns/tests - falco: - carryforward: true - paths: - - falco/datadog_checks/falco - - falco/tests - fluentd: - carryforward: true - paths: - - fluentd/datadog_checks/fluentd - - fluentd/tests - fluxcd: - carryforward: true - paths: - - fluxcd/datadog_checks/fluxcd - - fluxcd/tests - fly_io: - carryforward: true - paths: - - fly_io/datadog_checks/fly_io - - fly_io/tests - foundationdb: - carryforward: true - paths: - - foundationdb/datadog_checks/foundationdb - - foundationdb/tests - gearmand: - carryforward: true - paths: - - gearmand/datadog_checks/gearmand - - gearmand/tests - gitlab: - carryforward: true - paths: - - gitlab/datadog_checks/gitlab - - gitlab/tests - gitlab_runner: - carryforward: true - paths: - - gitlab_runner/datadog_checks/gitlab_runner - - gitlab_runner/tests - glusterfs: - carryforward: true - paths: - - glusterfs/datadog_checks/glusterfs - - glusterfs/tests - go_expvar: - carryforward: true - paths: - - go_expvar/datadog_checks/go_expvar - - go_expvar/tests - guarddog: - carryforward: true - paths: - - guarddog/datadog_checks/guarddog - - guarddog/tests - gunicorn: - carryforward: true - paths: - - gunicorn/datadog_checks/gunicorn - - gunicorn/tests - haproxy: - carryforward: true - paths: - - haproxy/datadog_checks/haproxy - - haproxy/tests - harbor: - carryforward: true - paths: - - harbor/datadog_checks/harbor - - harbor/tests - hazelcast: - carryforward: true - paths: - - hazelcast/datadog_checks/hazelcast - - hazelcast/tests - hdfs_datanode: - carryforward: true - paths: - - hdfs_datanode/datadog_checks/hdfs_datanode - - hdfs_datanode/tests - hdfs_namenode: - carryforward: true - paths: - - hdfs_namenode/datadog_checks/hdfs_namenode - - hdfs_namenode/tests - http_check: - carryforward: true - paths: - - http_check/datadog_checks/http_check - - http_check/tests - hugging_face_tgi: - carryforward: true - paths: - - hugging_face_tgi/datadog_checks/hugging_face_tgi - - hugging_face_tgi/tests - ibm_ace: - carryforward: true - paths: - - ibm_ace/datadog_checks/ibm_ace - - ibm_ace/tests - ibm_db2: - carryforward: true - paths: - - ibm_db2/datadog_checks/ibm_db2 - - ibm_db2/tests - ibm_i: - carryforward: true - paths: - - ibm_i/datadog_checks/ibm_i - - ibm_i/tests - ibm_mq: - carryforward: true - paths: - - ibm_mq/datadog_checks/ibm_mq - - ibm_mq/tests - ibm_spectrum_lsf: - carryforward: true - paths: - - ibm_spectrum_lsf/datadog_checks/ibm_spectrum_lsf - - ibm_spectrum_lsf/tests - ibm_was: - carryforward: true - paths: - - ibm_was/datadog_checks/ibm_was - - ibm_was/tests - iis: - carryforward: true - paths: - - iis/datadog_checks/iis - - iis/tests - impala: - carryforward: true - paths: - - impala/datadog_checks/impala - - impala/tests - infiniband: - carryforward: true - paths: - - infiniband/datadog_checks/infiniband - - infiniband/tests - istio: - carryforward: true - paths: - - istio/datadog_checks/istio - - istio/tests - kafka_actions: - carryforward: true - paths: - - kafka_actions/datadog_checks/kafka_actions - - kafka_actions/tests - kafka_consumer: - carryforward: true - paths: - - kafka_consumer/datadog_checks/kafka_consumer - - kafka_consumer/tests - karpenter: - carryforward: true - paths: - - karpenter/datadog_checks/karpenter - - karpenter/tests - keda: - carryforward: true - paths: - - keda/datadog_checks/keda - - keda/tests - kong: - carryforward: true - paths: - - kong/datadog_checks/kong - - kong/tests - krakend: - carryforward: true - paths: - - krakend/datadog_checks/krakend - - krakend/tests - kube_apiserver_metrics: - carryforward: true - paths: - - kube_apiserver_metrics/datadog_checks/kube_apiserver_metrics - - kube_apiserver_metrics/tests - kube_controller_manager: - carryforward: true - paths: - - kube_controller_manager/datadog_checks/kube_controller_manager - - kube_controller_manager/tests - kube_dns: - carryforward: true - paths: - - kube_dns/datadog_checks/kube_dns - - kube_dns/tests - kube_metrics_server: - carryforward: true - paths: - - kube_metrics_server/datadog_checks/kube_metrics_server - - kube_metrics_server/tests - kube_proxy: - carryforward: true - paths: - - kube_proxy/datadog_checks/kube_proxy - - kube_proxy/tests - kube_scheduler: - carryforward: true - paths: - - kube_scheduler/datadog_checks/kube_scheduler - - kube_scheduler/tests - kubeflow: - carryforward: true - paths: - - kubeflow/datadog_checks/kubeflow - - kubeflow/tests - kubelet: - carryforward: true - paths: - - kubelet/datadog_checks/kubelet - - kubelet/tests - kubernetes_cluster_autoscaler: - carryforward: true - paths: - - kubernetes_cluster_autoscaler/datadog_checks/kubernetes_cluster_autoscaler - - kubernetes_cluster_autoscaler/tests - kubernetes_state: - carryforward: true - paths: - - kubernetes_state/datadog_checks/kubernetes_state - - kubernetes_state/tests - kubevirt_api: - carryforward: true - paths: - - kubevirt_api/datadog_checks/kubevirt_api - - kubevirt_api/tests - kubevirt_controller: - carryforward: true - paths: - - kubevirt_controller/datadog_checks/kubevirt_controller - - kubevirt_controller/tests - kubevirt_handler: - carryforward: true - paths: - - kubevirt_handler/datadog_checks/kubevirt_handler - - kubevirt_handler/tests - kuma: - carryforward: true - paths: - - kuma/datadog_checks/kuma - - kuma/tests - kyototycoon: - carryforward: true - paths: - - kyototycoon/datadog_checks/kyototycoon - - kyototycoon/tests - kyverno: - carryforward: true - paths: - - kyverno/datadog_checks/kyverno - - kyverno/tests - lparstats: - carryforward: true - paths: - - lparstats/datadog_checks/lparstats - - lparstats/tests - lighttpd: - carryforward: true - paths: - - lighttpd/datadog_checks/lighttpd - - lighttpd/tests - linkerd: - carryforward: true - paths: - - linkerd/datadog_checks/linkerd - - linkerd/tests - linux_proc_extras: - carryforward: true - paths: - - linux_proc_extras/datadog_checks/linux_proc_extras - - linux_proc_extras/tests - litellm: - carryforward: true - paths: - - litellm/datadog_checks/litellm - - litellm/tests - lustre: - carryforward: true - paths: - - lustre/datadog_checks/lustre - - lustre/tests - mac_audit_logs: - carryforward: true - paths: - - mac_audit_logs/datadog_checks/mac_audit_logs - - mac_audit_logs/tests - mapr: - carryforward: true - paths: - - mapr/datadog_checks/mapr - - mapr/tests - mapreduce: - carryforward: true - paths: - - mapreduce/datadog_checks/mapreduce - - mapreduce/tests - marathon: - carryforward: true - paths: - - marathon/datadog_checks/marathon - - marathon/tests - marklogic: - carryforward: true - paths: - - marklogic/datadog_checks/marklogic - - marklogic/tests - mcache: - carryforward: true - paths: - - mcache/datadog_checks/mcache - - mcache/tests - mesos_master: - carryforward: true - paths: - - mesos_master/datadog_checks/mesos_master - - mesos_master/tests - mesos_slave: - carryforward: true - paths: - - mesos_slave/datadog_checks/mesos_slave - - mesos_slave/tests - milvus: - carryforward: true - paths: - - milvus/datadog_checks/milvus - - milvus/tests - mongo: - carryforward: true - paths: - - mongo/datadog_checks/mongo - - mongo/tests - mysql: - carryforward: true - paths: - - mysql/datadog_checks/mysql - - mysql/tests - n8n: - carryforward: true - paths: - - n8n/datadog_checks/n8n - - n8n/tests - nagios: - carryforward: true - paths: - - nagios/datadog_checks/nagios - - nagios/tests - network: - carryforward: true - paths: - - network/datadog_checks/network - - network/tests - nfsstat: - carryforward: true - paths: - - nfsstat/datadog_checks/nfsstat - - nfsstat/tests - nginx: - carryforward: true - paths: - - nginx/datadog_checks/nginx - - nginx/tests - nginx_ingress_controller: - carryforward: true - paths: - - nginx_ingress_controller/datadog_checks/nginx_ingress_controller - - nginx_ingress_controller/tests - nifi: - carryforward: true - paths: - - nifi/datadog_checks/nifi - - nifi/tests - nutanix: - carryforward: true - paths: - - nutanix/datadog_checks/nutanix - - nutanix/tests - nvidia_nim: - carryforward: true - paths: - - nvidia_nim/datadog_checks/nvidia_nim - - nvidia_nim/tests - nvidia_triton: - carryforward: true - paths: - - nvidia_triton/datadog_checks/nvidia_triton - - nvidia_triton/tests - octopus_deploy: - carryforward: true - paths: - - octopus_deploy/datadog_checks/octopus_deploy - - octopus_deploy/tests - openldap: - carryforward: true - paths: - - openldap/datadog_checks/openldap - - openldap/tests - openmetrics: - carryforward: true - paths: - - openmetrics/datadog_checks/openmetrics - - openmetrics/tests - openstack: - carryforward: true - paths: - - openstack/datadog_checks/openstack - - openstack/tests - openstack_controller: - carryforward: true - paths: - - openstack_controller/datadog_checks/openstack_controller - - openstack_controller/tests - pdh_check: - carryforward: true - paths: - - pdh_check/datadog_checks/pdh_check - - pdh_check/tests - pgbouncer: - carryforward: true - paths: - - pgbouncer/datadog_checks/pgbouncer - - pgbouncer/tests - php_fpm: - carryforward: true - paths: - - php_fpm/datadog_checks/php_fpm - - php_fpm/tests - postfix: - carryforward: true - paths: - - postfix/datadog_checks/postfix - - postfix/tests - postgres: - carryforward: true - paths: - - postgres/datadog_checks/postgres - - postgres/tests - powerdns_recursor: - carryforward: true - paths: - - powerdns_recursor/datadog_checks/powerdns_recursor - - powerdns_recursor/tests - prefect: - carryforward: true - paths: - - prefect/datadog_checks/prefect - - prefect/tests - process: - carryforward: true - paths: - - process/datadog_checks/process - - process/tests - prometheus: - carryforward: true - paths: - - prometheus/datadog_checks/prometheus - - prometheus/tests - proxmox: - carryforward: true - paths: - - proxmox/datadog_checks/proxmox - - proxmox/tests - proxysql: - carryforward: true - paths: - - proxysql/datadog_checks/proxysql - - proxysql/tests - pulsar: - carryforward: true - paths: - - pulsar/datadog_checks/pulsar - - pulsar/tests - quarkus: - carryforward: true - paths: - - quarkus/datadog_checks/quarkus - - quarkus/tests - rabbitmq: - carryforward: true - paths: - - rabbitmq/datadog_checks/rabbitmq - - rabbitmq/tests - ray: - carryforward: true - paths: - - ray/datadog_checks/ray - - ray/tests - redisdb: - carryforward: true - paths: - - redisdb/datadog_checks/redisdb - - redisdb/tests - rethinkdb: - carryforward: true - paths: - - rethinkdb/datadog_checks/rethinkdb - - rethinkdb/tests - riak: - carryforward: true - paths: - - riak/datadog_checks/riak - - riak/tests - riakcs: - carryforward: true - paths: - - riakcs/datadog_checks/riakcs - - riakcs/tests - sap_hana: - carryforward: true - paths: - - sap_hana/datadog_checks/sap_hana - - sap_hana/tests - scylla: - carryforward: true - paths: - - scylla/datadog_checks/scylla - - scylla/tests - silk: - carryforward: true - paths: - - silk/datadog_checks/silk - - silk/tests - silverstripe_cms: - carryforward: true - paths: - - silverstripe_cms/datadog_checks/silverstripe_cms - - silverstripe_cms/tests - singlestore: - carryforward: true - paths: - - singlestore/datadog_checks/singlestore - - singlestore/tests - slurm: - carryforward: true - paths: - - slurm/datadog_checks/slurm - - slurm/tests - snmp: - carryforward: true - paths: - - snmp/datadog_checks/snmp - - snmp/tests - sonarqube: - carryforward: true - paths: - - sonarqube/datadog_checks/sonarqube - - sonarqube/tests - sonatype_nexus: - carryforward: true - paths: - - sonatype_nexus/datadog_checks/sonatype_nexus - - sonatype_nexus/tests - spark: - carryforward: true - paths: - - spark/datadog_checks/spark - - spark/tests - sqlserver: - carryforward: true - paths: - - sqlserver/datadog_checks/sqlserver - - sqlserver/tests - squid: - carryforward: true - paths: - - squid/datadog_checks/squid - - squid/tests - ssh_check: - carryforward: true - paths: - - ssh_check/datadog_checks/ssh_check - - ssh_check/tests - statsd: - carryforward: true - paths: - - statsd/datadog_checks/statsd - - statsd/tests - strimzi: - carryforward: true - paths: - - strimzi/datadog_checks/strimzi - - strimzi/tests - supabase: - carryforward: true - paths: - - supabase/datadog_checks/supabase - - supabase/tests - supervisord: - carryforward: true - paths: - - supervisord/datadog_checks/supervisord - - supervisord/tests - system_core: - carryforward: true - paths: - - system_core/datadog_checks/system_core - - system_core/tests - system_swap: - carryforward: true - paths: - - system_swap/datadog_checks/system_swap - - system_swap/tests - tcp_check: - carryforward: true - paths: - - tcp_check/datadog_checks/tcp_check - - tcp_check/tests - teamcity: - carryforward: true - paths: - - teamcity/datadog_checks/teamcity - - teamcity/tests - tekton: - carryforward: true - paths: - - tekton/datadog_checks/tekton - - tekton/tests - teleport: - carryforward: true - paths: - - teleport/datadog_checks/teleport - - teleport/tests - temporal: - carryforward: true - paths: - - temporal/datadog_checks/temporal - - temporal/tests - teradata: - carryforward: true - paths: - - teradata/datadog_checks/teradata - - teradata/tests - tibco_ems: - carryforward: true - paths: - - tibco_ems/datadog_checks/tibco_ems - - tibco_ems/tests - tls: - carryforward: true - paths: - - tls/datadog_checks/tls - - tls/tests - tokumx: - carryforward: true - paths: - - tokumx/datadog_checks/tokumx - - tokumx/tests - torchserve: - carryforward: true - paths: - - torchserve/datadog_checks/torchserve - - torchserve/tests - traefik_mesh: - carryforward: true - paths: - - traefik_mesh/datadog_checks/traefik_mesh - - traefik_mesh/tests - traffic_server: - carryforward: true - paths: - - traffic_server/datadog_checks/traffic_server - - traffic_server/tests - twemproxy: - carryforward: true - paths: - - twemproxy/datadog_checks/twemproxy - - twemproxy/tests - twistlock: - carryforward: true - paths: - - twistlock/datadog_checks/twistlock - - twistlock/tests - varnish: - carryforward: true - paths: - - varnish/datadog_checks/varnish - - varnish/tests - vault: - carryforward: true - paths: - - vault/datadog_checks/vault - - vault/tests - velero: - carryforward: true - paths: - - velero/datadog_checks/velero - - velero/tests - vertica: - carryforward: true - paths: - - vertica/datadog_checks/vertica - - vertica/tests - vllm: - carryforward: true - paths: - - vllm/datadog_checks/vllm - - vllm/tests - voltdb: - carryforward: true - paths: - - voltdb/datadog_checks/voltdb - - voltdb/tests - vsphere: - carryforward: true - paths: - - vsphere/datadog_checks/vsphere - - vsphere/tests - weaviate: - carryforward: true - paths: - - weaviate/datadog_checks/weaviate - - weaviate/tests - win32_event_log: - carryforward: true - paths: - - win32_event_log/datadog_checks/win32_event_log - - win32_event_log/tests - windows_performance_counters: - carryforward: true - paths: - - windows_performance_counters/datadog_checks/windows_performance_counters - - windows_performance_counters/tests - windows_service: - carryforward: true - paths: - - windows_service/datadog_checks/windows_service - - windows_service/tests - wmi_check: - carryforward: true - paths: - - wmi_check/datadog_checks/wmi_check - - wmi_check/tests - yarn: - carryforward: true - paths: - - yarn/datadog_checks/yarn - - yarn/tests - zk: - carryforward: true - paths: - - zk/datadog_checks/zk - - zk/tests diff --git a/.github/workflows/config/labeler.yml b/.github/workflows/config/labeler.yml index 0ea2917444c7a..0363017ec6cf1 100644 --- a/.github/workflows/config/labeler.yml +++ b/.github/workflows/config/labeler.yml @@ -22,7 +22,7 @@ dev/testing: - changed-files: - any-glob-to-any-file: - .github/workflows/** - - .codecov.yml + - code-coverage.datadog.yml dev/tooling: - changed-files: - any-glob-to-any-file: diff --git a/.github/workflows/master-windows.yml b/.github/workflows/master-windows.yml index e1cdd3c037d71..0e810f7e1b299 100644 --- a/.github/workflows/master-windows.yml +++ b/.github/workflows/master-windows.yml @@ -78,8 +78,6 @@ jobs: (success() || failure()) runs-on: ubuntu-latest permissions: - # needed for codecov, allows the action to get a JWT signed by Github - id-token: write contents: read steps: @@ -92,13 +90,6 @@ jobs: path: coverage-reports merge-multiple: false - - name: Upload coverage to Codecov - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - directory: coverage-reports - fail_ci_if_error: false - - name: Upload coverage to Datadog if: always() continue-on-error: true diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 45786b66173ae..374a0cc211ba5 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -75,8 +75,6 @@ jobs: (success() || failure()) runs-on: ubuntu-latest permissions: - # needed for codecov, allows the action to get a JWT signed by Github - id-token: write contents: read steps: @@ -89,13 +87,6 @@ jobs: path: coverage-reports merge-multiple: false - - name: Upload coverage to Codecov - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - directory: coverage-reports - fail_ci_if_error: false - - name: Upload coverage to Datadog if: always() continue-on-error: true diff --git a/.github/workflows/nightly-base-package-windows.yml b/.github/workflows/nightly-base-package-windows.yml index 12ba22fa2b292..53f1d4094f507 100644 --- a/.github/workflows/nightly-base-package-windows.yml +++ b/.github/workflows/nightly-base-package-windows.yml @@ -17,8 +17,6 @@ jobs: uses: ./.github/workflows/test-all-windows.yml permissions: - # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/.github/workflows/nightly-base-package.yml b/.github/workflows/nightly-base-package.yml index d28d529ebe0ea..1d3459ebc02e9 100644 --- a/.github/workflows/nightly-base-package.yml +++ b/.github/workflows/nightly-base-package.yml @@ -15,8 +15,6 @@ jobs: uses: ./.github/workflows/test-all.yml permissions: - # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/.github/workflows/pr-all-windows.yml b/.github/workflows/pr-all-windows.yml index a3a5c824a1d7c..e9bc38d991eb0 100644 --- a/.github/workflows/pr-all-windows.yml +++ b/.github/workflows/pr-all-windows.yml @@ -52,8 +52,6 @@ jobs: (success() || failure()) runs-on: ubuntu-latest permissions: - # needed for codecov, allows the action to get a JWT signed by Github - id-token: write contents: read steps: @@ -66,13 +64,6 @@ jobs: path: coverage-reports merge-multiple: false - - name: Upload coverage to Codecov - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - directory: coverage-reports - fail_ci_if_error: false - - name: Upload coverage to Datadog if: always() continue-on-error: true diff --git a/.github/workflows/pr-all.yml b/.github/workflows/pr-all.yml index fb9fe8ca4cb30..0bb3bf1aafd48 100644 --- a/.github/workflows/pr-all.yml +++ b/.github/workflows/pr-all.yml @@ -55,8 +55,6 @@ jobs: (success() || failure()) runs-on: ubuntu-latest permissions: - # needed for codecov, allows the action to get a JWT signed by Github - id-token: write contents: read steps: @@ -69,13 +67,6 @@ jobs: path: coverage-reports merge-multiple: false - - name: Upload coverage to Codecov - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - directory: coverage-reports - fail_ci_if_error: false - - name: Upload coverage to Datadog if: always() continue-on-error: true diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 824471469d4dc..759ea53b9edd6 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -112,8 +112,6 @@ jobs: inputs.pytest-args != '-m flaky' runs-on: ubuntu-latest permissions: - # needed for codecov, allows the action to get a JWT signed by Github - id-token: write contents: read steps: @@ -126,13 +124,6 @@ jobs: path: coverage-reports merge-multiple: false - - name: Upload coverage to Codecov - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - directory: coverage-reports - fail_ci_if_error: false - - name: Upload coverage to Datadog if: always() continue-on-error: true diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a6810d189a3d0..7315836ddb7b4 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -24,7 +24,5 @@ jobs: secrets: inherit permissions: - # needed for codecov in pr-test.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/.github/workflows/test-agent-target.yml b/.github/workflows/test-agent-target.yml index 5b13976ce3381..a90be1218a566 100644 --- a/.github/workflows/test-agent-target.yml +++ b/.github/workflows/test-agent-target.yml @@ -63,7 +63,5 @@ jobs: context: "test-agent-target" secrets: inherit permissions: - # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/.github/workflows/test-agent-windows.yml b/.github/workflows/test-agent-windows.yml index dba54b173a1fe..5a1fde593115c 100644 --- a/.github/workflows/test-agent-windows.yml +++ b/.github/workflows/test-agent-windows.yml @@ -51,7 +51,5 @@ jobs: context: "test-agent" secrets: inherit permissions: - # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/.github/workflows/test-agent.yml b/.github/workflows/test-agent.yml index e04504ab428e8..c70461895d14d 100644 --- a/.github/workflows/test-agent.yml +++ b/.github/workflows/test-agent.yml @@ -51,7 +51,5 @@ jobs: context: "test-agent" secrets: inherit permissions: - # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/.github/workflows/test-fips-e2e.yml b/.github/workflows/test-fips-e2e.yml index 23ae8619da71f..6b97beb9ffefd 100644 --- a/.github/workflows/test-fips-e2e.yml +++ b/.github/workflows/test-fips-e2e.yml @@ -43,7 +43,7 @@ jobs: DD_TRACE_ANALYTICS_ENABLED: "true" permissions: - # needed for dd-sts and codecov in test-target.yml, allows the action to get a JWT signed by Github + # needed for dd-sts id-token: write # needed for compute-matrix in test-target.yml contents: read @@ -121,16 +121,6 @@ jobs: name: "test-results-${{ inputs.target || 'tls' }}" path: "${{ env.TEST_RESULTS_BASE_DIR }}" - - name: Upload coverage data - if: > - !github.event.repository.private && - always() - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - files: "${{ inputs.target || 'tls' }}/coverage.xml" - flags: "${{ inputs.target || 'tls' }}" - - name: Upload coverage to Datadog if: > !github.event.repository.private && diff --git a/.github/workflows/weekly-latest-windows.yml b/.github/workflows/weekly-latest-windows.yml index 435bc089c4a6c..032e60eca0019 100644 --- a/.github/workflows/weekly-latest-windows.yml +++ b/.github/workflows/weekly-latest-windows.yml @@ -18,7 +18,5 @@ jobs: context: "weekly-latest" secrets: inherit permissions: - # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/.github/workflows/weekly-latest.yml b/.github/workflows/weekly-latest.yml index 1fe57dbc0fe3a..5b8b3747a7ada 100644 --- a/.github/workflows/weekly-latest.yml +++ b/.github/workflows/weekly-latest.yml @@ -16,7 +16,5 @@ jobs: context: "weekly-latest" secrets: inherit permissions: - # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/README.md b/README.md index fd081156a41a1..bf828e8b81e7d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ | | | | --- | --- | -| CI/CD | [![CI - Test][1]][2] [![CI - Coverage][17]][18] | +| CI/CD | [![CI - Test][1]][2] | | Docs | [![Docs - Release][19]][20] | | Meta | [![Hatch project][26]][27] [![Linting - Ruff][24]][25] [![Code style - black][21]][22] [![Typing - Mypy][28]][29] [![License - BSD-3-Clause][30]][31] | @@ -41,8 +41,6 @@ For more information on integrations, please reference our [documentation][11] a [13]: https://docs.datadoghq.com/help/ [15]: https://github.com/DataDog/integrations-core/blob/6.2.1/requirements-integration-core.txt [16]: https://github.com/DataDog/integrations-core/blob/ea2dfbf1e8859333af4c8db50553eb72a3b466f9/requirements-agent-release.txt -[17]: https://codecov.io/github/DataDog/integrations-core/coverage.svg?branch=master -[18]: https://codecov.io/github/DataDog/integrations-core?branch=master [19]: https://github.com/DataDog/integrations-core/workflows/docs/badge.svg [20]: https://github.com/DataDog/integrations-core/actions?workflow=docs [21]: https://img.shields.io/badge/code%20style-black-000000.svg diff --git a/code-coverage.datadog.yml b/code-coverage.datadog.yml index cb9fe798dbc94..0de01c2ce72c4 100644 --- a/code-coverage.datadog.yml +++ b/code-coverage.datadog.yml @@ -1,2 +1,717 @@ schema-version: v1 carryforward: true +gates: +- type: total_coverage_percentage + config: + threshold: 75 + services: + - '*' + - '!couchbase' + - '!datadog_checks_base' + - '!datadog_checks_dev' + - '!fluentd' + - '!foundationdb' + - '!gearmand' + - '!glusterfs' + - '!hdfs_namenode' + - '!ibm_i' + - '!istio' + - '!kafka_actions' + - '!mapr' + - '!mapreduce' + - '!openstack' + - '!silverstripe_cms' + - '!snmp' + - '!varnish' +# New custom gates for services whose thresholds had to be lowered when migrating to Datadog Code Coverage. +- type: total_coverage_percentage + config: + threshold: 74 + services: + - couchbase +- type: total_coverage_percentage + config: + threshold: 73 + services: + - foundationdb + - varnish +- type: total_coverage_percentage + config: + threshold: 72 + services: + - datadog_checks_base +- type: total_coverage_percentage + config: + threshold: 71 + services: + - fluentd + - hdfs_namenode + - mapr +- type: total_coverage_percentage + config: + threshold: 70 + services: + - glusterfs + - istio +- type: total_coverage_percentage + config: + threshold: 69 + services: + - mapreduce +- type: total_coverage_percentage + config: + threshold: 67 + services: + - ibm_i + - silverstripe_cms +- type: total_coverage_percentage + config: + threshold: 61 + services: + - gearmand +- type: total_coverage_percentage + config: + threshold: 58 + services: + - kafka_actions +- type: total_coverage_percentage + config: + threshold: 52 + services: + - datadog_checks_dev +# Existing custom gates. +- type: total_coverage_percentage + config: + threshold: 45 + services: + - openstack +- type: total_coverage_percentage + config: + threshold: 30 + services: + - snmp +services: +- id: active_directory + paths: + - active_directory/datadog_checks/active_directory/ +- id: activemq_xml + paths: + - activemq_xml/datadog_checks/activemq_xml/ +- id: aerospike + paths: + - aerospike/datadog_checks/aerospike/ +- id: airflow + paths: + - airflow/datadog_checks/airflow/ +- id: amazon_msk + paths: + - amazon_msk/datadog_checks/amazon_msk/ +- id: ambari + paths: + - ambari/datadog_checks/ambari/ +- id: apache + paths: + - apache/datadog_checks/apache/ +- id: appgate_sdp + paths: + - appgate_sdp/datadog_checks/appgate_sdp/ +- id: arangodb + paths: + - arangodb/datadog_checks/arangodb/ +- id: argo_rollouts + paths: + - argo_rollouts/datadog_checks/argo_rollouts/ +- id: argo_workflows + paths: + - argo_workflows/datadog_checks/argo_workflows/ +- id: argocd + paths: + - argocd/datadog_checks/argocd/ +- id: aspdotnet + paths: + - aspdotnet/datadog_checks/aspdotnet/ +- id: avi_vantage + paths: + - avi_vantage/datadog_checks/avi_vantage/ +- id: aws_neuron + paths: + - aws_neuron/datadog_checks/aws_neuron/ +- id: azure_iot_edge + paths: + - azure_iot_edge/datadog_checks/azure_iot_edge/ +- id: bentoml + paths: + - bentoml/datadog_checks/bentoml/ +- id: boundary + paths: + - boundary/datadog_checks/boundary/ +- id: btrfs + paths: + - btrfs/datadog_checks/btrfs/ +- id: cacti + paths: + - cacti/datadog_checks/cacti/ +- id: calico + paths: + - calico/datadog_checks/calico/ +- id: cassandra_nodetool + paths: + - cassandra_nodetool/datadog_checks/cassandra_nodetool/ +- id: celery + paths: + - celery/datadog_checks/celery/ +- id: ceph + paths: + - ceph/datadog_checks/ceph/ +- id: cert_manager + paths: + - cert_manager/datadog_checks/cert_manager/ +- id: checkpoint_harmony_endpoint + paths: + - checkpoint_harmony_endpoint/datadog_checks/checkpoint_harmony_endpoint/ +- id: cilium + paths: + - cilium/datadog_checks/cilium/ +- id: cisco_aci + paths: + - cisco_aci/datadog_checks/cisco_aci/ +- id: citrix_hypervisor + paths: + - citrix_hypervisor/datadog_checks/citrix_hypervisor/ +- id: clickhouse + paths: + - clickhouse/datadog_checks/clickhouse/ +- id: cloud_foundry_api + paths: + - cloud_foundry_api/datadog_checks/cloud_foundry_api/ +- id: cloudera + paths: + - cloudera/datadog_checks/cloudera/ +- id: cockroachdb + paths: + - cockroachdb/datadog_checks/cockroachdb/ +- id: consul + paths: + - consul/datadog_checks/consul/ +- id: control_m + paths: + - control_m/datadog_checks/control_m/ +- id: coredns + paths: + - coredns/datadog_checks/coredns/ +- id: couch + paths: + - couch/datadog_checks/couch/ +- id: couchbase + paths: + - couchbase/datadog_checks/couchbase/ +- id: crio + paths: + - crio/datadog_checks/crio/ +- id: datadog_checks_base + paths: + - datadog_checks_base/datadog_checks/base/ +- id: datadog_checks_dev + paths: + - datadog_checks_dev/datadog_checks/dev/ +- id: datadog_checks_downloader + paths: + - datadog_checks_downloader/datadog_checks/downloader/ +- id: datadog_cluster_agent + paths: + - datadog_cluster_agent/datadog_checks/datadog_cluster_agent/ +- id: dcgm + paths: + - dcgm/datadog_checks/dcgm/ +- id: ddev + paths: + - ddev/src/ddev/ +- id: directory + paths: + - directory/datadog_checks/directory/ +- id: disk + paths: + - disk/datadog_checks/disk/ +- id: dns_check + paths: + - dns_check/datadog_checks/dns_check/ +- id: do_query_actions + paths: + - do_query_actions/datadog_checks/do_query_actions/ +- id: dotnetclr + paths: + - dotnetclr/datadog_checks/dotnetclr/ +- id: druid + paths: + - druid/datadog_checks/druid/ +- id: duckdb + paths: + - duckdb/datadog_checks/duckdb/ +- id: ecs_fargate + paths: + - ecs_fargate/datadog_checks/ecs_fargate/ +- id: eks_fargate + paths: + - eks_fargate/datadog_checks/eks_fargate/ +- id: elastic + paths: + - elastic/datadog_checks/elastic/ +- id: envoy + paths: + - envoy/datadog_checks/envoy/ +- id: esxi + paths: + - esxi/datadog_checks/esxi/ +- id: etcd + paths: + - etcd/datadog_checks/etcd/ +- id: exchange_server + paths: + - exchange_server/datadog_checks/exchange_server/ +- id: external_dns + paths: + - external_dns/datadog_checks/external_dns/ +- id: falco + paths: + - falco/datadog_checks/falco/ +- id: fluentd + paths: + - fluentd/datadog_checks/fluentd/ +- id: fluxcd + paths: + - fluxcd/datadog_checks/fluxcd/ +- id: fly_io + paths: + - fly_io/datadog_checks/fly_io/ +- id: foundationdb + paths: + - foundationdb/datadog_checks/foundationdb/ +- id: gearmand + paths: + - gearmand/datadog_checks/gearmand/ +- id: gitlab + paths: + - gitlab/datadog_checks/gitlab/ +- id: gitlab_runner + paths: + - gitlab_runner/datadog_checks/gitlab_runner/ +- id: glusterfs + paths: + - glusterfs/datadog_checks/glusterfs/ +- id: go_expvar + paths: + - go_expvar/datadog_checks/go_expvar/ +- id: guarddog + paths: + - guarddog/datadog_checks/guarddog/ +- id: gunicorn + paths: + - gunicorn/datadog_checks/gunicorn/ +- id: haproxy + paths: + - haproxy/datadog_checks/haproxy/ +- id: harbor + paths: + - harbor/datadog_checks/harbor/ +- id: hazelcast + paths: + - hazelcast/datadog_checks/hazelcast/ +- id: hdfs_datanode + paths: + - hdfs_datanode/datadog_checks/hdfs_datanode/ +- id: hdfs_namenode + paths: + - hdfs_namenode/datadog_checks/hdfs_namenode/ +- id: http_check + paths: + - http_check/datadog_checks/http_check/ +- id: hugging_face_tgi + paths: + - hugging_face_tgi/datadog_checks/hugging_face_tgi/ +- id: ibm_ace + paths: + - ibm_ace/datadog_checks/ibm_ace/ +- id: ibm_db2 + paths: + - ibm_db2/datadog_checks/ibm_db2/ +- id: ibm_i + paths: + - ibm_i/datadog_checks/ibm_i/ +- id: ibm_mq + paths: + - ibm_mq/datadog_checks/ibm_mq/ +- id: ibm_spectrum_lsf + paths: + - ibm_spectrum_lsf/datadog_checks/ibm_spectrum_lsf/ +- id: ibm_was + paths: + - ibm_was/datadog_checks/ibm_was/ +- id: iis + paths: + - iis/datadog_checks/iis/ +- id: impala + paths: + - impala/datadog_checks/impala/ +- id: infiniband + paths: + - infiniband/datadog_checks/infiniband/ +- id: istio + paths: + - istio/datadog_checks/istio/ +- id: kafka_actions + paths: + - kafka_actions/datadog_checks/kafka_actions/ +- id: kafka_consumer + paths: + - kafka_consumer/datadog_checks/kafka_consumer/ +- id: karpenter + paths: + - karpenter/datadog_checks/karpenter/ +- id: keda + paths: + - keda/datadog_checks/keda/ +- id: kong + paths: + - kong/datadog_checks/kong/ +- id: krakend + paths: + - krakend/datadog_checks/krakend/ +- id: kube_apiserver_metrics + paths: + - kube_apiserver_metrics/datadog_checks/kube_apiserver_metrics/ +- id: kube_controller_manager + paths: + - kube_controller_manager/datadog_checks/kube_controller_manager/ +- id: kube_dns + paths: + - kube_dns/datadog_checks/kube_dns/ +- id: kube_metrics_server + paths: + - kube_metrics_server/datadog_checks/kube_metrics_server/ +- id: kube_proxy + paths: + - kube_proxy/datadog_checks/kube_proxy/ +- id: kube_scheduler + paths: + - kube_scheduler/datadog_checks/kube_scheduler/ +- id: kubeflow + paths: + - kubeflow/datadog_checks/kubeflow/ +- id: kubelet + paths: + - kubelet/datadog_checks/kubelet/ +- id: kubernetes_cluster_autoscaler + paths: + - kubernetes_cluster_autoscaler/datadog_checks/kubernetes_cluster_autoscaler/ +- id: kubernetes_state + paths: + - kubernetes_state/datadog_checks/kubernetes_state/ +- id: kubevirt_api + paths: + - kubevirt_api/datadog_checks/kubevirt_api/ +- id: kubevirt_controller + paths: + - kubevirt_controller/datadog_checks/kubevirt_controller/ +- id: kubevirt_handler + paths: + - kubevirt_handler/datadog_checks/kubevirt_handler/ +- id: kuma + paths: + - kuma/datadog_checks/kuma/ +- id: kyototycoon + paths: + - kyototycoon/datadog_checks/kyototycoon/ +- id: kyverno + paths: + - kyverno/datadog_checks/kyverno/ +- id: lighttpd + paths: + - lighttpd/datadog_checks/lighttpd/ +- id: linkerd + paths: + - linkerd/datadog_checks/linkerd/ +- id: linux_proc_extras + paths: + - linux_proc_extras/datadog_checks/linux_proc_extras/ +- id: litellm + paths: + - litellm/datadog_checks/litellm/ +- id: lparstats + paths: + - lparstats/datadog_checks/lparstats/ +- id: lustre + paths: + - lustre/datadog_checks/lustre/ +- id: mac_audit_logs + paths: + - mac_audit_logs/datadog_checks/mac_audit_logs/ +- id: mapr + paths: + - mapr/datadog_checks/mapr/ +- id: mapreduce + paths: + - mapreduce/datadog_checks/mapreduce/ +- id: marathon + paths: + - marathon/datadog_checks/marathon/ +- id: marklogic + paths: + - marklogic/datadog_checks/marklogic/ +- id: mcache + paths: + - mcache/datadog_checks/mcache/ +- id: mesos_master + paths: + - mesos_master/datadog_checks/mesos_master/ +- id: mesos_slave + paths: + - mesos_slave/datadog_checks/mesos_slave/ +- id: milvus + paths: + - milvus/datadog_checks/milvus/ +- id: mongo + paths: + - mongo/datadog_checks/mongo/ +- id: mysql + paths: + - mysql/datadog_checks/mysql/ +- id: n8n + paths: + - n8n/datadog_checks/n8n/ +- id: nagios + paths: + - nagios/datadog_checks/nagios/ +- id: network + paths: + - network/datadog_checks/network/ +- id: nfsstat + paths: + - nfsstat/datadog_checks/nfsstat/ +- id: nginx + paths: + - nginx/datadog_checks/nginx/ +- id: nginx_ingress_controller + paths: + - nginx_ingress_controller/datadog_checks/nginx_ingress_controller/ +- id: nifi + paths: + - nifi/datadog_checks/nifi/ +- id: nutanix + paths: + - nutanix/datadog_checks/nutanix/ +- id: nvidia_nim + paths: + - nvidia_nim/datadog_checks/nvidia_nim/ +- id: nvidia_triton + paths: + - nvidia_triton/datadog_checks/nvidia_triton/ +- id: octopus_deploy + paths: + - octopus_deploy/datadog_checks/octopus_deploy/ +- id: openldap + paths: + - openldap/datadog_checks/openldap/ +- id: openmetrics + paths: + - openmetrics/datadog_checks/openmetrics/ +- id: openstack + paths: + - openstack/datadog_checks/openstack/ +- id: openstack_controller + paths: + - openstack_controller/datadog_checks/openstack_controller/ +- id: pdh_check + paths: + - pdh_check/datadog_checks/pdh_check/ +- id: pgbouncer + paths: + - pgbouncer/datadog_checks/pgbouncer/ +- id: php_fpm + paths: + - php_fpm/datadog_checks/php_fpm/ +- id: postfix + paths: + - postfix/datadog_checks/postfix/ +- id: postgres + paths: + - postgres/datadog_checks/postgres/ +- id: powerdns_recursor + paths: + - powerdns_recursor/datadog_checks/powerdns_recursor/ +- id: prefect + paths: + - prefect/datadog_checks/prefect/ +- id: process + paths: + - process/datadog_checks/process/ +- id: prometheus + paths: + - prometheus/datadog_checks/prometheus/ +- id: proxmox + paths: + - proxmox/datadog_checks/proxmox/ +- id: proxysql + paths: + - proxysql/datadog_checks/proxysql/ +- id: pulsar + paths: + - pulsar/datadog_checks/pulsar/ +- id: quarkus + paths: + - quarkus/datadog_checks/quarkus/ +- id: rabbitmq + paths: + - rabbitmq/datadog_checks/rabbitmq/ +- id: ray + paths: + - ray/datadog_checks/ray/ +- id: redisdb + paths: + - redisdb/datadog_checks/redisdb/ +- id: rethinkdb + paths: + - rethinkdb/datadog_checks/rethinkdb/ +- id: riak + paths: + - riak/datadog_checks/riak/ +- id: riakcs + paths: + - riakcs/datadog_checks/riakcs/ +- id: sap_hana + paths: + - sap_hana/datadog_checks/sap_hana/ +- id: scylla + paths: + - scylla/datadog_checks/scylla/ +- id: silk + paths: + - silk/datadog_checks/silk/ +- id: silverstripe_cms + paths: + - silverstripe_cms/datadog_checks/silverstripe_cms/ +- id: singlestore + paths: + - singlestore/datadog_checks/singlestore/ +- id: slurm + paths: + - slurm/datadog_checks/slurm/ +- id: snmp + paths: + - snmp/datadog_checks/snmp/ +- id: sonarqube + paths: + - sonarqube/datadog_checks/sonarqube/ +- id: sonatype_nexus + paths: + - sonatype_nexus/datadog_checks/sonatype_nexus/ +- id: spark + paths: + - spark/datadog_checks/spark/ +- id: sqlserver + paths: + - sqlserver/datadog_checks/sqlserver/ +- id: squid + paths: + - squid/datadog_checks/squid/ +- id: ssh_check + paths: + - ssh_check/datadog_checks/ssh_check/ +- id: statsd + paths: + - statsd/datadog_checks/statsd/ +- id: strimzi + paths: + - strimzi/datadog_checks/strimzi/ +- id: supabase + paths: + - supabase/datadog_checks/supabase/ +- id: supervisord + paths: + - supervisord/datadog_checks/supervisord/ +- id: system_core + paths: + - system_core/datadog_checks/system_core/ +- id: system_swap + paths: + - system_swap/datadog_checks/system_swap/ +- id: tcp_check + paths: + - tcp_check/datadog_checks/tcp_check/ +- id: teamcity + paths: + - teamcity/datadog_checks/teamcity/ +- id: tekton + paths: + - tekton/datadog_checks/tekton/ +- id: teleport + paths: + - teleport/datadog_checks/teleport/ +- id: temporal + paths: + - temporal/datadog_checks/temporal/ +- id: teradata + paths: + - teradata/datadog_checks/teradata/ +- id: tibco_ems + paths: + - tibco_ems/datadog_checks/tibco_ems/ +- id: tls + paths: + - tls/datadog_checks/tls/ +- id: torchserve + paths: + - torchserve/datadog_checks/torchserve/ +- id: traefik_mesh + paths: + - traefik_mesh/datadog_checks/traefik_mesh/ +- id: traffic_server + paths: + - traffic_server/datadog_checks/traffic_server/ +- id: twemproxy + paths: + - twemproxy/datadog_checks/twemproxy/ +- id: twistlock + paths: + - twistlock/datadog_checks/twistlock/ +- id: varnish + paths: + - varnish/datadog_checks/varnish/ +- id: vault + paths: + - vault/datadog_checks/vault/ +- id: velero + paths: + - velero/datadog_checks/velero/ +- id: vertica + paths: + - vertica/datadog_checks/vertica/ +- id: vllm + paths: + - vllm/datadog_checks/vllm/ +- id: voltdb + paths: + - voltdb/datadog_checks/voltdb/ +- id: vsphere + paths: + - vsphere/datadog_checks/vsphere/ +- id: weaviate + paths: + - weaviate/datadog_checks/weaviate/ +- id: win32_event_log + paths: + - win32_event_log/datadog_checks/win32_event_log/ +- id: windows_performance_counters + paths: + - windows_performance_counters/datadog_checks/windows_performance_counters/ +- id: windows_service + paths: + - windows_service/datadog_checks/windows_service/ +- id: wmi_check + paths: + - wmi_check/datadog_checks/wmi_check/ +- id: yarn + paths: + - yarn/datadog_checks/yarn/ +- id: zk + paths: + - zk/datadog_checks/zk/ diff --git a/ddev/changelog.d/23360.changed b/ddev/changelog.d/23360.changed new file mode 100644 index 0000000000000..2769d0165dec4 --- /dev/null +++ b/ddev/changelog.d/23360.changed @@ -0,0 +1 @@ +Migrated ``ddev validate ci`` from Codecov to Datadog Code Coverage. diff --git a/ddev/src/ddev/cli/validate/all/orchestrator.py b/ddev/src/ddev/cli/validate/all/orchestrator.py index 3ed7b21439293..2a5fa3b7a3da7 100644 --- a/ddev/src/ddev/cli/validate/all/orchestrator.py +++ b/ddev/src/ddev/cli/validate/all/orchestrator.py @@ -42,7 +42,7 @@ class ValidationConfig: description="Verify check versions match the Agent requirements file", ), "ci": ValidationConfig( - description="Validate CI configuration and Codecov settings", + description="Validate CI configuration and code coverage settings", repo_wide=True, fix_flag="--sync", ), diff --git a/ddev/src/ddev/cli/validate/ci.py b/ddev/src/ddev/cli/validate/ci.py index d968b24d87023..ee50af3cc6800 100644 --- a/ddev/src/ddev/cli/validate/ci.py +++ b/ddev/src/ddev/cli/validate/ci.py @@ -3,6 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from __future__ import annotations +from collections import Counter from typing import TYPE_CHECKING, Any import click @@ -10,33 +11,20 @@ if TYPE_CHECKING: from ddev.cli.application import Application +DEFAULT_COVERAGE_THRESHOLD = 75 -def read_file(file, encoding='utf-8'): - # type: (str, str) -> str - with open(file, 'r', encoding=encoding) as f: - return f.read() - -def write_file(file, contents, encoding='utf-8'): - with open(file, 'w', encoding=encoding) as f: - f.write(contents) - - -def code_coverage_enabled(check_name, app): +def code_coverage_enabled(check_name: str, app: Application) -> bool: if check_name in ('datadog_checks_base', 'datadog_checks_dev', 'datadog_checks_downloader', 'ddev'): return True return app.repo.integrations.get(check_name).is_agent_check -def get_coverage_sources(check_name, app): +def get_coverage_sources(check_name: str, app: Application) -> list[str]: package_path = app.repo.integrations.get(check_name).package_directory package_dir = package_path.relative_to(app.repo.path) - return sorted([str(package_dir.as_posix()), f'{check_name}/tests']) - - -def sort_projects(projects): - return sorted(projects.items(), key=lambda item: (item[0] != 'default', item[0])) + return [f'{package_dir.as_posix()}/'] @click.command() @@ -46,9 +34,7 @@ def ci(app: Application, sync: bool): """Validate CI infrastructure configuration.""" import hashlib import json - import os import re - from collections import defaultdict import yaml @@ -237,184 +223,168 @@ def ci(app: Application, sync: bool): app.abort('CI configuration is not in sync, try again with the `--sync` flag') validation_tracker = app.create_validation_tracker('CI configuration validation') - error_message = '' - warning_message = '' repo_choice = app.repo.name valid_repos = ['core', 'marketplace', 'extras', 'internal'] if repo_choice not in valid_repos: app.abort(f'Unknown repository `{repo_choice}`') - # marketplace does not have a .codecov.yml file - if app.repo.name == 'marketplace': + if is_marketplace: + validation_tracker.success() + validation_tracker.display() return - testable_checks = {integration.name for integration in app.repo.integrations.iter_testable('all')} + _validate_code_coverage(app, sync, validation_tracker, repo_choice) - cached_display_names: defaultdict[str, str] = defaultdict(str) - codecov_config_relative_path = '.codecov.yml' +def _validate_code_coverage( + app: Application, + sync: bool, + validation_tracker: Any, + repo_choice: str, +) -> None: + import yaml + + config_filename = 'code-coverage.datadog.yml' + config_path = app.repo.path / config_filename - path_split = str(codecov_config_relative_path).split('/') - codecov_config_path = os.path.join(app.repo.path, *path_split) - if not os.path.isfile(codecov_config_path): - error_message = 'Unable to find the Codecov config file' - validation_tracker.error((repo_choice,), message=error_message) + if not config_path.is_file(): + validation_tracker.error( + (repo_choice,), message=f'Unable to find the code coverage config file: {config_filename}' + ) validation_tracker.display() app.abort() - codecov_config = yaml.safe_load(read_file(codecov_config_path)) - projects = codecov_config.setdefault('coverage', {}).setdefault('status', {}).setdefault('project', {}) - defined_checks = set() - success = True - fixed = False - - for project, data in list(projects.items()): - if project == 'default': - continue + config = yaml.safe_load(config_path.read_text()) + if config is None: + config = {} - project_flags = data.get('flags', []) - if len(project_flags) != 1: - success = False - error_message += f'Project `{project}` must have exactly one flag\n' - continue + testable_checks = {integration.name for integration in app.repo.integrations.iter_testable('all')} + excluded_jobs = { + name for name, conf in app.repo.config.get('/overrides/ci', {}).items() if conf.get('exclude', False) + } - check_name = project_flags[0] + expected_checks = set() + for check in testable_checks: + if check not in excluded_jobs and code_coverage_enabled(check, app): + expected_checks.add(check) - if check_name in defined_checks: - success = False - error_message += f'Check `{check_name}` is defined as a flag in more than one project\n' - continue + existing_services = config.get('services') or [] + existing_service_id_list = [s['id'] for s in existing_services if 'id' in s] + existing_service_ids = set(existing_service_id_list) - defined_checks.add(check_name) - # Project names cannot contain spaces, see: - # https://github.com/DataDog/integrations-core/pull/6760#issuecomment-634976885 - if check_name in cached_display_names: - display_name = cached_display_names[check_name].replace(' ', '_') - else: - try: - integration = app.repo.integrations.get(check_name) - except OSError as e: - if str(e).startswith('Integration does not exist: '): - continue + success = True + fixed = False + error_message = '' - raise + duplicate_services = sorted( + service_id for service_id, count in Counter(existing_service_id_list).items() if count > 1 + ) + if duplicate_services: + num_duplicate = len(duplicate_services) + service_label = 'service IDs' if num_duplicate > 1 else 'service ID' + duplicate_service_names = ', '.join(duplicate_services) + message = f'Code coverage config has {num_duplicate} duplicate {service_label}: {duplicate_service_names}\n' - display_name = integration.display_name - display_name = display_name.replace(' ', '_') - cached_display_names[check_name] = display_name + if sync: + fixed = True + deduplicated_services = [] + seen_service_ids = set() + for service in existing_services: + service_id = service.get('id', '') + if service_id in seen_service_ids: + app.display_success(f'Removed duplicate service `{service_id}`\n') + continue - if project != display_name: - message = f'Project `{project}` should be called `{display_name}`\n' + seen_service_ids.add(service_id) + deduplicated_services.append(service) - if sync: - fixed = True - warning_message += message - if display_name not in projects: - projects[display_name] = data - del projects[project] - app.display_success(f'Renamed project to `{display_name}`\n') - else: - success = False - error_message += message + existing_services = deduplicated_services + else: + success = False + error_message += message - # This works because we ensure there is a 1 to 1 correspondence between projects and checks (flags) - excluded_jobs = { - name for name, config in app.repo.config.get('/overrides/ci', {}).items() if config.get('exclude', False) - } - missing_projects = testable_checks - set(defined_checks) - excluded_jobs - - not_agent_checks = set() - for check in set(missing_projects): - if not code_coverage_enabled(check, app): - not_agent_checks.add(check) - missing_projects.discard(check) - - if missing_projects: - num_missing_projects = len(missing_projects) - message = ( - f"Codecov config has {num_missing_projects} missing project{'s' if num_missing_projects > 1 else ''}\n" - ) + stale_services = sorted(existing_service_ids - expected_checks) + if stale_services: + num_stale = len(stale_services) + service_label = 'services' if num_stale > 1 else 'service' + stale_service_names = ', '.join(stale_services) + message = f'Code coverage config has {num_stale} stale {service_label}: {stale_service_names}\n' if sync: fixed = True - warning_message += message - - for missing_check in sorted(missing_projects): - display_name = app.repo.integrations.get(missing_check).display_name - display_name = display_name.replace(' ', '_') - projects[display_name] = {'target': 75, 'flags': [missing_check]} - app.display_success(f'Added project `{display_name}`\n') + existing_services = [s for s in existing_services if s.get('id', '') not in stale_services] + for service_id in stale_services: + app.display_success(f'Removed stale service `{service_id}`\n') else: success = False error_message += message - flags = codecov_config.setdefault('flags', {}) - defined_checks = set() - - for flag, data in list(flags.items()): - defined_checks.add(flag) - - expected_coverage_paths = get_coverage_sources(flag, app) - - configured_coverage_paths = data.get('paths', []) - if configured_coverage_paths != expected_coverage_paths: - message = f'Flag `{flag}` has incorrect coverage source paths\n' - - if sync: - fixed = True - warning_message += message - data['paths'] = expected_coverage_paths - app.display_success(f'Configured coverage paths for flag `{flag}`\n') - else: - success = False - error_message += message - - if not data.get('carryforward'): - message = f'Flag `{flag}` must have carryforward set to true\n' + # Validate existing services have correct paths + for service in existing_services: + service_id = service.get('id', '') + if service_id not in expected_checks: + continue + expected_paths = get_coverage_sources(service_id, app) + configured_paths = service.get('paths', []) + if sorted(configured_paths) != sorted(expected_paths): + message = f'Service `{service_id}` has incorrect coverage source paths\n' if sync: fixed = True - warning_message += message - data['carryforward'] = True - app.display_success(f'Enabled the carryforward feature for flag `{flag}`\n') + service['paths'] = expected_paths + app.display_success(f'Configured coverage paths for service `{service_id}`\n') else: success = False error_message += message - missing_flags = testable_checks - set(defined_checks) - excluded_jobs - for check in set(missing_flags): - if check in not_agent_checks or not code_coverage_enabled(check, app): - missing_flags.discard(check) - - if missing_flags: - num_missing_flags = len(missing_flags) - message = f"Codecov config has {num_missing_flags} missing flag{'s' if num_missing_flags > 1 else ''}\n" + missing_services = sorted(expected_checks - existing_service_ids) + if missing_services: + num_missing = len(missing_services) + message = f"Code coverage config has {num_missing} missing service{'s' if num_missing > 1 else ''}\n" if sync: fixed = True - warning_message += message + for check_name in missing_services: + existing_services.append( + { + 'id': check_name, + 'paths': get_coverage_sources(check_name, app), + } + ) + app.display_success(f'Added service `{check_name}`\n') + else: + success = False + error_message += message - for missing_check in sorted(missing_flags): - flags[missing_check] = {'carryforward': True, 'paths': get_coverage_sources(missing_check, app)} - app.display_success(f'Added flag `{missing_check}`\n') + gates = config.get('gates') or [] + if not gates: + message = 'Code coverage config has no coverage gates\n' + if sync: + fixed = True + gates.append( + { + 'type': 'total_coverage_percentage', + 'config': {'threshold': DEFAULT_COVERAGE_THRESHOLD}, + } + ) + config['gates'] = gates + app.display_success(f'Added default coverage gate with {DEFAULT_COVERAGE_THRESHOLD}% threshold\n') else: success = False error_message += message if not success: - message = 'Try running `ddev validate ci --sync`\n' - app.display_info(message) - validation_tracker.error((codecov_config_path,), message=error_message) - + app.display_info('Try running `ddev validate ci --sync`\n') + validation_tracker.error((str(config_path),), message=error_message) validation_tracker.display() app.abort() elif fixed: - codecov_config['coverage']['status']['project'] = dict(sort_projects(projects)) - codecov_config['flags'] = dict(sorted(flags.items())) - output = yaml.safe_dump(codecov_config, default_flow_style=False, sort_keys=False) - write_file(codecov_config_path, output) - app.display_success(f'Successfully fixed {codecov_config_relative_path}') + config['services'] = sorted(existing_services, key=lambda s: s.get('id', '')) + + output = yaml.safe_dump(config, default_flow_style=False, sort_keys=False) + config_path.write_text(output) + app.display_success(f'Successfully fixed {config_filename}') validation_tracker.success() validation_tracker.display() diff --git a/ddev/tests/cli/validate/all/test_github.py b/ddev/tests/cli/validate/all/test_github.py index a32d2f6915dde..b6a1d6df621d2 100644 --- a/ddev/tests/cli/validate/all/test_github.py +++ b/ddev/tests/cli/validate/all/test_github.py @@ -17,7 +17,7 @@ from ddev.cli.validate.all.orchestrator import ValidationConfig, ValidationResult CONFIGS = { - "ci": ValidationConfig(description="Validate CI configuration and Codecov settings", repo_wide=True), + "ci": ValidationConfig(description="Validate CI configuration and code coverage settings", repo_wide=True), "config": ValidationConfig(description="Validate default configuration files against spec.yaml"), "metadata": ValidationConfig(description="Validate metadata.csv metric definitions"), } @@ -135,7 +135,7 @@ def test_format_pr_comment_all_passed(helpers): | Validation | Description | Status | |---|---|---| - | `ci` | Validate CI configuration and Codecov settings | ✅ | + | `ci` | Validate CI configuration and code coverage settings | ✅ | | `config` | Validate default configuration files against spec.yaml | ✅ | """) @@ -161,7 +161,7 @@ def test_format_pr_comment_one_failure_with_target(helpers): | Validation | Description | Status | |---|---|---| - | `ci` | Validate CI configuration and Codecov settings | ✅ | + | `ci` | Validate CI configuration and code coverage settings | ✅ | """) assert format_pr_comment(results, CONFIGS, "changed", list(results)) == expected @@ -267,7 +267,7 @@ def test_format_step_summary_all_passed(helpers): | Validation | Description | Status | |---|---|---| - | `ci` | Validate CI configuration and Codecov settings | ✅ | + | `ci` | Validate CI configuration and code coverage settings | ✅ | | `config` | Validate default configuration files against spec.yaml | ✅ |""") assert format_step_summary(results, CONFIGS, "changed", list(results)) == expected @@ -282,7 +282,7 @@ def test_format_step_summary_with_failures(helpers): | Validation | Description | Status | |---|---|---| - | `ci` | Validate CI configuration and Codecov settings | ✅ | + | `ci` | Validate CI configuration and code coverage settings | ✅ | | `config` | Validate default configuration files against spec.yaml | ❌ | Run `ddev validate all changed --fix` to attempt to auto-fix supported validations.""") diff --git a/ddev/tests/cli/validate/test_ci.py b/ddev/tests/cli/validate/test_ci.py index 4b20c114d538f..c033eb81e1307 100644 --- a/ddev/tests/cli/validate/test_ci.py +++ b/ddev/tests/cli/validate/test_ci.py @@ -1,48 +1,12 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +from pathlib import Path + import pytest import yaml -def test_exactly_one_flag(ddev, repository, helpers): - codecov_yaml = repository.path / '.codecov.yml' - - with codecov_yaml.open(encoding='utf-8') as file: - codecov_yaml_info = yaml.safe_load(file) - - codecov_yaml_info['coverage']['status']['project']['ActiveMQ_XML']['flags'].append('test') - - output = yaml.safe_dump(codecov_yaml_info, default_flow_style=False, sort_keys=False) - with codecov_yaml.open(mode='w', encoding='utf-8') as file: - file.write(output) - - result = ddev("validate", "ci") - - assert result.exit_code == 1, result.output - error = "Project `ActiveMQ_XML` must have exactly one flag" - assert error in helpers.remove_trailing_spaces(result.output) - - -def test_carryforward_flag(ddev, repository, helpers): - codecov_yaml = repository.path / '.codecov.yml' - - with codecov_yaml.open(encoding='utf-8') as file: - temp = yaml.safe_load(file) - - temp['flags']['active_directory']['carryforward'] = False - - output = yaml.safe_dump(temp, default_flow_style=False, sort_keys=False) - with codecov_yaml.open(mode='w', encoding='utf-8') as file: - file.write(output) - - result = ddev("validate", "ci") - - assert result.exit_code == 1, result.output - error = "Flag `active_directory` must have carryforward set to true" - assert error in helpers.remove_trailing_spaces(result.output) - - def test_missing_hatch_toml(ddev, repository, helpers): import os @@ -56,126 +20,127 @@ def test_missing_hatch_toml(ddev, repository, helpers): assert error in helpers.remove_trailing_spaces(result.output) -def test_incorrect_project_name(ddev, repository, helpers): - codecov_yaml = repository.path / '.codecov.yml' - with codecov_yaml.open(encoding='utf-8') as file: - codecov_yaml_info = yaml.safe_load(file) +def test_validate_ci_success(ddev, helpers): + result = ddev('validate', 'ci') + assert result.exit_code == 0, result.output + assert helpers.remove_trailing_spaces(result.output) == helpers.dedent( + """ + CI configuration validation - temp = codecov_yaml_info['coverage']['status']['project']['Active_Directory'] - codecov_yaml_info['coverage']['status']['project']['active directory'] = temp - codecov_yaml_info['coverage']['status']['project'].pop('Active_Directory') + Passed: 1 + """ + ) - output = yaml.safe_dump(codecov_yaml_info, default_flow_style=False, sort_keys=False) - with codecov_yaml.open(mode='w', encoding='utf-8') as file: - file.write(output) - result = ddev("validate", "ci") - assert result.exit_code == 1, result.output - error = "Project `active directory` should be called `Active_Directory`" - assert error in helpers.remove_trailing_spaces(result.output) +def _remove_service(config_path): + with config_path.open(encoding='utf-8') as f: + config = yaml.safe_load(f) + config['services'] = [s for s in config.get('services', []) if s.get('id') != 'apache'] -def test_check_in_multiple_projects(ddev, repository, helpers): - codecov_yaml = repository.path / '.codecov.yml' - with codecov_yaml.open(encoding='utf-8') as file: - codecov_yaml_info = yaml.safe_load(file) + with config_path.open(mode='w', encoding='utf-8') as f: + yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False) - codecov_yaml_info['coverage']['status']['project']['Airflow']['flags'] = ['active_directory'] - output = yaml.safe_dump(codecov_yaml_info, default_flow_style=False, sort_keys=False) - with codecov_yaml.open(mode='w', encoding='utf-8') as file: - file.write(output) +def _set_wrong_paths(config_path): + with config_path.open(encoding='utf-8') as f: + config = yaml.safe_load(f) - result = ddev("validate", "ci") - assert result.exit_code == 1, result.output - error = "Check `active_directory` is defined as a flag in more than one project" - assert error in helpers.remove_trailing_spaces(result.output) + for service in config.get('services', []): + if service.get('id') == 'active_directory': + service['paths'] = ['wrong/path/'] + break + with config_path.open(mode='w', encoding='utf-8') as f: + yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False) -def test_codecov_missing_projects(ddev, repository, helpers): - codecov_yaml = repository.path / '.codecov.yml' - with codecov_yaml.open(encoding='utf-8') as file: - codecov_yaml_info = yaml.safe_load(file) - codecov_yaml_info['coverage']['status']['project'].pop('Apache') +def _add_stale_service(config_path): + with config_path.open(encoding='utf-8') as f: + config = yaml.safe_load(f) - output = yaml.safe_dump(codecov_yaml_info, default_flow_style=False, sort_keys=False) - with codecov_yaml.open(mode='w', encoding='utf-8') as file: - file.write(output) + config.setdefault('services', []).append({'id': 'stale_service', 'paths': ['stale_service/tests/']}) - result = ddev("validate", "ci") - assert result.exit_code == 1, result.output - error = "Codecov config has 1 missing project" - assert error in helpers.remove_trailing_spaces(result.output) + with config_path.open(mode='w', encoding='utf-8') as f: + yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False) -def test_incorrect_coverage_source_path(ddev, repository, helpers): - codecov_yaml = repository.path / '.codecov.yml' - with codecov_yaml.open(encoding='utf-8') as file: - codecov_yaml_info = yaml.safe_load(file) +def _add_duplicate_service(config_path: Path) -> None: + with config_path.open(encoding='utf-8') as f: + config = yaml.safe_load(f) - codecov_yaml_info['flags']['active_directory']['paths'] = [ - 'active_directory/datadog_checks/test', - 'active_directory/tests', - ] + duplicate_service = next(service for service in config['services'] if service.get('id') == 'active_directory') + config['services'].append({'id': duplicate_service['id'], 'paths': list(duplicate_service['paths'])}) - output = yaml.safe_dump(codecov_yaml_info, default_flow_style=False, sort_keys=False) - with codecov_yaml.open(mode='w', encoding='utf-8') as file: - file.write(output) + with config_path.open(mode='w', encoding='utf-8') as f: + yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False) - result = ddev("validate", "ci") - assert result.exit_code == 1, result.output - error = "Flag `active_directory` has incorrect coverage source paths" - assert error in helpers.remove_trailing_spaces(result.output) +def _remove_gates(config_path: Path) -> None: + with config_path.open(encoding='utf-8') as f: + config = yaml.safe_load(f) -def test_codecov_missing_flag(ddev, repository, helpers): - codecov_yaml = repository.path / '.codecov.yml' - with codecov_yaml.open(encoding='utf-8') as file: - codecov_yaml_info = yaml.safe_load(file) + config.pop('gates', None) - codecov_yaml_info['flags'].pop('active_directory') + with config_path.open(mode='w', encoding='utf-8') as f: + yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False) - output = yaml.safe_dump(codecov_yaml_info, default_flow_style=False, sort_keys=False) - with codecov_yaml.open(mode='w', encoding='utf-8') as file: - file.write(output) - result = ddev("validate", "ci") - assert result.exit_code == 1, result.output - error = "Codecov config has 1 missing flag" - assert error in helpers.remove_trailing_spaces(result.output) +def _clear_gates(config_path: Path) -> None: + with config_path.open(encoding='utf-8') as f: + config = yaml.safe_load(f) + + config['gates'] = [] + + with config_path.open(mode='w', encoding='utf-8') as f: + yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False) -# TODO We do not have an off the shelf fixture to generate a marketplace repository @pytest.mark.parametrize( - 'repository_name, repository_flag, expected_exit_code, expected_output', + 'corrupt_config, expected_error', [ - pytest.param('core', '-c', 1, 'Unable to find the Codecov config file', id='integrations-core'), + pytest.param(_remove_service, "Code coverage config has 1 missing service", id='missing_services'), + pytest.param( + _set_wrong_paths, + "Service `active_directory` has incorrect coverage source paths", + id='incorrect_paths', + ), + pytest.param( + _add_stale_service, "Code coverage config has 1 stale service: stale_service", id='stale_services' + ), + pytest.param( + _add_duplicate_service, + "Code coverage config has 1 duplicate service ID: active_directory", + id='duplicate_services', + ), + pytest.param(_remove_gates, "Code coverage config has no coverage gates", id='missing_gates'), + pytest.param(_clear_gates, "Code coverage config has no coverage gates", id='empty_gates'), ], ) -def test_codecov_file_missing( - ddev, repository, helpers, config_file, repository_name, repository_flag, expected_exit_code, expected_output -): - config_file.model.repos[repository_name] = str(repository.path) - config_file.save() +def test_code_coverage_config(ddev, repository, helpers, corrupt_config, expected_error): + result = ddev("validate", "ci", "--sync") + assert result.exit_code == 0, result.output - (repository.path / '.codecov.yml').unlink() + config_path = repository.path / 'code-coverage.datadog.yml' + corrupt_config(config_path) - result = ddev(repository_flag, "validate", "ci") - assert result.exit_code == expected_exit_code, result.output - assert expected_output in helpers.remove_trailing_spaces(result.output) + result = ddev("validate", "ci") + assert result.exit_code == 1, f"Expected validation to detect corrupted config: {result.output}" + assert expected_error in helpers.remove_trailing_spaces(result.output) + result = ddev("validate", "ci", "--sync") + assert result.exit_code == 0, f"Expected --sync to fix corrupted config: {result.output}" -def test_validate_ci_success(ddev, helpers): - result = ddev('validate', 'ci') - assert result.exit_code == 0, result.output - assert helpers.remove_trailing_spaces(result.output) == helpers.dedent( - """ - CI configuration validation + result = ddev("validate", "ci") + assert result.exit_code == 0, f"Expected validation to pass after sync: {result.output}" - Passed: 1 - """ - ) + +def test_code_coverage_file_missing(ddev, repository, helpers): + (repository.path / 'code-coverage.datadog.yml').unlink() + + result = ddev("-c", "validate", "ci") + assert result.exit_code == 1, result.output + assert "Unable to find the code coverage config file" in helpers.remove_trailing_spaces(result.output) @pytest.mark.parametrize( diff --git a/docs/developer/.snippets/links.txt b/docs/developer/.snippets/links.txt index 59a86366e995c..48502a9038fb6 100644 --- a/docs/developer/.snippets/links.txt +++ b/docs/developer/.snippets/links.txt @@ -13,7 +13,6 @@ [azp-templates-windows]: https://github.com/DataDog/integrations-core/blob/master/.azure-pipelines/templates/test-single-windows.yml [black-github]: https://github.com/psf/black [click-github]: https://github.com/pallets/click -[codecov-home]: https://codecov.io [config-spec-example-consumer]: https://github.com/DataDog/integrations-core/blob/master/datadog_checks_dev/datadog_checks/dev/tooling/configuration/consumers/example.py [config-spec-model-consumer]: https://github.com/DataDog/integrations-core/blob/master/datadog_checks_dev/datadog_checks/dev/tooling/configuration/consumers/model.py [config-spec-producer]: https://github.com/DataDog/integrations-core/blob/master/datadog_checks_dev/datadog_checks/dev/tooling/configuration/core.py diff --git a/docs/developer/index.md b/docs/developer/index.md index 1461f7774875e..161ac6ca58803 100644 --- a/docs/developer/index.md +++ b/docs/developer/index.md @@ -1,7 +1,6 @@ # Agent Integrations [![CI - Docs](https://github.com/DataDog/integrations-core/workflows/docs/badge.svg)](https://github.com/DataDog/integrations-core/actions?workflow=docs) -[![Coverage status](https://codecov.io/github/DataDog/integrations-core/coverage.svg?branch=master)](https://codecov.io/github/DataDog/integrations-core?branch=master) [![GitHub contributors](https://img.shields.io/github/contributors/DataDog/integrations-core)](https://github.com/DataDog/integrations-core) [![Downloads](https://pepy.tech/badge/datadog-checks-dev)](https://pepy.tech/project/datadog-checks-dev) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/datadog-checks-dev)](https://pypi.org/project/datadog-checks-dev) diff --git a/docs/developer/meta/ci/labels.md b/docs/developer/meta/ci/labels.md index c146c54a66180..ec3c451260956 100644 --- a/docs/developer/meta/ci/labels.md +++ b/docs/developer/meta/ci/labels.md @@ -10,7 +10,7 @@ The labeler is [configured](https://github.com/DataDog/integrations-core/blob/ma | --- | --- | | integration/<NAME> | any directory at the root that actually contains an integration | | documentation | any Markdown, [config specs](../config-specs.md), `manifest.json`, or anything in `/docs/` | -| dev/testing | [GitHub Actions](https://github.com/DataDog/integrations-core/tree/master/.github/workflows) or [Codecov](https://github.com/DataDog/integrations-core/blob/master/.codecov.yml) config | +| dev/testing | [GitHub Actions](https://github.com/DataDog/integrations-core/tree/master/.github/workflows) or [code coverage](https://github.com/DataDog/integrations-core/blob/master/code-coverage.datadog.yml) config | | dev/tooling | [GitLab](https://github.com/DataDog/integrations-core/tree/master/.gitlab) or [GitHub Actions](https://github.com/DataDog/integrations-core/tree/master/.github/workflows) config, or [ddev](../../ddev/about.md#cli) | | dependencies | any change in shipped dependencies | | release | any [base package](../../base/about.md), [dev package](../../ddev/about.md), or integration release | diff --git a/docs/developer/meta/ci/validation.md b/docs/developer/meta/ci/validation.md index 99bc19f770449..cf8ff1b5f5ef8 100644 --- a/docs/developer/meta/ci/validation.md +++ b/docs/developer/meta/ci/validation.md @@ -18,7 +18,7 @@ This validates that each integration version is in sync with the [`requirements- ddev validate ci ``` -This validates that all CI entries for integrations are valid. This includes checking if the integration has the correct [Codecov config](https://github.com/DataDog/integrations-core/blob/master/.codecov.yml), and has a valid [CI entry](testing.md#target-enumeration) if it is testable. +This validates that all CI entries for integrations are valid. This includes checking if the integration has the correct [Datadog Code Coverage config](https://github.com/DataDog/integrations-core/blob/master/code-coverage.datadog.yml), and has a valid [CI entry](testing.md#target-enumeration) if it is testable. !!! tip Run `ddev validate ci --sync` to resolve most errors. From d6365ccc3412c5726d8c972302976a5495267977 Mon Sep 17 00:00:00 2001 From: Juanpe Araque Date: Thu, 28 May 2026 14:20:16 +0100 Subject: [PATCH 16/44] Store ClickHouse advanced-queries metric definitions as JSON (#23829) * Store ClickHouse advanced-queries metric definitions as JSON The four advanced_queries Python modules shipped ~250 KB of redundant dict literals (per-entry 'name' was always '.', and every entry repeated its type). Move that data to compact per-system-table JSON files under datadog_checks/clickhouse/data/ and build the QueryManager-shaped dicts at runtime. The check registers a check_initializations callable so the JSON files are parsed once on the first check run. Module attributes like advanced_queries.SystemMetrics remain available through __getattr__ backed by the same cache, so tests that read those attributes directly keep working. The metric generator emits the new JSON format directly; the three system_*.tpl templates and the four old Python modules are removed. * Add changelog entry * Simplify advanced_queries loader and tighten the generator - Replace the initializer(check) factory with a plain warm_cache callable; the closure never depended on the check instance. - Restore __all__ on the package so the public surface is explicit. - Rename _cache to cache per the AGENTS.md module-name rule. - Mirror Python's default AttributeError format in __getattr__. - Tighten return-type annotations on load, _build_items, and __getattr__. - Raise loudly in generate_queries when a metric type appears with mixed scaled/unscaled entries instead of silently producing a wrong container. - Reclass the changelog from .changed to .fixed: the refactor preserves runtime behaviour byte-for-byte, so a patch bump is the right semver. * Tighten the advanced_queries loader - Guard warm_cache with an explicit "key not in cache" check so the JSON files are read at most once per process, even when __getattr__ populated the cache first. - Wrap load() exceptions as RuntimeError so a missing or malformed data file produces an actionable message in the check-init failure. - Discriminate the two JSON shapes on positive presence of "columns" rather than absence of "items". - Reclass the changelog as .changed: this is a significant internal refactor, not a bug fix, so .fixed was misleading. * Drop system_errors from the generator and widen load() error wrap - Wrap KeyError in load()'s error path so a malformed JSON file (one that parses but is missing expected keys) raises the same context-rich RuntimeError as a missing or invalid file. - Remove SYSTEM_ERRORS_SPEC and generate_system_errors() from the generator. The system_errors data is static; the committed data/system_errors.json is the single source of truth, and the other three queries continue to be generator-driven from ClickHouse source. * Tighten the advanced_queries loader and rename the generator's Template - Comment the load() branch that handles system_errors so the asymmetry is named at the call site instead of requiring readers to audit the JSON files. - Widen load()'s except tuple with TypeError and AttributeError so a malformed JSON shape (e.g. items shipped as a list) still raises the wrapped RuntimeError with the file name rather than leaking the bare underlying exception. - Narrow _build_items's compact parameter type to dict[str, list[str] | dict[str, str]] to mirror the producer annotation in generate_metrics.py. - Rename the generator's Template dataclass to FileTemplate so it doesn't silently shadow string.Template if the stdlib import is ever reintroduced. * Rename module cache to _cache and drop the one-member Templates enum - Rename advanced_queries.cache to _cache so the underscore signals it as module-internal mutable state (PEP 8 module-private). The module's __all__ already advertises only the four System* names. - Rephrase the load() inline comment around the columns shortcut so it names the discriminating shape rather than the system_errors file, which would mislead if a second verbatim file were added later. - Replace the single-member Templates enum in the generator with a TESTS_METRICS_TEMPLATE constant; the enum was a vestige from when it held the three QUERY_* templates that now live in QUERY_SPECS. * Test the advanced_queries loader directly Adds tests/test_advanced_queries.py covering the new loader logic: - module-level __getattr__ resolution + caching - compact format: source/match column shape, sorted items, name derivation including the dotted-key edge case (jemalloc.epoch) - verbatim format: system_errors columns pass through with the boolean: true tag preserved - RuntimeError wrap on every malformed-JSON path the load() except tuple is meant to cover (missing file, invalid JSON, items as list, items as scalar, missing required keys) with the cause chain preserved - warm_cache populates every known name and is idempotent * Move changelog to fixed * Scope the advanced_queries loader to bulk match queries The advanced_queries package is now organised around one named pattern: the SQL-returns-(value, metric_name)-and-dispatches-via-lookup-table shape that SystemEvents, SystemMetrics, and SystemAsynchronousMetrics all share. The compact JSON files exist specifically to compress that pattern. - Rename load() to load_match_query(); _build_items() to _expand_match_items(); NAMES to MATCH_QUERIES; _cache to _match_query_cache. Names now say what the loader does. - Inline SystemErrors as a plain Python literal in __init__.py. Its shape doesn't fit the bulk-match pattern, so the JSON compression has nothing to compress; data/system_errors.json is removed and the verbatim-columns branch in load() goes away with it. - Add a top-level docstring that describes the JSON schema, names the generator that produces the files, and points operators at the hatch run metrics:generate command. - Add clickhouse/AGENTS.md (with a CLAUDE.md @AGENTS.md indirection) giving anyone opening this directory a short orientation note plus the "don't hand-edit the JSON files" warning that JSON has no comment syntax to carry. - Update tests/test_advanced_queries.py for the rename and add coverage that SystemErrors stays out of the match-query cache. Runtime dict shape and metric names are byte-identical to before; verified by diffing the four module-attribute dumps against the pre-refactor master. --- clickhouse/AGENTS.md | 42 + clickhouse/CLAUDE.md | 1 + clickhouse/changelog.d/23829.fixed | 1 + .../clickhouse/advanced_queries/__init__.py | 134 +- .../advanced_queries/system_async_metrics.py | 273 -- .../advanced_queries/system_errors.py | 15 - .../advanced_queries/system_events.py | 3073 ----------------- .../advanced_queries/system_metrics.py | 780 ----- .../datadog_checks/clickhouse/clickhouse.py | 1 + .../clickhouse/data/system_async_metrics.json | 135 + .../clickhouse/data/system_events.json | 1052 ++++++ .../clickhouse/data/system_metrics.json | 441 +++ clickhouse/scripts/generate_metrics.py | 144 +- .../templates/system_async_metrics.tpl | 27 - .../scripts/templates/system_events.tpl | 27 - .../scripts/templates/system_metrics.tpl | 27 - clickhouse/tests/test_advanced_queries.py | 171 + 17 files changed, 2072 insertions(+), 4272 deletions(-) create mode 100644 clickhouse/AGENTS.md create mode 100644 clickhouse/CLAUDE.md create mode 100644 clickhouse/changelog.d/23829.fixed delete mode 100644 clickhouse/datadog_checks/clickhouse/advanced_queries/system_async_metrics.py delete mode 100644 clickhouse/datadog_checks/clickhouse/advanced_queries/system_errors.py delete mode 100644 clickhouse/datadog_checks/clickhouse/advanced_queries/system_events.py delete mode 100644 clickhouse/datadog_checks/clickhouse/advanced_queries/system_metrics.py create mode 100644 clickhouse/datadog_checks/clickhouse/data/system_async_metrics.json create mode 100644 clickhouse/datadog_checks/clickhouse/data/system_events.json create mode 100644 clickhouse/datadog_checks/clickhouse/data/system_metrics.json delete mode 100644 clickhouse/scripts/templates/system_async_metrics.tpl delete mode 100644 clickhouse/scripts/templates/system_events.tpl delete mode 100644 clickhouse/scripts/templates/system_metrics.tpl create mode 100644 clickhouse/tests/test_advanced_queries.py diff --git a/clickhouse/AGENTS.md b/clickhouse/AGENTS.md new file mode 100644 index 0000000000000..334a4b5138124 --- /dev/null +++ b/clickhouse/AGENTS.md @@ -0,0 +1,42 @@ +# ClickHouse integration agent notes + +A small orientation guide. Read this before touching `advanced_queries/` or +`scripts/generate_metrics.py`. + +## Bulk match queries live in JSON, not Python + +Three of the four advanced queries (`SystemEvents`, `SystemMetrics`, +`SystemAsynchronousMetrics`) are *bulk match queries*: one SQL that returns +`(value, metric_name)` rows and dispatches to per-name metric definitions +through a large lookup table (over 1,000 entries for `SystemEvents`). Those +lookup tables ship as compact JSON files under +`datadog_checks/clickhouse/data/system_*.json` and are reassembled into the +`QueryManager` shape at load time. + +Before changing anything in this area, read: + +- `datadog_checks/clickhouse/advanced_queries/__init__.py`: the loader + (`load_match_query`, `_expand_match_items`, `warm_cache`, `__getattr__`) and + the JSON-schema docstring at the top of the file. +- `scripts/generate_metrics.py`: the script that parses ClickHouse's C++ + source files and writes the three JSON files. + +The fourth query, `SystemErrors`, is a plain Python literal in the same +`__init__.py`. Its shape (one metric plus tag columns, no per-row lookup) +doesn't fit the bulk-match pattern, so the JSON compression has nothing to +compress for it. Don't move it into JSON; the dual format was deliberately +removed during the JSON migration. + +## Don't hand-edit the JSON files + +The three `data/system_*.json` files are autogenerated. JSON has no comment +syntax, so there's no "do not edit" header inside the files themselves; the +warning lives here. Any hand-edit is overwritten on the next run of: + +```shell +cd clickhouse && VERSIONS=24.8,25.3,25.8 hatch run metrics:generate +``` + +If you need to add a metric type or a new scale, edit `generate_metrics.py` +(specifically the `DD_VALUE_TYPES` mapping and the `generate_queries` +function). The script then writes the JSON. diff --git a/clickhouse/CLAUDE.md b/clickhouse/CLAUDE.md new file mode 100644 index 0000000000000..43c994c2d3617 --- /dev/null +++ b/clickhouse/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/clickhouse/changelog.d/23829.fixed b/clickhouse/changelog.d/23829.fixed new file mode 100644 index 0000000000000..5b90ecb3d5942 --- /dev/null +++ b/clickhouse/changelog.d/23829.fixed @@ -0,0 +1 @@ +Store advanced-queries metric definitions as JSON loaded on first check run. diff --git a/clickhouse/datadog_checks/clickhouse/advanced_queries/__init__.py b/clickhouse/datadog_checks/clickhouse/advanced_queries/__init__.py index 939110bd80c08..91dfd419a147f 100644 --- a/clickhouse/datadog_checks/clickhouse/advanced_queries/__init__.py +++ b/clickhouse/datadog_checks/clickhouse/advanced_queries/__init__.py @@ -1,10 +1,136 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +"""Advanced ClickHouse query definitions. -from .system_async_metrics import SystemAsynchronousMetrics -from .system_errors import SystemErrors -from .system_events import SystemEvents -from .system_metrics import SystemMetrics +This package exposes four ``QueryManager`` query dicts: + +- ``SystemEvents``, ``SystemMetrics``, and ``SystemAsynchronousMetrics`` are + *bulk match queries*. Each one runs a single SQL that returns + ``(value, metric_name)`` rows, then routes the metric-name column through a + per-name lookup table to emit hundreds of Datadog metrics from one statement. + Their lookup tables are large (over 1,000 entries for ``SystemEvents``), so + the data ships as compact JSON under ``data/system_*.json`` and is + reassembled into the ``QueryManager`` shape at load time. + +- ``SystemErrors`` does not follow the bulk-match pattern. It runs a SQL that + emits one ``errors.raised`` metric tagged by name/code/remote, with no + per-row metric lookup. It lives below as a plain Python literal because the + compression that justifies JSON has nothing to compress for a one-metric, + three-tag query. + +The compact JSON schema for a bulk match query is:: + + { + "name": "", + "query": "", + "value_column": "", + "match_column": "", + "prefix": "", + "items": { + "": ["", ...], # gauge / monotonic_gauge + "temporal_percent": {"": "", ...} # carries scale + } + } + +At load time, ``load_match_query`` synthesises the two-column scaffold +(``[{value_column source}, {match_column match}]``) and expands ``items`` into +the per-entry shape ``QueryManager`` consumes:: + + items[""] = {"name": f"{prefix}.{key}", "type": "" + [, "scale": ""]} + +The three ``data/system_*.json`` files are autogenerated from ClickHouse's C++ +source by ``clickhouse/scripts/generate_metrics.py``. To update them (typically +when supporting a new ClickHouse version), run from the ``clickhouse`` +directory:: + + VERSIONS=24.8,25.3,25.8 hatch run metrics:generate + +Don't edit those JSON files by hand; the generator overwrites them on the next +run. If you need a new metric type or a schema change, edit +``generate_metrics.py`` (the ``generate_queries`` function and the +``QUERY_SPECS`` table). +""" + +from __future__ import annotations + +import json +import os +from typing import Any __all__ = ['SystemAsynchronousMetrics', 'SystemErrors', 'SystemEvents', 'SystemMetrics'] + +DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'data') + +MATCH_QUERIES = { + 'SystemEvents': 'system_events', + 'SystemMetrics': 'system_metrics', + 'SystemAsynchronousMetrics': 'system_async_metrics', +} + +_match_query_cache: dict[str, dict[str, Any]] = {} + +SystemErrors: dict[str, Any] = { + 'name': 'system.errors', + 'query': 'SELECT value, name, code, remote FROM system.errors WHERE value > 0', + 'columns': [ + {'name': 'errors.raised', 'type': 'monotonic_count'}, + {'name': 'error_name', 'type': 'tag'}, + {'name': 'error_code', 'type': 'tag'}, + {'name': 'remote', 'type': 'tag', 'boolean': True}, + ], +} + + +def load_match_query(name: str) -> dict[str, Any]: + """Read ``data/.json`` and reconstitute the QueryManager-shaped dict.""" + try: + with open(os.path.join(DATA_DIR, f'{name}.json'), encoding='utf-8') as f: + spec = json.load(f) + items = _expand_match_items(spec['items'], spec['prefix']) + return { + 'name': spec['name'], + 'query': spec['query'], + 'columns': [ + {'name': spec['value_column'], 'type': 'source'}, + { + 'name': spec['match_column'], + 'type': 'match', + 'source': spec['value_column'], + 'items': items, + }, + ], + } + except (OSError, json.JSONDecodeError, KeyError, TypeError, AttributeError) as exc: + raise RuntimeError(f'failed to load advanced query {name!r}') from exc + + +def _expand_match_items( + compact: dict[str, list[str] | dict[str, str]], prefix: str +) -> dict[str, dict[str, Any]]: + """Expand the compact ``{type: keys | {key: scale}}`` map to the per-entry dict shape.""" + merged: dict[str, dict[str, Any]] = {} + for type_name, group in compact.items(): + if isinstance(group, dict): + for key, scale in group.items(): + merged[key] = {'name': f'{prefix}.{key}', 'type': type_name, 'scale': scale} + else: + for key in group: + merged[key] = {'name': f'{prefix}.{key}', 'type': type_name} + return dict(sorted(merged.items())) + + +def warm_cache() -> None: + """Populate the match-query cache for every known name. Idempotent.""" + for attr, file in MATCH_QUERIES.items(): + if attr not in _match_query_cache: + _match_query_cache[attr] = load_match_query(file) + + +def __getattr__(name: str) -> dict[str, Any]: + if name not in MATCH_QUERIES: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + if name not in _match_query_cache: + _match_query_cache[name] = load_match_query(MATCH_QUERIES[name]) + return _match_query_cache[name] diff --git a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_async_metrics.py b/clickhouse/datadog_checks/clickhouse/advanced_queries/system_async_metrics.py deleted file mode 100644 index c203ad9a785d9..0000000000000 --- a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_async_metrics.py +++ /dev/null @@ -1,273 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# This file is autogenerated. -# To change this file you should edit scripts/templates/system_async_metrics.tpl and then run the following command: -# hatch run metrics:generate - -# https://clickhouse.com/docs/operations/system-tables/asynchronous_metrics -SystemAsynchronousMetrics = { - 'name': 'system_asynchronous_metrics', - 'query': 'SELECT value, metric FROM system.asynchronous_metrics', - 'columns': [ - {'name': 'metric_value', 'type': 'source'}, - { - 'name': 'metric_name', - 'type': 'match', - 'source': 'metric_value', - 'items': { - 'AsynchronousHeavyMetricsCalculationTimeSpent': { - 'name': 'asynchronous_metrics.AsynchronousHeavyMetricsCalculationTimeSpent', - 'type': 'gauge', - }, - 'AsynchronousHeavyMetricsUpdateInterval': { - 'name': 'asynchronous_metrics.AsynchronousHeavyMetricsUpdateInterval', - 'type': 'gauge', - }, - 'AsynchronousMetricsCalculationTimeSpent': { - 'name': 'asynchronous_metrics.AsynchronousMetricsCalculationTimeSpent', - 'type': 'gauge', - }, - 'AsynchronousMetricsUpdateInterval': { - 'name': 'asynchronous_metrics.AsynchronousMetricsUpdateInterval', - 'type': 'gauge', - }, - 'CGroupMaxCPU': {'name': 'asynchronous_metrics.CGroupMaxCPU', 'type': 'gauge'}, - 'CGroupMemoryTotal': {'name': 'asynchronous_metrics.CGroupMemoryTotal', 'type': 'gauge'}, - 'CGroupMemoryUsed': {'name': 'asynchronous_metrics.CGroupMemoryUsed', 'type': 'gauge'}, - 'CGroupSystemTime': {'name': 'asynchronous_metrics.CGroupSystemTime', 'type': 'gauge'}, - 'CGroupSystemTimeNormalized': { - 'name': 'asynchronous_metrics.CGroupSystemTimeNormalized', - 'type': 'gauge', - }, - 'CGroupUserTime': {'name': 'asynchronous_metrics.CGroupUserTime', 'type': 'gauge'}, - 'CGroupUserTimeNormalized': {'name': 'asynchronous_metrics.CGroupUserTimeNormalized', 'type': 'gauge'}, - 'CompiledExpressionCacheBytes': { - 'name': 'asynchronous_metrics.CompiledExpressionCacheBytes', - 'type': 'gauge', - }, - 'CompiledExpressionCacheCount': { - 'name': 'asynchronous_metrics.CompiledExpressionCacheCount', - 'type': 'gauge', - }, - 'DictionaryTotalFailedUpdates': { - 'name': 'asynchronous_metrics.DictionaryTotalFailedUpdates', - 'type': 'gauge', - }, - 'FilesystemCacheBytes': {'name': 'asynchronous_metrics.FilesystemCacheBytes', 'type': 'gauge'}, - 'FilesystemCacheCapacity': {'name': 'asynchronous_metrics.FilesystemCacheCapacity', 'type': 'gauge'}, - 'FilesystemCacheFiles': {'name': 'asynchronous_metrics.FilesystemCacheFiles', 'type': 'gauge'}, - 'FilesystemLogsPathAvailableBytes': { - 'name': 'asynchronous_metrics.FilesystemLogsPathAvailableBytes', - 'type': 'gauge', - }, - 'FilesystemLogsPathAvailableINodes': { - 'name': 'asynchronous_metrics.FilesystemLogsPathAvailableINodes', - 'type': 'gauge', - }, - 'FilesystemLogsPathTotalBytes': { - 'name': 'asynchronous_metrics.FilesystemLogsPathTotalBytes', - 'type': 'gauge', - }, - 'FilesystemLogsPathTotalINodes': { - 'name': 'asynchronous_metrics.FilesystemLogsPathTotalINodes', - 'type': 'gauge', - }, - 'FilesystemLogsPathUsedBytes': { - 'name': 'asynchronous_metrics.FilesystemLogsPathUsedBytes', - 'type': 'gauge', - }, - 'FilesystemLogsPathUsedINodes': { - 'name': 'asynchronous_metrics.FilesystemLogsPathUsedINodes', - 'type': 'gauge', - }, - 'FilesystemMainPathAvailableBytes': { - 'name': 'asynchronous_metrics.FilesystemMainPathAvailableBytes', - 'type': 'gauge', - }, - 'FilesystemMainPathAvailableINodes': { - 'name': 'asynchronous_metrics.FilesystemMainPathAvailableINodes', - 'type': 'gauge', - }, - 'FilesystemMainPathTotalBytes': { - 'name': 'asynchronous_metrics.FilesystemMainPathTotalBytes', - 'type': 'gauge', - }, - 'FilesystemMainPathTotalINodes': { - 'name': 'asynchronous_metrics.FilesystemMainPathTotalINodes', - 'type': 'gauge', - }, - 'FilesystemMainPathUsedBytes': { - 'name': 'asynchronous_metrics.FilesystemMainPathUsedBytes', - 'type': 'gauge', - }, - 'FilesystemMainPathUsedINodes': { - 'name': 'asynchronous_metrics.FilesystemMainPathUsedINodes', - 'type': 'gauge', - }, - 'HashTableStatsCacheEntries': { - 'name': 'asynchronous_metrics.HashTableStatsCacheEntries', - 'type': 'gauge', - }, - 'HashTableStatsCacheHits': {'name': 'asynchronous_metrics.HashTableStatsCacheHits', 'type': 'gauge'}, - 'HashTableStatsCacheMisses': { - 'name': 'asynchronous_metrics.HashTableStatsCacheMisses', - 'type': 'gauge', - }, - 'IndexMarkCacheBytes': {'name': 'asynchronous_metrics.IndexMarkCacheBytes', 'type': 'gauge'}, - 'IndexMarkCacheFiles': {'name': 'asynchronous_metrics.IndexMarkCacheFiles', 'type': 'gauge'}, - 'IndexUncompressedCacheBytes': { - 'name': 'asynchronous_metrics.IndexUncompressedCacheBytes', - 'type': 'gauge', - }, - 'IndexUncompressedCacheCells': { - 'name': 'asynchronous_metrics.IndexUncompressedCacheCells', - 'type': 'gauge', - }, - 'Jitter': {'name': 'asynchronous_metrics.Jitter', 'type': 'gauge'}, - 'LoadAverage1': {'name': 'asynchronous_metrics.LoadAverage1', 'type': 'gauge'}, - 'LoadAverage15': {'name': 'asynchronous_metrics.LoadAverage15', 'type': 'gauge'}, - 'LoadAverage5': {'name': 'asynchronous_metrics.LoadAverage5', 'type': 'gauge'}, - 'MMapCacheCells': {'name': 'asynchronous_metrics.MMapCacheCells', 'type': 'gauge'}, - 'MarkCacheBytes': {'name': 'asynchronous_metrics.MarkCacheBytes', 'type': 'gauge'}, - 'MarkCacheFiles': {'name': 'asynchronous_metrics.MarkCacheFiles', 'type': 'gauge'}, - 'MaxPartCountForPartition': {'name': 'asynchronous_metrics.MaxPartCountForPartition', 'type': 'gauge'}, - 'MemoryCode': {'name': 'asynchronous_metrics.MemoryCode', 'type': 'gauge'}, - 'MemoryDataAndStack': {'name': 'asynchronous_metrics.MemoryDataAndStack', 'type': 'gauge'}, - 'MemoryResident': {'name': 'asynchronous_metrics.MemoryResident', 'type': 'gauge'}, - 'MemoryResidentMax': {'name': 'asynchronous_metrics.MemoryResidentMax', 'type': 'gauge'}, - 'MemoryShared': {'name': 'asynchronous_metrics.MemoryShared', 'type': 'gauge'}, - 'MemoryVirtual': {'name': 'asynchronous_metrics.MemoryVirtual', 'type': 'gauge'}, - 'NetworkTCPReceiveQueue': {'name': 'asynchronous_metrics.NetworkTCPReceiveQueue', 'type': 'gauge'}, - 'NetworkTCPSocketRemoteAddresses': { - 'name': 'asynchronous_metrics.NetworkTCPSocketRemoteAddresses', - 'type': 'gauge', - }, - 'NetworkTCPSockets': {'name': 'asynchronous_metrics.NetworkTCPSockets', 'type': 'gauge'}, - 'NetworkTCPTransmitQueue': {'name': 'asynchronous_metrics.NetworkTCPTransmitQueue', 'type': 'gauge'}, - 'NetworkTCPUnrecoveredRetransmits': { - 'name': 'asynchronous_metrics.NetworkTCPUnrecoveredRetransmits', - 'type': 'gauge', - }, - 'NumberOfDatabases': {'name': 'asynchronous_metrics.NumberOfDatabases', 'type': 'gauge'}, - 'NumberOfDetachedByUserParts': { - 'name': 'asynchronous_metrics.NumberOfDetachedByUserParts', - 'type': 'gauge', - }, - 'NumberOfDetachedParts': {'name': 'asynchronous_metrics.NumberOfDetachedParts', 'type': 'gauge'}, - 'NumberOfPendingMutations': {'name': 'asynchronous_metrics.NumberOfPendingMutations', 'type': 'gauge'}, - 'NumberOfPendingMutationsOverExecutionTime': { - 'name': 'asynchronous_metrics.NumberOfPendingMutationsOverExecutionTime', - 'type': 'gauge', - }, - 'NumberOfStuckMutations': {'name': 'asynchronous_metrics.NumberOfStuckMutations', 'type': 'gauge'}, - 'NumberOfTables': {'name': 'asynchronous_metrics.NumberOfTables', 'type': 'gauge'}, - 'NumberOfTablesSystem': {'name': 'asynchronous_metrics.NumberOfTablesSystem', 'type': 'gauge'}, - 'OSCPUOverload': {'name': 'asynchronous_metrics.OSCPUOverload', 'type': 'gauge'}, - 'OSContextSwitches': {'name': 'asynchronous_metrics.OSContextSwitches', 'type': 'gauge'}, - 'OSGuestNiceTimeNormalized': { - 'name': 'asynchronous_metrics.OSGuestNiceTimeNormalized', - 'type': 'gauge', - }, - 'OSGuestTimeNormalized': {'name': 'asynchronous_metrics.OSGuestTimeNormalized', 'type': 'gauge'}, - 'OSIOWaitTimeNormalized': {'name': 'asynchronous_metrics.OSIOWaitTimeNormalized', 'type': 'gauge'}, - 'OSIdleTimeNormalized': {'name': 'asynchronous_metrics.OSIdleTimeNormalized', 'type': 'gauge'}, - 'OSInterrupts': {'name': 'asynchronous_metrics.OSInterrupts', 'type': 'gauge'}, - 'OSIrqTimeNormalized': {'name': 'asynchronous_metrics.OSIrqTimeNormalized', 'type': 'gauge'}, - 'OSMemoryAvailable': {'name': 'asynchronous_metrics.OSMemoryAvailable', 'type': 'gauge'}, - 'OSMemoryBuffers': {'name': 'asynchronous_metrics.OSMemoryBuffers', 'type': 'gauge'}, - 'OSMemoryCached': {'name': 'asynchronous_metrics.OSMemoryCached', 'type': 'gauge'}, - 'OSMemoryFreePlusCached': {'name': 'asynchronous_metrics.OSMemoryFreePlusCached', 'type': 'gauge'}, - 'OSMemoryFreeWithoutCached': { - 'name': 'asynchronous_metrics.OSMemoryFreeWithoutCached', - 'type': 'gauge', - }, - 'OSMemorySwapCached': {'name': 'asynchronous_metrics.OSMemorySwapCached', 'type': 'gauge'}, - 'OSMemoryTotal': {'name': 'asynchronous_metrics.OSMemoryTotal', 'type': 'gauge'}, - 'OSNiceTimeNormalized': {'name': 'asynchronous_metrics.OSNiceTimeNormalized', 'type': 'gauge'}, - 'OSOpenFiles': {'name': 'asynchronous_metrics.OSOpenFiles', 'type': 'gauge'}, - 'OSProcessesBlocked': {'name': 'asynchronous_metrics.OSProcessesBlocked', 'type': 'gauge'}, - 'OSProcessesCreated': {'name': 'asynchronous_metrics.OSProcessesCreated', 'type': 'gauge'}, - 'OSProcessesRunning': {'name': 'asynchronous_metrics.OSProcessesRunning', 'type': 'gauge'}, - 'OSSoftIrqTimeNormalized': {'name': 'asynchronous_metrics.OSSoftIrqTimeNormalized', 'type': 'gauge'}, - 'OSStealTimeNormalized': {'name': 'asynchronous_metrics.OSStealTimeNormalized', 'type': 'gauge'}, - 'OSSystemTimeNormalized': {'name': 'asynchronous_metrics.OSSystemTimeNormalized', 'type': 'gauge'}, - 'OSThreadsRunnable': {'name': 'asynchronous_metrics.OSThreadsRunnable', 'type': 'gauge'}, - 'OSThreadsTotal': {'name': 'asynchronous_metrics.OSThreadsTotal', 'type': 'gauge'}, - 'OSUptime': {'name': 'asynchronous_metrics.OSUptime', 'type': 'gauge'}, - 'OSUserTimeNormalized': {'name': 'asynchronous_metrics.OSUserTimeNormalized', 'type': 'gauge'}, - 'PageCacheBytes': {'name': 'asynchronous_metrics.PageCacheBytes', 'type': 'gauge'}, - 'PageCacheCells': {'name': 'asynchronous_metrics.PageCacheCells', 'type': 'gauge'}, - 'PageCacheMaxBytes': {'name': 'asynchronous_metrics.PageCacheMaxBytes', 'type': 'gauge'}, - 'PageCachePinnedBytes': {'name': 'asynchronous_metrics.PageCachePinnedBytes', 'type': 'gauge'}, - 'PrimaryIndexCacheBytes': {'name': 'asynchronous_metrics.PrimaryIndexCacheBytes', 'type': 'gauge'}, - 'PrimaryIndexCacheFiles': {'name': 'asynchronous_metrics.PrimaryIndexCacheFiles', 'type': 'gauge'}, - 'QueryCacheBytes': {'name': 'asynchronous_metrics.QueryCacheBytes', 'type': 'gauge'}, - 'QueryCacheEntries': {'name': 'asynchronous_metrics.QueryCacheEntries', 'type': 'gauge'}, - 'ReplicasMaxAbsoluteDelay': {'name': 'asynchronous_metrics.ReplicasMaxAbsoluteDelay', 'type': 'gauge'}, - 'ReplicasMaxInsertsInQueue': { - 'name': 'asynchronous_metrics.ReplicasMaxInsertsInQueue', - 'type': 'gauge', - }, - 'ReplicasMaxMergesInQueue': {'name': 'asynchronous_metrics.ReplicasMaxMergesInQueue', 'type': 'gauge'}, - 'ReplicasMaxQueueSize': {'name': 'asynchronous_metrics.ReplicasMaxQueueSize', 'type': 'gauge'}, - 'ReplicasMaxRelativeDelay': {'name': 'asynchronous_metrics.ReplicasMaxRelativeDelay', 'type': 'gauge'}, - 'ReplicasSumInsertsInQueue': { - 'name': 'asynchronous_metrics.ReplicasSumInsertsInQueue', - 'type': 'gauge', - }, - 'ReplicasSumMergesInQueue': {'name': 'asynchronous_metrics.ReplicasSumMergesInQueue', 'type': 'gauge'}, - 'ReplicasSumQueueSize': {'name': 'asynchronous_metrics.ReplicasSumQueueSize', 'type': 'gauge'}, - 'TotalBytesOfMergeTreeTables': { - 'name': 'asynchronous_metrics.TotalBytesOfMergeTreeTables', - 'type': 'gauge', - }, - 'TotalBytesOfMergeTreeTablesSystem': { - 'name': 'asynchronous_metrics.TotalBytesOfMergeTreeTablesSystem', - 'type': 'gauge', - }, - 'TotalIndexGranularityBytesInMemory': { - 'name': 'asynchronous_metrics.TotalIndexGranularityBytesInMemory', - 'type': 'gauge', - }, - 'TotalIndexGranularityBytesInMemoryAllocated': { - 'name': 'asynchronous_metrics.TotalIndexGranularityBytesInMemoryAllocated', - 'type': 'gauge', - }, - 'TotalPartsOfMergeTreeTables': { - 'name': 'asynchronous_metrics.TotalPartsOfMergeTreeTables', - 'type': 'gauge', - }, - 'TotalPartsOfMergeTreeTablesSystem': { - 'name': 'asynchronous_metrics.TotalPartsOfMergeTreeTablesSystem', - 'type': 'gauge', - }, - 'TotalPrimaryKeyBytesInMemory': { - 'name': 'asynchronous_metrics.TotalPrimaryKeyBytesInMemory', - 'type': 'gauge', - }, - 'TotalPrimaryKeyBytesInMemoryAllocated': { - 'name': 'asynchronous_metrics.TotalPrimaryKeyBytesInMemoryAllocated', - 'type': 'gauge', - }, - 'TotalRowsOfMergeTreeTables': { - 'name': 'asynchronous_metrics.TotalRowsOfMergeTreeTables', - 'type': 'gauge', - }, - 'TotalRowsOfMergeTreeTablesSystem': { - 'name': 'asynchronous_metrics.TotalRowsOfMergeTreeTablesSystem', - 'type': 'gauge', - }, - 'TrackedMemory': {'name': 'asynchronous_metrics.TrackedMemory', 'type': 'gauge'}, - 'UncompressedCacheBytes': {'name': 'asynchronous_metrics.UncompressedCacheBytes', 'type': 'gauge'}, - 'UncompressedCacheCells': {'name': 'asynchronous_metrics.UncompressedCacheCells', 'type': 'gauge'}, - 'UnreclaimableRSS': {'name': 'asynchronous_metrics.UnreclaimableRSS', 'type': 'gauge'}, - 'Uptime': {'name': 'asynchronous_metrics.Uptime', 'type': 'gauge'}, - 'VMMaxMapCount': {'name': 'asynchronous_metrics.VMMaxMapCount', 'type': 'gauge'}, - 'VMNumMaps': {'name': 'asynchronous_metrics.VMNumMaps', 'type': 'gauge'}, - 'jemalloc.epoch': {'name': 'asynchronous_metrics.jemalloc.epoch', 'type': 'gauge'}, - }, - }, - ], -} diff --git a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_errors.py b/clickhouse/datadog_checks/clickhouse/advanced_queries/system_errors.py deleted file mode 100644 index 685e5b6ffbe8a..0000000000000 --- a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_errors.py +++ /dev/null @@ -1,15 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# https://clickhouse.com/docs/operations/system-tables/errors -SystemErrors = { - 'name': 'system.errors', - 'query': 'SELECT value, name, code, remote FROM system.errors WHERE value > 0', - 'columns': [ - {'name': 'errors.raised', 'type': 'monotonic_count'}, - {'name': 'error_name', 'type': 'tag'}, - {'name': 'error_code', 'type': 'tag'}, - {'name': 'remote', 'type': 'tag', 'boolean': True}, - ], -} diff --git a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_events.py b/clickhouse/datadog_checks/clickhouse/advanced_queries/system_events.py deleted file mode 100644 index 3a8ef7ef2662c..0000000000000 --- a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_events.py +++ /dev/null @@ -1,3073 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# This file is autogenerated. -# To change this file you should edit scripts/templates/system_events.tpl and then run the following command: -# hatch run metrics:generate - -# https://clickhouse.com/docs/operations/system-tables/events -SystemEvents = { - 'name': 'system_events', - 'query': 'SELECT value, event FROM system.events', - 'columns': [ - {'name': 'metric_value', 'type': 'source'}, - { - 'name': 'metric_name', - 'type': 'match', - 'source': 'metric_value', - 'items': { - 'AIORead': {'name': 'events.AIORead', 'type': 'monotonic_gauge'}, - 'AIOReadBytes': {'name': 'events.AIOReadBytes', 'type': 'monotonic_gauge'}, - 'AIOWrite': {'name': 'events.AIOWrite', 'type': 'monotonic_gauge'}, - 'AIOWriteBytes': {'name': 'events.AIOWriteBytes', 'type': 'monotonic_gauge'}, - 'AddressesDiscovered': {'name': 'events.AddressesDiscovered', 'type': 'monotonic_gauge'}, - 'AddressesExpired': {'name': 'events.AddressesExpired', 'type': 'monotonic_gauge'}, - 'AddressesMarkedAsFailed': {'name': 'events.AddressesMarkedAsFailed', 'type': 'monotonic_gauge'}, - 'AggregatingSortedMilliseconds': { - 'name': 'events.AggregatingSortedMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'AggregationHashTablesInitializedAsTwoLevel': { - 'name': 'events.AggregationHashTablesInitializedAsTwoLevel', - 'type': 'monotonic_gauge', - }, - 'AggregationOptimizedEqualRangesOfKeys': { - 'name': 'events.AggregationOptimizedEqualRangesOfKeys', - 'type': 'monotonic_gauge', - }, - 'AggregationPreallocatedElementsInHashTables': { - 'name': 'events.AggregationPreallocatedElementsInHashTables', - 'type': 'monotonic_gauge', - }, - 'AnalyzePatchRangesMicroseconds': { - 'name': 'events.AnalyzePatchRangesMicroseconds', - 'type': 'monotonic_gauge', - }, - 'ApplyPatchesMicroseconds': {'name': 'events.ApplyPatchesMicroseconds', 'type': 'monotonic_gauge'}, - 'ArenaAllocBytes': {'name': 'events.ArenaAllocBytes', 'type': 'monotonic_gauge'}, - 'ArenaAllocChunks': {'name': 'events.ArenaAllocChunks', 'type': 'monotonic_gauge'}, - 'AsyncInsertBytes': {'name': 'events.AsyncInsertBytes', 'type': 'monotonic_gauge'}, - 'AsyncInsertCacheHits': {'name': 'events.AsyncInsertCacheHits', 'type': 'monotonic_gauge'}, - 'AsyncInsertQuery': {'name': 'events.AsyncInsertQuery', 'type': 'monotonic_gauge'}, - 'AsyncInsertRows': {'name': 'events.AsyncInsertRows', 'type': 'monotonic_gauge'}, - 'AsyncLoaderWaitMicroseconds': { - 'name': 'events.AsyncLoaderWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'AsyncLoggingConsoleDroppedMessages': { - 'name': 'events.AsyncLoggingConsoleDroppedMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingConsoleTotalMessages': { - 'name': 'events.AsyncLoggingConsoleTotalMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingErrorFileLogDroppedMessages': { - 'name': 'events.AsyncLoggingErrorFileLogDroppedMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingErrorFileLogTotalMessages': { - 'name': 'events.AsyncLoggingErrorFileLogTotalMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingFileLogDroppedMessages': { - 'name': 'events.AsyncLoggingFileLogDroppedMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingFileLogTotalMessages': { - 'name': 'events.AsyncLoggingFileLogTotalMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingSyslogDroppedMessages': { - 'name': 'events.AsyncLoggingSyslogDroppedMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingSyslogTotalMessages': { - 'name': 'events.AsyncLoggingSyslogTotalMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingTextLogDroppedMessages': { - 'name': 'events.AsyncLoggingTextLogDroppedMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingTextLogTotalMessages': { - 'name': 'events.AsyncLoggingTextLogTotalMessages', - 'type': 'monotonic_gauge', - }, - 'AsynchronousReadWaitMicroseconds': { - 'name': 'events.AsynchronousReadWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'AsynchronousReaderIgnoredBytes': { - 'name': 'events.AsynchronousReaderIgnoredBytes', - 'type': 'monotonic_gauge', - }, - 'AsynchronousRemoteReadWaitMicroseconds': { - 'name': 'events.AsynchronousRemoteReadWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'AzureCommitBlockList': {'name': 'events.AzureCommitBlockList', 'type': 'monotonic_gauge'}, - 'AzureCopyObject': {'name': 'events.AzureCopyObject', 'type': 'monotonic_gauge'}, - 'AzureCreateContainer': {'name': 'events.AzureCreateContainer', 'type': 'monotonic_gauge'}, - 'AzureDeleteObjects': {'name': 'events.AzureDeleteObjects', 'type': 'monotonic_gauge'}, - 'AzureGetObject': {'name': 'events.AzureGetObject', 'type': 'monotonic_gauge'}, - 'AzureGetProperties': {'name': 'events.AzureGetProperties', 'type': 'monotonic_gauge'}, - 'AzureGetRequestThrottlerCount': { - 'name': 'events.AzureGetRequestThrottlerCount', - 'type': 'monotonic_gauge', - }, - 'AzureGetRequestThrottlerSleepMicroseconds': { - 'name': 'events.AzureGetRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'AzureListObjects': {'name': 'events.AzureListObjects', 'type': 'monotonic_gauge'}, - 'AzurePutRequestThrottlerCount': { - 'name': 'events.AzurePutRequestThrottlerCount', - 'type': 'monotonic_gauge', - }, - 'AzurePutRequestThrottlerSleepMicroseconds': { - 'name': 'events.AzurePutRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'AzureReadMicroseconds': { - 'name': 'events.AzureReadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'AzureReadRequestsCount': {'name': 'events.AzureReadRequestsCount', 'type': 'monotonic_gauge'}, - 'AzureReadRequestsErrors': {'name': 'events.AzureReadRequestsErrors', 'type': 'monotonic_gauge'}, - 'AzureReadRequestsRedirects': {'name': 'events.AzureReadRequestsRedirects', 'type': 'monotonic_gauge'}, - 'AzureReadRequestsThrottling': { - 'name': 'events.AzureReadRequestsThrottling', - 'type': 'monotonic_gauge', - }, - 'AzureStageBlock': {'name': 'events.AzureStageBlock', 'type': 'monotonic_gauge'}, - 'AzureUpload': {'name': 'events.AzureUpload', 'type': 'monotonic_gauge'}, - 'AzureWriteMicroseconds': { - 'name': 'events.AzureWriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'AzureWriteRequestsCount': {'name': 'events.AzureWriteRequestsCount', 'type': 'monotonic_gauge'}, - 'AzureWriteRequestsErrors': {'name': 'events.AzureWriteRequestsErrors', 'type': 'monotonic_gauge'}, - 'AzureWriteRequestsRedirects': { - 'name': 'events.AzureWriteRequestsRedirects', - 'type': 'monotonic_gauge', - }, - 'AzureWriteRequestsThrottling': { - 'name': 'events.AzureWriteRequestsThrottling', - 'type': 'monotonic_gauge', - }, - 'BackgroundLoadingMarksTasks': { - 'name': 'events.BackgroundLoadingMarksTasks', - 'type': 'monotonic_gauge', - }, - 'BackupEntriesCollectorForTablesDataMicroseconds': { - 'name': 'events.BackupEntriesCollectorForTablesDataMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'BackupEntriesCollectorMicroseconds': { - 'name': 'events.BackupEntriesCollectorMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'BackupEntriesCollectorRunPostTasksMicroseconds': { - 'name': 'events.BackupEntriesCollectorRunPostTasksMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'BackupLockFileReads': {'name': 'events.BackupLockFileReads', 'type': 'monotonic_gauge'}, - 'BackupPreparingFileInfosMicroseconds': { - 'name': 'events.BackupPreparingFileInfosMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'BackupReadLocalBytesToCalculateChecksums': { - 'name': 'events.BackupReadLocalBytesToCalculateChecksums', - 'type': 'monotonic_gauge', - }, - 'BackupReadLocalFilesToCalculateChecksums': { - 'name': 'events.BackupReadLocalFilesToCalculateChecksums', - 'type': 'monotonic_gauge', - }, - 'BackupReadMetadataMicroseconds': { - 'name': 'events.BackupReadMetadataMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'BackupReadRemoteBytesToCalculateChecksums': { - 'name': 'events.BackupReadRemoteBytesToCalculateChecksums', - 'type': 'monotonic_gauge', - }, - 'BackupReadRemoteFilesToCalculateChecksums': { - 'name': 'events.BackupReadRemoteFilesToCalculateChecksums', - 'type': 'monotonic_gauge', - }, - 'BackupThrottlerBytes': {'name': 'events.BackupThrottlerBytes', 'type': 'monotonic_gauge'}, - 'BackupThrottlerSleepMicroseconds': { - 'name': 'events.BackupThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'BackupWriteMetadataMicroseconds': { - 'name': 'events.BackupWriteMetadataMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'BackupsOpenedForRead': {'name': 'events.BackupsOpenedForRead', 'type': 'monotonic_gauge'}, - 'BackupsOpenedForUnlock': {'name': 'events.BackupsOpenedForUnlock', 'type': 'monotonic_gauge'}, - 'BackupsOpenedForWrite': {'name': 'events.BackupsOpenedForWrite', 'type': 'monotonic_gauge'}, - 'BuildPatchesJoinMicroseconds': { - 'name': 'events.BuildPatchesJoinMicroseconds', - 'type': 'monotonic_gauge', - }, - 'BuildPatchesMergeMicroseconds': { - 'name': 'events.BuildPatchesMergeMicroseconds', - 'type': 'monotonic_gauge', - }, - 'CacheWarmerBytesDownloaded': {'name': 'events.CacheWarmerBytesDownloaded', 'type': 'monotonic_gauge'}, - 'CacheWarmerDataPartsDownloaded': { - 'name': 'events.CacheWarmerDataPartsDownloaded', - 'type': 'monotonic_gauge', - }, - 'CachedReadBufferCacheWriteBytes': { - 'name': 'events.CachedReadBufferCacheWriteBytes', - 'type': 'monotonic_gauge', - }, - 'CachedReadBufferCacheWriteMicroseconds': { - 'name': 'events.CachedReadBufferCacheWriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CachedReadBufferCreateBufferMicroseconds': { - 'name': 'events.CachedReadBufferCreateBufferMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CachedReadBufferPredownloadedBytes': { - 'name': 'events.CachedReadBufferPredownloadedBytes', - 'type': 'monotonic_gauge', - }, - 'CachedReadBufferReadFromCacheBytes': { - 'name': 'events.CachedReadBufferReadFromCacheBytes', - 'type': 'monotonic_gauge', - }, - 'CachedReadBufferReadFromCacheHits': { - 'name': 'events.CachedReadBufferReadFromCacheHits', - 'type': 'monotonic_gauge', - }, - 'CachedReadBufferReadFromCacheMicroseconds': { - 'name': 'events.CachedReadBufferReadFromCacheMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CachedReadBufferReadFromCacheMisses': { - 'name': 'events.CachedReadBufferReadFromCacheMisses', - 'type': 'monotonic_gauge', - }, - 'CachedReadBufferReadFromSourceBytes': { - 'name': 'events.CachedReadBufferReadFromSourceBytes', - 'type': 'monotonic_gauge', - }, - 'CachedReadBufferReadFromSourceMicroseconds': { - 'name': 'events.CachedReadBufferReadFromSourceMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CachedReadBufferWaitReadBufferMicroseconds': { - 'name': 'events.CachedReadBufferWaitReadBufferMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CachedWriteBufferCacheWriteBytes': { - 'name': 'events.CachedWriteBufferCacheWriteBytes', - 'type': 'monotonic_gauge', - }, - 'CachedWriteBufferCacheWriteMicroseconds': { - 'name': 'events.CachedWriteBufferCacheWriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CannotRemoveEphemeralNode': {'name': 'events.CannotRemoveEphemeralNode', 'type': 'monotonic_gauge'}, - 'CannotWriteToWriteBufferDiscard': { - 'name': 'events.CannotWriteToWriteBufferDiscard', - 'type': 'monotonic_gauge', - }, - 'CoalescingSortedMilliseconds': { - 'name': 'events.CoalescingSortedMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'CollapsingSortedMilliseconds': { - 'name': 'events.CollapsingSortedMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'CommonBackgroundExecutorTaskCancelMicroseconds': { - 'name': 'events.CommonBackgroundExecutorTaskCancelMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CommonBackgroundExecutorTaskExecuteStepMicroseconds': { - 'name': 'events.CommonBackgroundExecutorTaskExecuteStepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CommonBackgroundExecutorTaskResetMicroseconds': { - 'name': 'events.CommonBackgroundExecutorTaskResetMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CommonBackgroundExecutorWaitMicroseconds': { - 'name': 'events.CommonBackgroundExecutorWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CompileExpressionsBytes': {'name': 'events.CompileExpressionsBytes', 'type': 'monotonic_gauge'}, - 'CompileExpressionsMicroseconds': { - 'name': 'events.CompileExpressionsMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CompileFunction': {'name': 'events.CompileFunction', 'type': 'monotonic_gauge'}, - 'CompiledFunctionExecute': {'name': 'events.CompiledFunctionExecute', 'type': 'monotonic_gauge'}, - 'CompressedReadBufferBlocks': {'name': 'events.CompressedReadBufferBlocks', 'type': 'monotonic_gauge'}, - 'CompressedReadBufferBytes': {'name': 'events.CompressedReadBufferBytes', 'type': 'monotonic_gauge'}, - 'CompressedReadBufferChecksumDoesntMatch': { - 'name': 'events.CompressedReadBufferChecksumDoesntMatch', - 'type': 'monotonic_gauge', - }, - 'CompressedReadBufferChecksumDoesntMatchMicroseconds': { - 'name': 'events.CompressedReadBufferChecksumDoesntMatchMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CompressedReadBufferChecksumDoesntMatchSingleBitMismatch': { - 'name': 'events.CompressedReadBufferChecksumDoesntMatchSingleBitMismatch', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlDownscales': { - 'name': 'events.ConcurrencyControlDownscales', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlPreemptedMicroseconds': { - 'name': 'events.ConcurrencyControlPreemptedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ConcurrencyControlPreemptions': { - 'name': 'events.ConcurrencyControlPreemptions', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlQueriesDelayed': { - 'name': 'events.ConcurrencyControlQueriesDelayed', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlSlotsAcquired': { - 'name': 'events.ConcurrencyControlSlotsAcquired', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlSlotsAcquiredNonCompeting': { - 'name': 'events.ConcurrencyControlSlotsAcquiredNonCompeting', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlSlotsDelayed': { - 'name': 'events.ConcurrencyControlSlotsDelayed', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlSlotsGranted': { - 'name': 'events.ConcurrencyControlSlotsGranted', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlUpscales': {'name': 'events.ConcurrencyControlUpscales', 'type': 'monotonic_gauge'}, - 'ConcurrencyControlWaitMicroseconds': { - 'name': 'events.ConcurrencyControlWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ConcurrentQuerySlotsAcquired': { - 'name': 'events.ConcurrentQuerySlotsAcquired', - 'type': 'monotonic_gauge', - }, - 'ConcurrentQueryWaitMicroseconds': { - 'name': 'events.ConcurrentQueryWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ConnectionPoolIsFullMicroseconds': { - 'name': 'events.ConnectionPoolIsFullMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ContextLock': {'name': 'events.ContextLock', 'type': 'monotonic_gauge'}, - 'ContextLockWaitMicroseconds': { - 'name': 'events.ContextLockWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeAssignmentRequest': { - 'name': 'events.CoordinatedMergesMergeAssignmentRequest', - 'type': 'monotonic_gauge', - }, - 'CoordinatedMergesMergeAssignmentRequestMicroseconds': { - 'name': 'events.CoordinatedMergesMergeAssignmentRequestMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeAssignmentResponse': { - 'name': 'events.CoordinatedMergesMergeAssignmentResponse', - 'type': 'monotonic_gauge', - }, - 'CoordinatedMergesMergeAssignmentResponseMicroseconds': { - 'name': 'events.CoordinatedMergesMergeAssignmentResponseMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeCoordinatorFetchMetadataMicroseconds': { - 'name': 'events.CoordinatedMergesMergeCoordinatorFetchMetadataMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeCoordinatorFilterMicroseconds': { - 'name': 'events.CoordinatedMergesMergeCoordinatorFilterMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeCoordinatorLockStateExclusivelyCount': { - 'name': 'events.CoordinatedMergesMergeCoordinatorLockStateExclusivelyCount', - 'type': 'monotonic_gauge', - }, - 'CoordinatedMergesMergeCoordinatorLockStateExclusivelyMicroseconds': { - 'name': 'events.CoordinatedMergesMergeCoordinatorLockStateExclusivelyMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeCoordinatorLockStateForShareCount': { - 'name': 'events.CoordinatedMergesMergeCoordinatorLockStateForShareCount', - 'type': 'monotonic_gauge', - }, - 'CoordinatedMergesMergeCoordinatorLockStateForShareMicroseconds': { - 'name': 'events.CoordinatedMergesMergeCoordinatorLockStateForShareMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeCoordinatorSelectMergesMicroseconds': { - 'name': 'events.CoordinatedMergesMergeCoordinatorSelectMergesMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeCoordinatorUpdateCount': { - 'name': 'events.CoordinatedMergesMergeCoordinatorUpdateCount', - 'type': 'monotonic_gauge', - }, - 'CoordinatedMergesMergeCoordinatorUpdateMicroseconds': { - 'name': 'events.CoordinatedMergesMergeCoordinatorUpdateMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeWorkerUpdateCount': { - 'name': 'events.CoordinatedMergesMergeWorkerUpdateCount', - 'type': 'monotonic_gauge', - }, - 'CoordinatedMergesMergeWorkerUpdateMicroseconds': { - 'name': 'events.CoordinatedMergesMergeWorkerUpdateMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CreatedLogEntryForMerge': {'name': 'events.CreatedLogEntryForMerge', 'type': 'monotonic_gauge'}, - 'CreatedLogEntryForMutation': {'name': 'events.CreatedLogEntryForMutation', 'type': 'monotonic_gauge'}, - 'CreatedReadBufferDirectIO': {'name': 'events.CreatedReadBufferDirectIO', 'type': 'monotonic_gauge'}, - 'CreatedReadBufferDirectIOFailed': { - 'name': 'events.CreatedReadBufferDirectIOFailed', - 'type': 'monotonic_gauge', - }, - 'CreatedReadBufferMMap': {'name': 'events.CreatedReadBufferMMap', 'type': 'monotonic_gauge'}, - 'CreatedReadBufferMMapFailed': { - 'name': 'events.CreatedReadBufferMMapFailed', - 'type': 'monotonic_gauge', - }, - 'CreatedReadBufferOrdinary': {'name': 'events.CreatedReadBufferOrdinary', 'type': 'monotonic_gauge'}, - 'DNSError': {'name': 'events.DNSError', 'type': 'monotonic_gauge'}, - 'DataAfterMutationDiffersFromReplica': { - 'name': 'events.DataAfterMutationDiffersFromReplica', - 'type': 'monotonic_gauge', - }, - 'DefaultImplementationForNullsRows': { - 'name': 'events.DefaultImplementationForNullsRows', - 'type': 'monotonic_gauge', - }, - 'DefaultImplementationForNullsRowsWithNulls': { - 'name': 'events.DefaultImplementationForNullsRowsWithNulls', - 'type': 'monotonic_gauge', - }, - 'DelayedInserts': {'name': 'events.DelayedInserts', 'type': 'monotonic_gauge'}, - 'DelayedInsertsMilliseconds': { - 'name': 'events.DelayedInsertsMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'DelayedMutations': {'name': 'events.DelayedMutations', 'type': 'monotonic_gauge'}, - 'DelayedMutationsMilliseconds': { - 'name': 'events.DelayedMutationsMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'DeltaLakePartitionPrunedFiles': { - 'name': 'events.DeltaLakePartitionPrunedFiles', - 'type': 'monotonic_gauge', - }, - 'DictCacheKeysExpired': {'name': 'events.DictCacheKeysExpired', 'type': 'monotonic_gauge'}, - 'DictCacheKeysHit': {'name': 'events.DictCacheKeysHit', 'type': 'monotonic_gauge'}, - 'DictCacheKeysNotFound': {'name': 'events.DictCacheKeysNotFound', 'type': 'monotonic_gauge'}, - 'DictCacheKeysRequested': {'name': 'events.DictCacheKeysRequested', 'type': 'monotonic_gauge'}, - 'DictCacheKeysRequestedFound': { - 'name': 'events.DictCacheKeysRequestedFound', - 'type': 'monotonic_gauge', - }, - 'DictCacheKeysRequestedMiss': {'name': 'events.DictCacheKeysRequestedMiss', 'type': 'monotonic_gauge'}, - 'DictCacheLockReadNs': { - 'name': 'events.DictCacheLockReadNs', - 'type': 'temporal_percent', - 'scale': 'nanosecond', - }, - 'DictCacheLockWriteNs': { - 'name': 'events.DictCacheLockWriteNs', - 'type': 'temporal_percent', - 'scale': 'nanosecond', - }, - 'DictCacheRequestTimeNs': { - 'name': 'events.DictCacheRequestTimeNs', - 'type': 'temporal_percent', - 'scale': 'nanosecond', - }, - 'DictCacheRequests': {'name': 'events.DictCacheRequests', 'type': 'monotonic_gauge'}, - 'DirectorySync': {'name': 'events.DirectorySync', 'type': 'monotonic_gauge'}, - 'DirectorySyncElapsedMicroseconds': { - 'name': 'events.DirectorySyncElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskAzureCommitBlockList': {'name': 'events.DiskAzureCommitBlockList', 'type': 'monotonic_gauge'}, - 'DiskAzureCopyObject': {'name': 'events.DiskAzureCopyObject', 'type': 'monotonic_gauge'}, - 'DiskAzureCreateContainer': {'name': 'events.DiskAzureCreateContainer', 'type': 'monotonic_gauge'}, - 'DiskAzureDeleteObjects': {'name': 'events.DiskAzureDeleteObjects', 'type': 'monotonic_gauge'}, - 'DiskAzureGetObject': {'name': 'events.DiskAzureGetObject', 'type': 'monotonic_gauge'}, - 'DiskAzureGetProperties': {'name': 'events.DiskAzureGetProperties', 'type': 'monotonic_gauge'}, - 'DiskAzureGetRequestThrottlerCount': { - 'name': 'events.DiskAzureGetRequestThrottlerCount', - 'type': 'monotonic_gauge', - }, - 'DiskAzureGetRequestThrottlerSleepMicroseconds': { - 'name': 'events.DiskAzureGetRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskAzureListObjects': {'name': 'events.DiskAzureListObjects', 'type': 'monotonic_gauge'}, - 'DiskAzurePutRequestThrottlerCount': { - 'name': 'events.DiskAzurePutRequestThrottlerCount', - 'type': 'monotonic_gauge', - }, - 'DiskAzurePutRequestThrottlerSleepMicroseconds': { - 'name': 'events.DiskAzurePutRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskAzureReadMicroseconds': { - 'name': 'events.DiskAzureReadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskAzureReadRequestsCount': {'name': 'events.DiskAzureReadRequestsCount', 'type': 'monotonic_gauge'}, - 'DiskAzureReadRequestsErrors': { - 'name': 'events.DiskAzureReadRequestsErrors', - 'type': 'monotonic_gauge', - }, - 'DiskAzureReadRequestsRedirects': { - 'name': 'events.DiskAzureReadRequestsRedirects', - 'type': 'monotonic_gauge', - }, - 'DiskAzureReadRequestsThrottling': { - 'name': 'events.DiskAzureReadRequestsThrottling', - 'type': 'monotonic_gauge', - }, - 'DiskAzureStageBlock': {'name': 'events.DiskAzureStageBlock', 'type': 'monotonic_gauge'}, - 'DiskAzureUpload': {'name': 'events.DiskAzureUpload', 'type': 'monotonic_gauge'}, - 'DiskAzureWriteMicroseconds': { - 'name': 'events.DiskAzureWriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskAzureWriteRequestsCount': { - 'name': 'events.DiskAzureWriteRequestsCount', - 'type': 'monotonic_gauge', - }, - 'DiskAzureWriteRequestsErrors': { - 'name': 'events.DiskAzureWriteRequestsErrors', - 'type': 'monotonic_gauge', - }, - 'DiskAzureWriteRequestsRedirects': { - 'name': 'events.DiskAzureWriteRequestsRedirects', - 'type': 'monotonic_gauge', - }, - 'DiskAzureWriteRequestsThrottling': { - 'name': 'events.DiskAzureWriteRequestsThrottling', - 'type': 'monotonic_gauge', - }, - 'DiskConnectionsCreated': {'name': 'events.DiskConnectionsCreated', 'type': 'monotonic_gauge'}, - 'DiskConnectionsElapsedMicroseconds': { - 'name': 'events.DiskConnectionsElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskConnectionsErrors': {'name': 'events.DiskConnectionsErrors', 'type': 'monotonic_gauge'}, - 'DiskConnectionsExpired': {'name': 'events.DiskConnectionsExpired', 'type': 'monotonic_gauge'}, - 'DiskConnectionsPreserved': {'name': 'events.DiskConnectionsPreserved', 'type': 'monotonic_gauge'}, - 'DiskConnectionsReset': {'name': 'events.DiskConnectionsReset', 'type': 'monotonic_gauge'}, - 'DiskConnectionsReused': {'name': 'events.DiskConnectionsReused', 'type': 'monotonic_gauge'}, - 'DiskPlainRewritableAzureDirectoryCreated': { - 'name': 'events.DiskPlainRewritableAzureDirectoryCreated', - 'type': 'monotonic_gauge', - }, - 'DiskPlainRewritableAzureDirectoryRemoved': { - 'name': 'events.DiskPlainRewritableAzureDirectoryRemoved', - 'type': 'monotonic_gauge', - }, - 'DiskPlainRewritableLegacyLayoutDiskCount': { - 'name': 'events.DiskPlainRewritableLegacyLayoutDiskCount', - 'type': 'monotonic_gauge', - }, - 'DiskPlainRewritableLocalDirectoryCreated': { - 'name': 'events.DiskPlainRewritableLocalDirectoryCreated', - 'type': 'monotonic_gauge', - }, - 'DiskPlainRewritableLocalDirectoryRemoved': { - 'name': 'events.DiskPlainRewritableLocalDirectoryRemoved', - 'type': 'monotonic_gauge', - }, - 'DiskPlainRewritableS3DirectoryCreated': { - 'name': 'events.DiskPlainRewritableS3DirectoryCreated', - 'type': 'monotonic_gauge', - }, - 'DiskPlainRewritableS3DirectoryRemoved': { - 'name': 'events.DiskPlainRewritableS3DirectoryRemoved', - 'type': 'monotonic_gauge', - }, - 'DiskReadElapsedMicroseconds': { - 'name': 'events.DiskReadElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskS3AbortMultipartUpload': {'name': 'events.DiskS3AbortMultipartUpload', 'type': 'monotonic_gauge'}, - 'DiskS3CompleteMultipartUpload': { - 'name': 'events.DiskS3CompleteMultipartUpload', - 'type': 'monotonic_gauge', - }, - 'DiskS3CopyObject': {'name': 'events.DiskS3CopyObject', 'type': 'monotonic_gauge'}, - 'DiskS3CreateMultipartUpload': { - 'name': 'events.DiskS3CreateMultipartUpload', - 'type': 'monotonic_gauge', - }, - 'DiskS3DeleteObjects': {'name': 'events.DiskS3DeleteObjects', 'type': 'monotonic_gauge'}, - 'DiskS3GetObject': {'name': 'events.DiskS3GetObject', 'type': 'monotonic_gauge'}, - 'DiskS3GetObjectAttributes': {'name': 'events.DiskS3GetObjectAttributes', 'type': 'monotonic_gauge'}, - 'DiskS3GetRequestThrottlerCount': { - 'name': 'events.DiskS3GetRequestThrottlerCount', - 'type': 'monotonic_gauge', - }, - 'DiskS3GetRequestThrottlerSleepMicroseconds': { - 'name': 'events.DiskS3GetRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskS3HeadObject': {'name': 'events.DiskS3HeadObject', 'type': 'monotonic_gauge'}, - 'DiskS3ListObjects': {'name': 'events.DiskS3ListObjects', 'type': 'monotonic_gauge'}, - 'DiskS3PutObject': {'name': 'events.DiskS3PutObject', 'type': 'monotonic_gauge'}, - 'DiskS3PutRequestThrottlerCount': { - 'name': 'events.DiskS3PutRequestThrottlerCount', - 'type': 'monotonic_gauge', - }, - 'DiskS3PutRequestThrottlerSleepMicroseconds': { - 'name': 'events.DiskS3PutRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskS3ReadMicroseconds': { - 'name': 'events.DiskS3ReadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskS3ReadRequestAttempts': {'name': 'events.DiskS3ReadRequestAttempts', 'type': 'monotonic_gauge'}, - 'DiskS3ReadRequestRetryableErrors': { - 'name': 'events.DiskS3ReadRequestRetryableErrors', - 'type': 'monotonic_gauge', - }, - 'DiskS3ReadRequestsCount': {'name': 'events.DiskS3ReadRequestsCount', 'type': 'monotonic_gauge'}, - 'DiskS3ReadRequestsErrors': {'name': 'events.DiskS3ReadRequestsErrors', 'type': 'monotonic_gauge'}, - 'DiskS3ReadRequestsRedirects': { - 'name': 'events.DiskS3ReadRequestsRedirects', - 'type': 'monotonic_gauge', - }, - 'DiskS3ReadRequestsThrottling': { - 'name': 'events.DiskS3ReadRequestsThrottling', - 'type': 'monotonic_gauge', - }, - 'DiskS3UploadPart': {'name': 'events.DiskS3UploadPart', 'type': 'monotonic_gauge'}, - 'DiskS3UploadPartCopy': {'name': 'events.DiskS3UploadPartCopy', 'type': 'monotonic_gauge'}, - 'DiskS3WriteMicroseconds': { - 'name': 'events.DiskS3WriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskS3WriteRequestAttempts': {'name': 'events.DiskS3WriteRequestAttempts', 'type': 'monotonic_gauge'}, - 'DiskS3WriteRequestRetryableErrors': { - 'name': 'events.DiskS3WriteRequestRetryableErrors', - 'type': 'monotonic_gauge', - }, - 'DiskS3WriteRequestsCount': {'name': 'events.DiskS3WriteRequestsCount', 'type': 'monotonic_gauge'}, - 'DiskS3WriteRequestsErrors': {'name': 'events.DiskS3WriteRequestsErrors', 'type': 'monotonic_gauge'}, - 'DiskS3WriteRequestsRedirects': { - 'name': 'events.DiskS3WriteRequestsRedirects', - 'type': 'monotonic_gauge', - }, - 'DiskS3WriteRequestsThrottling': { - 'name': 'events.DiskS3WriteRequestsThrottling', - 'type': 'monotonic_gauge', - }, - 'DiskWriteElapsedMicroseconds': { - 'name': 'events.DiskWriteElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheConnectAttempts': {'name': 'events.DistrCacheConnectAttempts', 'type': 'monotonic_gauge'}, - 'DistrCacheConnectMicroseconds': { - 'name': 'events.DistrCacheConnectMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheDataPacketsBytes': {'name': 'events.DistrCacheDataPacketsBytes', 'type': 'monotonic_gauge'}, - 'DistrCacheFallbackReadMicroseconds': { - 'name': 'events.DistrCacheFallbackReadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheGetClient': {'name': 'events.DistrCacheGetClient', 'type': 'gauge'}, - 'DistrCacheGetClientMicroseconds': { - 'name': 'events.DistrCacheGetClientMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheGetResponseMicroseconds': { - 'name': 'events.DistrCacheGetResponseMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheHashRingRebuilds': {'name': 'events.DistrCacheHashRingRebuilds', 'type': 'monotonic_gauge'}, - 'DistrCacheHoldConnections': {'name': 'events.DistrCacheHoldConnections', 'type': 'gauge'}, - 'DistrCacheIgnoredBytesWhileWaitingProfileEvents': { - 'name': 'events.DistrCacheIgnoredBytesWhileWaitingProfileEvents', - 'type': 'monotonic_gauge', - }, - 'DistrCacheLockRegistryMicroseconds': { - 'name': 'events.DistrCacheLockRegistryMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheMakeRequestErrors': { - 'name': 'events.DistrCacheMakeRequestErrors', - 'type': 'monotonic_gauge', - }, - 'DistrCacheNextImplMicroseconds': { - 'name': 'events.DistrCacheNextImplMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheOpenedConnections': { - 'name': 'events.DistrCacheOpenedConnections', - 'type': 'monotonic_gauge', - }, - 'DistrCacheOpenedConnectionsBypassingPool': { - 'name': 'events.DistrCacheOpenedConnectionsBypassingPool', - 'type': 'monotonic_gauge', - }, - 'DistrCachePackets': {'name': 'events.DistrCachePackets', 'type': 'monotonic_gauge'}, - 'DistrCachePacketsBytes': {'name': 'events.DistrCachePacketsBytes', 'type': 'monotonic_gauge'}, - 'DistrCachePrecomputeRangesMicroseconds': { - 'name': 'events.DistrCachePrecomputeRangesMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheRangeChange': {'name': 'events.DistrCacheRangeChange', 'type': 'monotonic_gauge'}, - 'DistrCacheRangeResetBackward': { - 'name': 'events.DistrCacheRangeResetBackward', - 'type': 'monotonic_gauge', - }, - 'DistrCacheRangeResetForward': { - 'name': 'events.DistrCacheRangeResetForward', - 'type': 'monotonic_gauge', - }, - 'DistrCacheReadBytesFromCache': { - 'name': 'events.DistrCacheReadBytesFromCache', - 'type': 'monotonic_gauge', - }, - 'DistrCacheReadBytesFromFallbackBuffer': { - 'name': 'events.DistrCacheReadBytesFromFallbackBuffer', - 'type': 'monotonic_gauge', - }, - 'DistrCacheReadErrors': {'name': 'events.DistrCacheReadErrors', 'type': 'monotonic_gauge'}, - 'DistrCacheReadMicroseconds': { - 'name': 'events.DistrCacheReadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheReceiveResponseErrors': { - 'name': 'events.DistrCacheReceiveResponseErrors', - 'type': 'monotonic_gauge', - }, - 'DistrCacheReconnectsAfterTimeout': { - 'name': 'events.DistrCacheReconnectsAfterTimeout', - 'type': 'monotonic_gauge', - }, - 'DistrCacheRegistryUpdateMicroseconds': { - 'name': 'events.DistrCacheRegistryUpdateMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheRegistryUpdates': {'name': 'events.DistrCacheRegistryUpdates', 'type': 'monotonic_gauge'}, - 'DistrCacheReusedConnections': { - 'name': 'events.DistrCacheReusedConnections', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerAckRequestPackets': { - 'name': 'events.DistrCacheServerAckRequestPackets', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerCachedReadBufferCacheHits': { - 'name': 'events.DistrCacheServerCachedReadBufferCacheHits', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerCachedReadBufferCacheMisses': { - 'name': 'events.DistrCacheServerCachedReadBufferCacheMisses', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerContinueRequestPackets': { - 'name': 'events.DistrCacheServerContinueRequestPackets', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerCredentialsRefresh': { - 'name': 'events.DistrCacheServerCredentialsRefresh', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerEndRequestPackets': { - 'name': 'events.DistrCacheServerEndRequestPackets', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerNewS3CachedClients': { - 'name': 'events.DistrCacheServerNewS3CachedClients', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerProcessRequestMicroseconds': { - 'name': 'events.DistrCacheServerProcessRequestMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheServerReceivedCredentialsRefreshPackets': { - 'name': 'events.DistrCacheServerReceivedCredentialsRefreshPackets', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerReusedS3CachedClients': { - 'name': 'events.DistrCacheServerReusedS3CachedClients', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerStartRequestPackets': { - 'name': 'events.DistrCacheServerStartRequestPackets', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerSwitches': {'name': 'events.DistrCacheServerSwitches', 'type': 'monotonic_gauge'}, - 'DistrCacheServerUpdates': {'name': 'events.DistrCacheServerUpdates', 'type': 'monotonic_gauge'}, - 'DistrCacheStartRangeMicroseconds': { - 'name': 'events.DistrCacheStartRangeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheUnusedDataPacketsBytes': { - 'name': 'events.DistrCacheUnusedDataPacketsBytes', - 'type': 'monotonic_gauge', - }, - 'DistrCacheUnusedPackets': {'name': 'events.DistrCacheUnusedPackets', 'type': 'monotonic_gauge'}, - 'DistrCacheUnusedPacketsBufferAllocations': { - 'name': 'events.DistrCacheUnusedPacketsBufferAllocations', - 'type': 'monotonic_gauge', - }, - 'DistrCacheUnusedPacketsBytes': { - 'name': 'events.DistrCacheUnusedPacketsBytes', - 'type': 'monotonic_gauge', - }, - 'DistributedAsyncInsertionFailures': { - 'name': 'events.DistributedAsyncInsertionFailures', - 'type': 'monotonic_gauge', - }, - 'DistributedConnectionFailAtAll': { - 'name': 'events.DistributedConnectionFailAtAll', - 'type': 'monotonic_gauge', - }, - 'DistributedConnectionFailTry': { - 'name': 'events.DistributedConnectionFailTry', - 'type': 'monotonic_gauge', - }, - 'DistributedConnectionMissingTable': { - 'name': 'events.DistributedConnectionMissingTable', - 'type': 'monotonic_gauge', - }, - 'DistributedConnectionReconnectCount': { - 'name': 'events.DistributedConnectionReconnectCount', - 'type': 'monotonic_gauge', - }, - 'DistributedConnectionSkipReadOnlyReplica': { - 'name': 'events.DistributedConnectionSkipReadOnlyReplica', - 'type': 'monotonic_gauge', - }, - 'DistributedConnectionStaleReplica': { - 'name': 'events.DistributedConnectionStaleReplica', - 'type': 'monotonic_gauge', - }, - 'DistributedConnectionTries': {'name': 'events.DistributedConnectionTries', 'type': 'monotonic_gauge'}, - 'DistributedConnectionUsable': { - 'name': 'events.DistributedConnectionUsable', - 'type': 'monotonic_gauge', - }, - 'DistributedDelayedInserts': {'name': 'events.DistributedDelayedInserts', 'type': 'monotonic_gauge'}, - 'DistributedDelayedInsertsMilliseconds': { - 'name': 'events.DistributedDelayedInsertsMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'DistributedRejectedInserts': {'name': 'events.DistributedRejectedInserts', 'type': 'monotonic_gauge'}, - 'DistributedSyncInsertionTimeoutExceeded': { - 'name': 'events.DistributedSyncInsertionTimeoutExceeded', - 'type': 'monotonic_gauge', - }, - 'DuplicatedInsertedBlocks': {'name': 'events.DuplicatedInsertedBlocks', 'type': 'monotonic_gauge'}, - 'EngineFileLikeReadFiles': {'name': 'events.EngineFileLikeReadFiles', 'type': 'monotonic_gauge'}, - 'ExecuteShellCommand': {'name': 'events.ExecuteShellCommand', 'type': 'monotonic_gauge'}, - 'ExternalAggregationCompressedBytes': { - 'name': 'events.ExternalAggregationCompressedBytes', - 'type': 'monotonic_gauge', - }, - 'ExternalAggregationMerge': {'name': 'events.ExternalAggregationMerge', 'type': 'monotonic_gauge'}, - 'ExternalAggregationUncompressedBytes': { - 'name': 'events.ExternalAggregationUncompressedBytes', - 'type': 'monotonic_gauge', - }, - 'ExternalAggregationWritePart': { - 'name': 'events.ExternalAggregationWritePart', - 'type': 'monotonic_gauge', - }, - 'ExternalDataSourceLocalCacheReadBytes': { - 'name': 'events.ExternalDataSourceLocalCacheReadBytes', - 'type': 'monotonic_gauge', - }, - 'ExternalJoinCompressedBytes': { - 'name': 'events.ExternalJoinCompressedBytes', - 'type': 'monotonic_gauge', - }, - 'ExternalJoinMerge': {'name': 'events.ExternalJoinMerge', 'type': 'monotonic_gauge'}, - 'ExternalJoinUncompressedBytes': { - 'name': 'events.ExternalJoinUncompressedBytes', - 'type': 'monotonic_gauge', - }, - 'ExternalJoinWritePart': {'name': 'events.ExternalJoinWritePart', 'type': 'monotonic_gauge'}, - 'ExternalProcessingCompressedBytesTotal': { - 'name': 'events.ExternalProcessingCompressedBytesTotal', - 'type': 'monotonic_gauge', - }, - 'ExternalProcessingFilesTotal': { - 'name': 'events.ExternalProcessingFilesTotal', - 'type': 'monotonic_gauge', - }, - 'ExternalProcessingUncompressedBytesTotal': { - 'name': 'events.ExternalProcessingUncompressedBytesTotal', - 'type': 'monotonic_gauge', - }, - 'ExternalSortCompressedBytes': { - 'name': 'events.ExternalSortCompressedBytes', - 'type': 'monotonic_gauge', - }, - 'ExternalSortMerge': {'name': 'events.ExternalSortMerge', 'type': 'monotonic_gauge'}, - 'ExternalSortUncompressedBytes': { - 'name': 'events.ExternalSortUncompressedBytes', - 'type': 'monotonic_gauge', - }, - 'ExternalSortWritePart': {'name': 'events.ExternalSortWritePart', 'type': 'monotonic_gauge'}, - 'FailedAsyncInsertQuery': {'name': 'events.FailedAsyncInsertQuery', 'type': 'monotonic_gauge'}, - 'FailedInsertQuery': {'name': 'events.FailedInsertQuery', 'type': 'monotonic_gauge'}, - 'FailedQuery': {'name': 'events.FailedQuery', 'type': 'monotonic_gauge'}, - 'FailedSelectQuery': {'name': 'events.FailedSelectQuery', 'type': 'monotonic_gauge'}, - 'FetchBackgroundExecutorTaskCancelMicroseconds': { - 'name': 'events.FetchBackgroundExecutorTaskCancelMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FetchBackgroundExecutorTaskExecuteStepMicroseconds': { - 'name': 'events.FetchBackgroundExecutorTaskExecuteStepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FetchBackgroundExecutorTaskResetMicroseconds': { - 'name': 'events.FetchBackgroundExecutorTaskResetMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FetchBackgroundExecutorWaitMicroseconds': { - 'name': 'events.FetchBackgroundExecutorWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileOpen': {'name': 'events.FileOpen', 'type': 'monotonic_gauge'}, - 'FileSegmentCacheWriteMicroseconds': { - 'name': 'events.FileSegmentCacheWriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentCompleteMicroseconds': { - 'name': 'events.FileSegmentCompleteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentFailToIncreasePriority': { - 'name': 'events.FileSegmentFailToIncreasePriority', - 'type': 'monotonic_gauge', - }, - 'FileSegmentHolderCompleteMicroseconds': { - 'name': 'events.FileSegmentHolderCompleteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentLockMicroseconds': { - 'name': 'events.FileSegmentLockMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentPredownloadMicroseconds': { - 'name': 'events.FileSegmentPredownloadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentReadMicroseconds': { - 'name': 'events.FileSegmentReadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentRemoveMicroseconds': { - 'name': 'events.FileSegmentRemoveMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentUseMicroseconds': { - 'name': 'events.FileSegmentUseMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentUsedBytes': {'name': 'events.FileSegmentUsedBytes', 'type': 'monotonic_gauge'}, - 'FileSegmentWaitMicroseconds': { - 'name': 'events.FileSegmentWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentWaitReadBufferMicroseconds': { - 'name': 'events.FileSegmentWaitReadBufferMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentWriteMicroseconds': { - 'name': 'events.FileSegmentWriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSync': {'name': 'events.FileSync', 'type': 'monotonic_gauge'}, - 'FileSyncElapsedMicroseconds': { - 'name': 'events.FileSyncElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheBackgroundDownloadQueuePush': { - 'name': 'events.FilesystemCacheBackgroundDownloadQueuePush', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheBackgroundEvictedBytes': { - 'name': 'events.FilesystemCacheBackgroundEvictedBytes', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheBackgroundEvictedFileSegments': { - 'name': 'events.FilesystemCacheBackgroundEvictedFileSegments', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheCreatedKeyDirectories': { - 'name': 'events.FilesystemCacheCreatedKeyDirectories', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheEvictMicroseconds': { - 'name': 'events.FilesystemCacheEvictMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheEvictedBytes': { - 'name': 'events.FilesystemCacheEvictedBytes', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheEvictedFileSegments': { - 'name': 'events.FilesystemCacheEvictedFileSegments', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheEvictedFileSegmentsDuringPriorityIncrease': { - 'name': 'events.FilesystemCacheEvictedFileSegmentsDuringPriorityIncrease', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheEvictionReusedIterator': { - 'name': 'events.FilesystemCacheEvictionReusedIterator', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheEvictionSkippedEvictingFileSegments': { - 'name': 'events.FilesystemCacheEvictionSkippedEvictingFileSegments', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheEvictionSkippedFileSegments': { - 'name': 'events.FilesystemCacheEvictionSkippedFileSegments', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheEvictionTries': { - 'name': 'events.FilesystemCacheEvictionTries', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheFailToReserveSpaceBecauseOfCacheResize': { - 'name': 'events.FilesystemCacheFailToReserveSpaceBecauseOfCacheResize', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheFailToReserveSpaceBecauseOfLockContention': { - 'name': 'events.FilesystemCacheFailToReserveSpaceBecauseOfLockContention', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheFailedEvictionCandidates': { - 'name': 'events.FilesystemCacheFailedEvictionCandidates', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheFreeSpaceKeepingThreadRun': { - 'name': 'events.FilesystemCacheFreeSpaceKeepingThreadRun', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheFreeSpaceKeepingThreadWorkMilliseconds': { - 'name': 'events.FilesystemCacheFreeSpaceKeepingThreadWorkMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'FilesystemCacheGetMicroseconds': { - 'name': 'events.FilesystemCacheGetMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheGetOrSetMicroseconds': { - 'name': 'events.FilesystemCacheGetOrSetMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheHoldFileSegments': { - 'name': 'events.FilesystemCacheHoldFileSegments', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheLoadMetadataMicroseconds': { - 'name': 'events.FilesystemCacheLoadMetadataMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheLockCacheMicroseconds': { - 'name': 'events.FilesystemCacheLockCacheMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheLockKeyMicroseconds': { - 'name': 'events.FilesystemCacheLockKeyMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheLockMetadataMicroseconds': { - 'name': 'events.FilesystemCacheLockMetadataMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheReserveAttempts': { - 'name': 'events.FilesystemCacheReserveAttempts', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheReserveMicroseconds': { - 'name': 'events.FilesystemCacheReserveMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheUnusedHoldFileSegments': { - 'name': 'events.FilesystemCacheUnusedHoldFileSegments', - 'type': 'monotonic_gauge', - }, - 'FilterTransformPassedBytes': {'name': 'events.FilterTransformPassedBytes', 'type': 'monotonic_gauge'}, - 'FilterTransformPassedRows': {'name': 'events.FilterTransformPassedRows', 'type': 'monotonic_gauge'}, - 'FilteringMarksWithPrimaryKeyMicroseconds': { - 'name': 'events.FilteringMarksWithPrimaryKeyMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilteringMarksWithSecondaryKeysMicroseconds': { - 'name': 'events.FilteringMarksWithSecondaryKeysMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FunctionExecute': {'name': 'events.FunctionExecute', 'type': 'monotonic_gauge'}, - 'GWPAsanAllocateFailed': {'name': 'events.GWPAsanAllocateFailed', 'type': 'monotonic_gauge'}, - 'GWPAsanAllocateSuccess': {'name': 'events.GWPAsanAllocateSuccess', 'type': 'monotonic_gauge'}, - 'GWPAsanFree': {'name': 'events.GWPAsanFree', 'type': 'monotonic_gauge'}, - 'GatheredColumns': {'name': 'events.GatheredColumns', 'type': 'monotonic_gauge'}, - 'GatheringColumnMilliseconds': { - 'name': 'events.GatheringColumnMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'GlobalThreadPoolExpansions': {'name': 'events.GlobalThreadPoolExpansions', 'type': 'monotonic_gauge'}, - 'GlobalThreadPoolJobWaitTimeMicroseconds': { - 'name': 'events.GlobalThreadPoolJobWaitTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'GlobalThreadPoolJobs': {'name': 'events.GlobalThreadPoolJobs', 'type': 'monotonic_gauge'}, - 'GlobalThreadPoolLockWaitMicroseconds': { - 'name': 'events.GlobalThreadPoolLockWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'GlobalThreadPoolShrinks': {'name': 'events.GlobalThreadPoolShrinks', 'type': 'monotonic_gauge'}, - 'GlobalThreadPoolThreadCreationMicroseconds': { - 'name': 'events.GlobalThreadPoolThreadCreationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'HTTPConnectionsCreated': {'name': 'events.HTTPConnectionsCreated', 'type': 'monotonic_gauge'}, - 'HTTPConnectionsElapsedMicroseconds': { - 'name': 'events.HTTPConnectionsElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'HTTPConnectionsErrors': {'name': 'events.HTTPConnectionsErrors', 'type': 'monotonic_gauge'}, - 'HTTPConnectionsExpired': {'name': 'events.HTTPConnectionsExpired', 'type': 'monotonic_gauge'}, - 'HTTPConnectionsPreserved': {'name': 'events.HTTPConnectionsPreserved', 'type': 'monotonic_gauge'}, - 'HTTPConnectionsReset': {'name': 'events.HTTPConnectionsReset', 'type': 'monotonic_gauge'}, - 'HTTPConnectionsReused': {'name': 'events.HTTPConnectionsReused', 'type': 'monotonic_gauge'}, - 'HTTPServerConnectionsClosed': { - 'name': 'events.HTTPServerConnectionsClosed', - 'type': 'monotonic_gauge', - }, - 'HTTPServerConnectionsCreated': { - 'name': 'events.HTTPServerConnectionsCreated', - 'type': 'monotonic_gauge', - }, - 'HTTPServerConnectionsExpired': { - 'name': 'events.HTTPServerConnectionsExpired', - 'type': 'monotonic_gauge', - }, - 'HTTPServerConnectionsPreserved': { - 'name': 'events.HTTPServerConnectionsPreserved', - 'type': 'monotonic_gauge', - }, - 'HTTPServerConnectionsReset': {'name': 'events.HTTPServerConnectionsReset', 'type': 'monotonic_gauge'}, - 'HTTPServerConnectionsReused': { - 'name': 'events.HTTPServerConnectionsReused', - 'type': 'monotonic_gauge', - }, - 'HardPageFaults': {'name': 'events.HardPageFaults', 'type': 'monotonic_gauge'}, - 'HashJoinPreallocatedElementsInHashTables': { - 'name': 'events.HashJoinPreallocatedElementsInHashTables', - 'type': 'monotonic_gauge', - }, - 'HedgedRequestsChangeReplica': { - 'name': 'events.HedgedRequestsChangeReplica', - 'type': 'monotonic_gauge', - }, - 'IOBufferAllocBytes': {'name': 'events.IOBufferAllocBytes', 'type': 'monotonic_gauge'}, - 'IOBufferAllocs': {'name': 'events.IOBufferAllocs', 'type': 'monotonic_gauge'}, - 'IOUringCQEsCompleted': {'name': 'events.IOUringCQEsCompleted', 'type': 'monotonic_gauge'}, - 'IOUringCQEsFailed': {'name': 'events.IOUringCQEsFailed', 'type': 'monotonic_gauge'}, - 'IOUringSQEsResubmitsAsync': {'name': 'events.IOUringSQEsResubmitsAsync', 'type': 'monotonic_gauge'}, - 'IOUringSQEsResubmitsSync': {'name': 'events.IOUringSQEsResubmitsSync', 'type': 'monotonic_gauge'}, - 'IOUringSQEsSubmitted': {'name': 'events.IOUringSQEsSubmitted', 'type': 'monotonic_gauge'}, - 'IcebergIteratorInitializationMicroseconds': { - 'name': 'events.IcebergIteratorInitializationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'IcebergMetadataFilesCacheHits': { - 'name': 'events.IcebergMetadataFilesCacheHits', - 'type': 'monotonic_gauge', - }, - 'IcebergMetadataFilesCacheMisses': { - 'name': 'events.IcebergMetadataFilesCacheMisses', - 'type': 'monotonic_gauge', - }, - 'IcebergMetadataFilesCacheWeightLost': { - 'name': 'events.IcebergMetadataFilesCacheWeightLost', - 'type': 'monotonic_gauge', - }, - 'IcebergMetadataReadWaitTimeMicroseconds': { - 'name': 'events.IcebergMetadataReadWaitTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'IcebergMetadataReturnedObjectInfos': { - 'name': 'events.IcebergMetadataReturnedObjectInfos', - 'type': 'monotonic_gauge', - }, - 'IcebergMetadataUpdateMicroseconds': { - 'name': 'events.IcebergMetadataUpdateMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'IcebergMinMaxIndexPrunedFiles': { - 'name': 'events.IcebergMinMaxIndexPrunedFiles', - 'type': 'monotonic_gauge', - }, - 'IcebergPartitionPrunedFiles': { - 'name': 'events.IcebergPartitionPrunedFiles', - 'type': 'monotonic_gauge', - }, - 'IcebergPartitionPrunnedFiles': { - 'name': 'events.IcebergPartitionPrunnedFiles', - 'type': 'monotonic_gauge', - }, - 'IcebergTrivialCountOptimizationApplied': { - 'name': 'events.IcebergTrivialCountOptimizationApplied', - 'type': 'monotonic_gauge', - }, - 'IcebergVersionHintUsed': {'name': 'events.IcebergVersionHintUsed', 'type': 'monotonic_gauge'}, - 'IgnoredColdParts': {'name': 'events.IgnoredColdParts', 'type': 'monotonic_gauge'}, - 'IndexBinarySearchAlgorithm': {'name': 'events.IndexBinarySearchAlgorithm', 'type': 'monotonic_gauge'}, - 'IndexGenericExclusionSearchAlgorithm': { - 'name': 'events.IndexGenericExclusionSearchAlgorithm', - 'type': 'monotonic_gauge', - }, - 'InitialQuery': {'name': 'events.InitialQuery', 'type': 'monotonic_gauge'}, - 'InsertQueriesWithSubqueries': { - 'name': 'events.InsertQueriesWithSubqueries', - 'type': 'monotonic_gauge', - }, - 'InsertQuery': {'name': 'events.InsertQuery', 'type': 'monotonic_gauge'}, - 'InsertQueryTimeMicroseconds': { - 'name': 'events.InsertQueryTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'InsertedBytes': {'name': 'events.InsertedBytes', 'type': 'monotonic_gauge'}, - 'InsertedCompactParts': {'name': 'events.InsertedCompactParts', 'type': 'monotonic_gauge'}, - 'InsertedRows': {'name': 'events.InsertedRows', 'type': 'monotonic_gauge'}, - 'InsertedWideParts': {'name': 'events.InsertedWideParts', 'type': 'monotonic_gauge'}, - 'InterfaceHTTPReceiveBytes': {'name': 'events.InterfaceHTTPReceiveBytes', 'type': 'monotonic_gauge'}, - 'InterfaceHTTPSendBytes': {'name': 'events.InterfaceHTTPSendBytes', 'type': 'monotonic_gauge'}, - 'InterfaceInterserverReceiveBytes': { - 'name': 'events.InterfaceInterserverReceiveBytes', - 'type': 'monotonic_gauge', - }, - 'InterfaceInterserverSendBytes': { - 'name': 'events.InterfaceInterserverSendBytes', - 'type': 'monotonic_gauge', - }, - 'InterfaceMySQLReceiveBytes': {'name': 'events.InterfaceMySQLReceiveBytes', 'type': 'monotonic_gauge'}, - 'InterfaceMySQLSendBytes': {'name': 'events.InterfaceMySQLSendBytes', 'type': 'monotonic_gauge'}, - 'InterfaceNativeReceiveBytes': { - 'name': 'events.InterfaceNativeReceiveBytes', - 'type': 'monotonic_gauge', - }, - 'InterfaceNativeSendBytes': {'name': 'events.InterfaceNativeSendBytes', 'type': 'monotonic_gauge'}, - 'InterfacePostgreSQLReceiveBytes': { - 'name': 'events.InterfacePostgreSQLReceiveBytes', - 'type': 'monotonic_gauge', - }, - 'InterfacePostgreSQLSendBytes': { - 'name': 'events.InterfacePostgreSQLSendBytes', - 'type': 'monotonic_gauge', - }, - 'InterfacePrometheusReceiveBytes': { - 'name': 'events.InterfacePrometheusReceiveBytes', - 'type': 'monotonic_gauge', - }, - 'InterfacePrometheusSendBytes': { - 'name': 'events.InterfacePrometheusSendBytes', - 'type': 'monotonic_gauge', - }, - 'JoinBuildTableRowCount': {'name': 'events.JoinBuildTableRowCount', 'type': 'monotonic_gauge'}, - 'JoinProbeTableRowCount': {'name': 'events.JoinProbeTableRowCount', 'type': 'monotonic_gauge'}, - 'JoinResultRowCount': {'name': 'events.JoinResultRowCount', 'type': 'monotonic_gauge'}, - 'KafkaBackgroundReads': {'name': 'events.KafkaBackgroundReads', 'type': 'monotonic_gauge'}, - 'KafkaCommitFailures': {'name': 'events.KafkaCommitFailures', 'type': 'monotonic_gauge'}, - 'KafkaCommits': {'name': 'events.KafkaCommits', 'type': 'monotonic_gauge'}, - 'KafkaConsumerErrors': {'name': 'events.KafkaConsumerErrors', 'type': 'monotonic_gauge'}, - 'KafkaDirectReads': {'name': 'events.KafkaDirectReads', 'type': 'monotonic_gauge'}, - 'KafkaMessagesFailed': {'name': 'events.KafkaMessagesFailed', 'type': 'monotonic_gauge'}, - 'KafkaMessagesPolled': {'name': 'events.KafkaMessagesPolled', 'type': 'monotonic_gauge'}, - 'KafkaMessagesProduced': {'name': 'events.KafkaMessagesProduced', 'type': 'monotonic_gauge'}, - 'KafkaMessagesRead': {'name': 'events.KafkaMessagesRead', 'type': 'monotonic_gauge'}, - 'KafkaProducerErrors': {'name': 'events.KafkaProducerErrors', 'type': 'monotonic_gauge'}, - 'KafkaProducerFlushes': {'name': 'events.KafkaProducerFlushes', 'type': 'monotonic_gauge'}, - 'KafkaRebalanceAssignments': {'name': 'events.KafkaRebalanceAssignments', 'type': 'monotonic_gauge'}, - 'KafkaRebalanceErrors': {'name': 'events.KafkaRebalanceErrors', 'type': 'monotonic_gauge'}, - 'KafkaRebalanceRevocations': {'name': 'events.KafkaRebalanceRevocations', 'type': 'monotonic_gauge'}, - 'KafkaRowsRead': {'name': 'events.KafkaRowsRead', 'type': 'monotonic_gauge'}, - 'KafkaRowsRejected': {'name': 'events.KafkaRowsRejected', 'type': 'monotonic_gauge'}, - 'KafkaRowsWritten': {'name': 'events.KafkaRowsWritten', 'type': 'monotonic_gauge'}, - 'KafkaWrites': {'name': 'events.KafkaWrites', 'type': 'monotonic_gauge'}, - 'KeeperBatchMaxCount': {'name': 'events.KeeperBatchMaxCount', 'type': 'monotonic_gauge'}, - 'KeeperBatchMaxTotalSize': {'name': 'events.KeeperBatchMaxTotalSize', 'type': 'monotonic_gauge'}, - 'KeeperCheckRequest': {'name': 'events.KeeperCheckRequest', 'type': 'monotonic_gauge'}, - 'KeeperCommitWaitElapsedMicroseconds': { - 'name': 'events.KeeperCommitWaitElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'KeeperCommits': {'name': 'events.KeeperCommits', 'type': 'monotonic_gauge'}, - 'KeeperCommitsFailed': {'name': 'events.KeeperCommitsFailed', 'type': 'monotonic_gauge'}, - 'KeeperCreateRequest': {'name': 'events.KeeperCreateRequest', 'type': 'monotonic_gauge'}, - 'KeeperExistsRequest': {'name': 'events.KeeperExistsRequest', 'type': 'monotonic_gauge'}, - 'KeeperGetRequest': {'name': 'events.KeeperGetRequest', 'type': 'monotonic_gauge'}, - 'KeeperLatency': {'name': 'events.KeeperLatency', 'type': 'temporal_percent', 'scale': 'millisecond'}, - 'KeeperListRequest': {'name': 'events.KeeperListRequest', 'type': 'monotonic_gauge'}, - 'KeeperLogsEntryReadFromCommitCache': { - 'name': 'events.KeeperLogsEntryReadFromCommitCache', - 'type': 'monotonic_gauge', - }, - 'KeeperLogsEntryReadFromFile': { - 'name': 'events.KeeperLogsEntryReadFromFile', - 'type': 'monotonic_gauge', - }, - 'KeeperLogsEntryReadFromLatestCache': { - 'name': 'events.KeeperLogsEntryReadFromLatestCache', - 'type': 'monotonic_gauge', - }, - 'KeeperLogsPrefetchedEntries': { - 'name': 'events.KeeperLogsPrefetchedEntries', - 'type': 'monotonic_gauge', - }, - 'KeeperMultiReadRequest': {'name': 'events.KeeperMultiReadRequest', 'type': 'monotonic_gauge'}, - 'KeeperMultiRequest': {'name': 'events.KeeperMultiRequest', 'type': 'monotonic_gauge'}, - 'KeeperPacketsReceived': {'name': 'events.KeeperPacketsReceived', 'type': 'monotonic_gauge'}, - 'KeeperPacketsSent': {'name': 'events.KeeperPacketsSent', 'type': 'monotonic_gauge'}, - 'KeeperPreprocessElapsedMicroseconds': { - 'name': 'events.KeeperPreprocessElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'KeeperProcessElapsedMicroseconds': { - 'name': 'events.KeeperProcessElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'KeeperReadSnapshot': {'name': 'events.KeeperReadSnapshot', 'type': 'monotonic_gauge'}, - 'KeeperReconfigRequest': {'name': 'events.KeeperReconfigRequest', 'type': 'monotonic_gauge'}, - 'KeeperRemoveRequest': {'name': 'events.KeeperRemoveRequest', 'type': 'monotonic_gauge'}, - 'KeeperRequestRejectedDueToSoftMemoryLimitCount': { - 'name': 'events.KeeperRequestRejectedDueToSoftMemoryLimitCount', - 'type': 'monotonic_gauge', - }, - 'KeeperRequestTotal': {'name': 'events.KeeperRequestTotal', 'type': 'monotonic_gauge'}, - 'KeeperSaveSnapshot': {'name': 'events.KeeperSaveSnapshot', 'type': 'monotonic_gauge'}, - 'KeeperSetRequest': {'name': 'events.KeeperSetRequest', 'type': 'monotonic_gauge'}, - 'KeeperSnapshotApplys': {'name': 'events.KeeperSnapshotApplys', 'type': 'monotonic_gauge'}, - 'KeeperSnapshotApplysFailed': {'name': 'events.KeeperSnapshotApplysFailed', 'type': 'monotonic_gauge'}, - 'KeeperSnapshotCreations': {'name': 'events.KeeperSnapshotCreations', 'type': 'monotonic_gauge'}, - 'KeeperSnapshotCreationsFailed': { - 'name': 'events.KeeperSnapshotCreationsFailed', - 'type': 'monotonic_gauge', - }, - 'KeeperStorageLockWaitMicroseconds': { - 'name': 'events.KeeperStorageLockWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'KeeperTotalElapsedMicroseconds': { - 'name': 'events.KeeperTotalElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LoadedDataParts': {'name': 'events.LoadedDataParts', 'type': 'monotonic_gauge'}, - 'LoadedDataPartsMicroseconds': { - 'name': 'events.LoadedDataPartsMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LoadedMarksCount': {'name': 'events.LoadedMarksCount', 'type': 'monotonic_gauge'}, - 'LoadedMarksFiles': {'name': 'events.LoadedMarksFiles', 'type': 'monotonic_gauge'}, - 'LoadedMarksMemoryBytes': {'name': 'events.LoadedMarksMemoryBytes', 'type': 'monotonic_gauge'}, - 'LoadedPrimaryIndexBytes': {'name': 'events.LoadedPrimaryIndexBytes', 'type': 'monotonic_gauge'}, - 'LoadedPrimaryIndexFiles': {'name': 'events.LoadedPrimaryIndexFiles', 'type': 'monotonic_gauge'}, - 'LoadedPrimaryIndexRows': {'name': 'events.LoadedPrimaryIndexRows', 'type': 'monotonic_gauge'}, - 'LoadingMarksTasksCanceled': {'name': 'events.LoadingMarksTasksCanceled', 'type': 'monotonic_gauge'}, - 'LocalReadThrottlerBytes': {'name': 'events.LocalReadThrottlerBytes', 'type': 'monotonic_gauge'}, - 'LocalReadThrottlerSleepMicroseconds': { - 'name': 'events.LocalReadThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LocalThreadPoolBusyMicroseconds': { - 'name': 'events.LocalThreadPoolBusyMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LocalThreadPoolExpansions': {'name': 'events.LocalThreadPoolExpansions', 'type': 'monotonic_gauge'}, - 'LocalThreadPoolJobWaitTimeMicroseconds': { - 'name': 'events.LocalThreadPoolJobWaitTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LocalThreadPoolJobs': { - 'name': 'events.LocalThreadPoolJobs', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LocalThreadPoolLockWaitMicroseconds': { - 'name': 'events.LocalThreadPoolLockWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LocalThreadPoolShrinks': {'name': 'events.LocalThreadPoolShrinks', 'type': 'monotonic_gauge'}, - 'LocalThreadPoolThreadCreationMicroseconds': { - 'name': 'events.LocalThreadPoolThreadCreationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LocalWriteThrottlerBytes': {'name': 'events.LocalWriteThrottlerBytes', 'type': 'monotonic_gauge'}, - 'LocalWriteThrottlerSleepMicroseconds': { - 'name': 'events.LocalWriteThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LogDebug': {'name': 'events.LogDebug', 'type': 'monotonic_gauge'}, - 'LogError': {'name': 'events.LogError', 'type': 'monotonic_gauge'}, - 'LogFatal': {'name': 'events.LogFatal', 'type': 'monotonic_gauge'}, - 'LogInfo': {'name': 'events.LogInfo', 'type': 'monotonic_gauge'}, - 'LogTest': {'name': 'events.LogTest', 'type': 'monotonic_gauge'}, - 'LogTrace': {'name': 'events.LogTrace', 'type': 'monotonic_gauge'}, - 'LogWarning': {'name': 'events.LogWarning', 'type': 'monotonic_gauge'}, - 'LoggerElapsedNanoseconds': { - 'name': 'events.LoggerElapsedNanoseconds', - 'type': 'temporal_percent', - 'scale': 'nanosecond', - }, - 'MMappedFileCacheHits': {'name': 'events.MMappedFileCacheHits', 'type': 'monotonic_gauge'}, - 'MMappedFileCacheMisses': {'name': 'events.MMappedFileCacheMisses', 'type': 'monotonic_gauge'}, - 'MainConfigLoads': {'name': 'events.MainConfigLoads', 'type': 'monotonic_gauge'}, - 'MarkCacheEvictedBytes': {'name': 'events.MarkCacheEvictedBytes', 'type': 'monotonic_gauge'}, - 'MarkCacheEvictedFiles': {'name': 'events.MarkCacheEvictedFiles', 'type': 'monotonic_gauge'}, - 'MarkCacheEvictedMarks': {'name': 'events.MarkCacheEvictedMarks', 'type': 'monotonic_gauge'}, - 'MarkCacheHits': {'name': 'events.MarkCacheHits', 'type': 'monotonic_gauge'}, - 'MarkCacheMisses': {'name': 'events.MarkCacheMisses', 'type': 'monotonic_gauge'}, - 'MemoryAllocatorPurge': {'name': 'events.MemoryAllocatorPurge', 'type': 'monotonic_gauge'}, - 'MemoryAllocatorPurgeTimeMicroseconds': { - 'name': 'events.MemoryAllocatorPurgeTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MemoryOvercommitWaitTimeMicroseconds': { - 'name': 'events.MemoryOvercommitWaitTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MemoryWorkerRun': {'name': 'events.MemoryWorkerRun', 'type': 'monotonic_gauge'}, - 'MemoryWorkerRunElapsedMicroseconds': { - 'name': 'events.MemoryWorkerRunElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'Merge': {'name': 'events.Merge', 'type': 'monotonic_gauge'}, - 'MergeExecuteMilliseconds': { - 'name': 'events.MergeExecuteMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeHorizontalStageExecuteMilliseconds': { - 'name': 'events.MergeHorizontalStageExecuteMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeHorizontalStageTotalMilliseconds': { - 'name': 'events.MergeHorizontalStageTotalMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeMutateBackgroundExecutorTaskCancelMicroseconds': { - 'name': 'events.MergeMutateBackgroundExecutorTaskCancelMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeMutateBackgroundExecutorTaskExecuteStepMicroseconds': { - 'name': 'events.MergeMutateBackgroundExecutorTaskExecuteStepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeMutateBackgroundExecutorTaskResetMicroseconds': { - 'name': 'events.MergeMutateBackgroundExecutorTaskResetMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeMutateBackgroundExecutorWaitMicroseconds': { - 'name': 'events.MergeMutateBackgroundExecutorWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergePrewarmStageExecuteMilliseconds': { - 'name': 'events.MergePrewarmStageExecuteMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergePrewarmStageTotalMilliseconds': { - 'name': 'events.MergePrewarmStageTotalMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeProjectionStageExecuteMilliseconds': { - 'name': 'events.MergeProjectionStageExecuteMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeProjectionStageTotalMilliseconds': { - 'name': 'events.MergeProjectionStageTotalMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeSourceParts': {'name': 'events.MergeSourceParts', 'type': 'monotonic_gauge'}, - 'MergeTotalMilliseconds': { - 'name': 'events.MergeTotalMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeTreeAllRangesAnnouncementsSent': { - 'name': 'events.MergeTreeAllRangesAnnouncementsSent', - 'type': 'monotonic_gauge', - }, - 'MergeTreeAllRangesAnnouncementsSentElapsedMicroseconds': { - 'name': 'events.MergeTreeAllRangesAnnouncementsSentElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataProjectionWriterBlocks': { - 'name': 'events.MergeTreeDataProjectionWriterBlocks', - 'type': 'monotonic_gauge', - }, - 'MergeTreeDataProjectionWriterBlocksAlreadySorted': { - 'name': 'events.MergeTreeDataProjectionWriterBlocksAlreadySorted', - 'type': 'monotonic_gauge', - }, - 'MergeTreeDataProjectionWriterCompressedBytes': { - 'name': 'events.MergeTreeDataProjectionWriterCompressedBytes', - 'type': 'monotonic_gauge', - }, - 'MergeTreeDataProjectionWriterMergingBlocksMicroseconds': { - 'name': 'events.MergeTreeDataProjectionWriterMergingBlocksMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataProjectionWriterRows': { - 'name': 'events.MergeTreeDataProjectionWriterRows', - 'type': 'monotonic_gauge', - }, - 'MergeTreeDataProjectionWriterSortingBlocksMicroseconds': { - 'name': 'events.MergeTreeDataProjectionWriterSortingBlocksMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataProjectionWriterUncompressedBytes': { - 'name': 'events.MergeTreeDataProjectionWriterUncompressedBytes', - 'type': 'monotonic_gauge', - }, - 'MergeTreeDataWriterBlocks': {'name': 'events.MergeTreeDataWriterBlocks', 'type': 'monotonic_gauge'}, - 'MergeTreeDataWriterBlocksAlreadySorted': { - 'name': 'events.MergeTreeDataWriterBlocksAlreadySorted', - 'type': 'monotonic_gauge', - }, - 'MergeTreeDataWriterCompressedBytes': { - 'name': 'events.MergeTreeDataWriterCompressedBytes', - 'type': 'monotonic_gauge', - }, - 'MergeTreeDataWriterMergingBlocksMicroseconds': { - 'name': 'events.MergeTreeDataWriterMergingBlocksMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataWriterProjectionsCalculationMicroseconds': { - 'name': 'events.MergeTreeDataWriterProjectionsCalculationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataWriterRows': {'name': 'events.MergeTreeDataWriterRows', 'type': 'monotonic_gauge'}, - 'MergeTreeDataWriterSkipIndicesCalculationMicroseconds': { - 'name': 'events.MergeTreeDataWriterSkipIndicesCalculationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataWriterSortingBlocksMicroseconds': { - 'name': 'events.MergeTreeDataWriterSortingBlocksMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataWriterStatisticsCalculationMicroseconds': { - 'name': 'events.MergeTreeDataWriterStatisticsCalculationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataWriterUncompressedBytes': { - 'name': 'events.MergeTreeDataWriterUncompressedBytes', - 'type': 'monotonic_gauge', - }, - 'MergeTreePrefetchedReadPoolInit': { - 'name': 'events.MergeTreePrefetchedReadPoolInit', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeReadTaskRequestsReceived': { - 'name': 'events.MergeTreeReadTaskRequestsReceived', - 'type': 'monotonic_gauge', - }, - 'MergeTreeReadTaskRequestsSent': { - 'name': 'events.MergeTreeReadTaskRequestsSent', - 'type': 'monotonic_gauge', - }, - 'MergeTreeReadTaskRequestsSentElapsedMicroseconds': { - 'name': 'events.MergeTreeReadTaskRequestsSentElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeVerticalStageExecuteMilliseconds': { - 'name': 'events.MergeVerticalStageExecuteMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeVerticalStageTotalMilliseconds': { - 'name': 'events.MergeVerticalStageTotalMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergedColumns': {'name': 'events.MergedColumns', 'type': 'monotonic_gauge'}, - 'MergedIntoCompactParts': {'name': 'events.MergedIntoCompactParts', 'type': 'monotonic_gauge'}, - 'MergedIntoWideParts': {'name': 'events.MergedIntoWideParts', 'type': 'monotonic_gauge'}, - 'MergedRows': {'name': 'events.MergedRows', 'type': 'monotonic_gauge'}, - 'MergedUncompressedBytes': {'name': 'events.MergedUncompressedBytes', 'type': 'monotonic_gauge'}, - 'MergerMutatorPartsInRangesForMergeCount': { - 'name': 'events.MergerMutatorPartsInRangesForMergeCount', - 'type': 'monotonic_gauge', - }, - 'MergerMutatorPrepareRangesForMergeElapsedMicroseconds': { - 'name': 'events.MergerMutatorPrepareRangesForMergeElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergerMutatorRangesForMergeCount': { - 'name': 'events.MergerMutatorRangesForMergeCount', - 'type': 'monotonic_gauge', - }, - 'MergerMutatorSelectPartsForMergeElapsedMicroseconds': { - 'name': 'events.MergerMutatorSelectPartsForMergeElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergerMutatorSelectRangePartsCount': { - 'name': 'events.MergerMutatorSelectRangePartsCount', - 'type': 'monotonic_gauge', - }, - 'MergerMutatorsGetPartsForMergeElapsedMicroseconds': { - 'name': 'events.MergerMutatorsGetPartsForMergeElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergesThrottlerBytes': {'name': 'events.MergesThrottlerBytes', 'type': 'monotonic_gauge'}, - 'MergesThrottlerSleepMicroseconds': { - 'name': 'events.MergesThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergingSortedMilliseconds': { - 'name': 'events.MergingSortedMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MetadataFromKeeperBackgroundCleanupErrors': { - 'name': 'events.MetadataFromKeeperBackgroundCleanupErrors', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperBackgroundCleanupObjects': { - 'name': 'events.MetadataFromKeeperBackgroundCleanupObjects', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperBackgroundCleanupTransactions': { - 'name': 'events.MetadataFromKeeperBackgroundCleanupTransactions', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperCacheHit': {'name': 'events.MetadataFromKeeperCacheHit', 'type': 'monotonic_gauge'}, - 'MetadataFromKeeperCacheMiss': { - 'name': 'events.MetadataFromKeeperCacheMiss', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperCacheUpdateMicroseconds': { - 'name': 'events.MetadataFromKeeperCacheUpdateMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MetadataFromKeeperCleanupTransactionCommit': { - 'name': 'events.MetadataFromKeeperCleanupTransactionCommit', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperCleanupTransactionCommitRetry': { - 'name': 'events.MetadataFromKeeperCleanupTransactionCommitRetry', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperIndividualOperations': { - 'name': 'events.MetadataFromKeeperIndividualOperations', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperIndividualOperationsMicroseconds': { - 'name': 'events.MetadataFromKeeperIndividualOperationsMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MetadataFromKeeperOperations': { - 'name': 'events.MetadataFromKeeperOperations', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperReconnects': { - 'name': 'events.MetadataFromKeeperReconnects', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperTransactionCommit': { - 'name': 'events.MetadataFromKeeperTransactionCommit', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperTransactionCommitRetry': { - 'name': 'events.MetadataFromKeeperTransactionCommitRetry', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperUpdateCacheOneLevel': { - 'name': 'events.MetadataFromKeeperUpdateCacheOneLevel', - 'type': 'monotonic_gauge', - }, - 'MoveBackgroundExecutorTaskCancelMicroseconds': { - 'name': 'events.MoveBackgroundExecutorTaskCancelMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MoveBackgroundExecutorTaskExecuteStepMicroseconds': { - 'name': 'events.MoveBackgroundExecutorTaskExecuteStepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MoveBackgroundExecutorTaskResetMicroseconds': { - 'name': 'events.MoveBackgroundExecutorTaskResetMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MoveBackgroundExecutorWaitMicroseconds': { - 'name': 'events.MoveBackgroundExecutorWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MutateTaskProjectionsCalculationMicroseconds': { - 'name': 'events.MutateTaskProjectionsCalculationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MutatedRows': {'name': 'events.MutatedRows', 'type': 'monotonic_gauge'}, - 'MutatedUncompressedBytes': {'name': 'events.MutatedUncompressedBytes', 'type': 'monotonic_gauge'}, - 'MutationAffectedRowsUpperBound': { - 'name': 'events.MutationAffectedRowsUpperBound', - 'type': 'monotonic_gauge', - }, - 'MutationAllPartColumns': {'name': 'events.MutationAllPartColumns', 'type': 'monotonic_gauge'}, - 'MutationCreatedEmptyParts': {'name': 'events.MutationCreatedEmptyParts', 'type': 'monotonic_gauge'}, - 'MutationExecuteMilliseconds': { - 'name': 'events.MutationExecuteMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MutationSomePartColumns': {'name': 'events.MutationSomePartColumns', 'type': 'monotonic_gauge'}, - 'MutationTotalMilliseconds': { - 'name': 'events.MutationTotalMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MutationTotalParts': {'name': 'events.MutationTotalParts', 'type': 'monotonic_gauge'}, - 'MutationUntouchedParts': {'name': 'events.MutationUntouchedParts', 'type': 'monotonic_gauge'}, - 'MutationsAppliedOnFlyInAllParts': {'name': 'events.MutationsAppliedOnFlyInAllParts', 'type': 'gauge'}, - 'MutationsAppliedOnFlyInAllReadTasks': { - 'name': 'events.MutationsAppliedOnFlyInAllReadTasks', - 'type': 'monotonic_gauge', - }, - 'MutationsThrottlerBytes': {'name': 'events.MutationsThrottlerBytes', 'type': 'monotonic_gauge'}, - 'MutationsThrottlerSleepMicroseconds': { - 'name': 'events.MutationsThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'NetworkReceiveBytes': {'name': 'events.NetworkReceiveBytes', 'type': 'monotonic_gauge'}, - 'NetworkReceiveElapsedMicroseconds': { - 'name': 'events.NetworkReceiveElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'NetworkSendBytes': {'name': 'events.NetworkSendBytes', 'type': 'monotonic_gauge'}, - 'NetworkSendElapsedMicroseconds': { - 'name': 'events.NetworkSendElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'NotCreatedLogEntryForMerge': {'name': 'events.NotCreatedLogEntryForMerge', 'type': 'monotonic_gauge'}, - 'NotCreatedLogEntryForMutation': { - 'name': 'events.NotCreatedLogEntryForMutation', - 'type': 'monotonic_gauge', - }, - 'OSCPUVirtualTimeMicroseconds': { - 'name': 'events.OSCPUVirtualTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'OSCPUWaitMicroseconds': { - 'name': 'events.OSCPUWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'OSIOWaitMicroseconds': { - 'name': 'events.OSIOWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'OSReadBytes': {'name': 'events.OSReadBytes', 'type': 'monotonic_gauge'}, - 'OSReadChars': {'name': 'events.OSReadChars', 'type': 'monotonic_gauge'}, - 'OSWriteBytes': {'name': 'events.OSWriteBytes', 'type': 'monotonic_gauge'}, - 'OSWriteChars': {'name': 'events.OSWriteChars', 'type': 'monotonic_gauge'}, - 'ObjectStorageQueueCancelledFiles': { - 'name': 'events.ObjectStorageQueueCancelledFiles', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueCleanupMaxSetSizeOrTTLMicroseconds': { - 'name': 'events.ObjectStorageQueueCleanupMaxSetSizeOrTTLMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ObjectStorageQueueCommitRequests': { - 'name': 'events.ObjectStorageQueueCommitRequests', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueExceptionsDuringInsert': { - 'name': 'events.ObjectStorageQueueExceptionsDuringInsert', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueExceptionsDuringRead': { - 'name': 'events.ObjectStorageQueueExceptionsDuringRead', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueFailedFiles': { - 'name': 'events.ObjectStorageQueueFailedFiles', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueFailedToBatchSetProcessing': { - 'name': 'events.ObjectStorageQueueFailedToBatchSetProcessing', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueFilteredFiles': { - 'name': 'events.ObjectStorageQueueFilteredFiles', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueInsertIterations': { - 'name': 'events.ObjectStorageQueueInsertIterations', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueListedFiles': { - 'name': 'events.ObjectStorageQueueListedFiles', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueLockLocalFileStatusesMicroseconds': { - 'name': 'events.ObjectStorageQueueLockLocalFileStatusesMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ObjectStorageQueueProcessedFiles': { - 'name': 'events.ObjectStorageQueueProcessedFiles', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueProcessedRows': { - 'name': 'events.ObjectStorageQueueProcessedRows', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueuePullMicroseconds': { - 'name': 'events.ObjectStorageQueuePullMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ObjectStorageQueueReadBytes': { - 'name': 'events.ObjectStorageQueueReadBytes', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueReadFiles': { - 'name': 'events.ObjectStorageQueueReadFiles', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueReadRows': {'name': 'events.ObjectStorageQueueReadRows', 'type': 'monotonic_gauge'}, - 'ObjectStorageQueueRemovedObjects': { - 'name': 'events.ObjectStorageQueueRemovedObjects', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueSuccessfulCommits': { - 'name': 'events.ObjectStorageQueueSuccessfulCommits', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueTrySetProcessingFailed': { - 'name': 'events.ObjectStorageQueueTrySetProcessingFailed', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueTrySetProcessingRequests': { - 'name': 'events.ObjectStorageQueueTrySetProcessingRequests', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueTrySetProcessingSucceeded': { - 'name': 'events.ObjectStorageQueueTrySetProcessingSucceeded', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueUnsuccessfulCommits': { - 'name': 'events.ObjectStorageQueueUnsuccessfulCommits', - 'type': 'monotonic_gauge', - }, - 'ObsoleteReplicatedParts': {'name': 'events.ObsoleteReplicatedParts', 'type': 'monotonic_gauge'}, - 'OpenedFileCacheHits': {'name': 'events.OpenedFileCacheHits', 'type': 'monotonic_gauge'}, - 'OpenedFileCacheMicroseconds': { - 'name': 'events.OpenedFileCacheMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'OpenedFileCacheMisses': {'name': 'events.OpenedFileCacheMisses', 'type': 'monotonic_gauge'}, - 'OtherQueryTimeMicroseconds': { - 'name': 'events.OtherQueryTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'OverflowAny': {'name': 'events.OverflowAny', 'type': 'monotonic_gauge'}, - 'OverflowBreak': {'name': 'events.OverflowBreak', 'type': 'monotonic_gauge'}, - 'OverflowThrow': {'name': 'events.OverflowThrow', 'type': 'monotonic_gauge'}, - 'PageCacheBytesUnpinnedRoundedToHugePages': { - 'name': 'events.PageCacheBytesUnpinnedRoundedToHugePages', - 'type': 'gauge', - }, - 'PageCacheBytesUnpinnedRoundedToPages': { - 'name': 'events.PageCacheBytesUnpinnedRoundedToPages', - 'type': 'gauge', - }, - 'PageCacheChunkDataHits': {'name': 'events.PageCacheChunkDataHits', 'type': 'gauge'}, - 'PageCacheChunkDataMisses': {'name': 'events.PageCacheChunkDataMisses', 'type': 'gauge'}, - 'PageCacheChunkDataPartialHits': {'name': 'events.PageCacheChunkDataPartialHits', 'type': 'gauge'}, - 'PageCacheChunkMisses': {'name': 'events.PageCacheChunkMisses', 'type': 'gauge'}, - 'PageCacheChunkShared': {'name': 'events.PageCacheChunkShared', 'type': 'gauge'}, - 'PageCacheHits': {'name': 'events.PageCacheHits', 'type': 'monotonic_gauge'}, - 'PageCacheMisses': {'name': 'events.PageCacheMisses', 'type': 'monotonic_gauge'}, - 'PageCacheOvercommitResize': {'name': 'events.PageCacheOvercommitResize', 'type': 'monotonic_gauge'}, - 'PageCacheReadBytes': {'name': 'events.PageCacheReadBytes', 'type': 'monotonic_gauge'}, - 'PageCacheResized': {'name': 'events.PageCacheResized', 'type': 'monotonic_gauge'}, - 'PageCacheWeightLost': {'name': 'events.PageCacheWeightLost', 'type': 'monotonic_gauge'}, - 'ParallelReplicasAnnouncementMicroseconds': { - 'name': 'events.ParallelReplicasAnnouncementMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasAvailableCount': { - 'name': 'events.ParallelReplicasAvailableCount', - 'type': 'monotonic_gauge', - }, - 'ParallelReplicasCollectingOwnedSegmentsMicroseconds': { - 'name': 'events.ParallelReplicasCollectingOwnedSegmentsMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasDeniedRequests': { - 'name': 'events.ParallelReplicasDeniedRequests', - 'type': 'monotonic_gauge', - }, - 'ParallelReplicasHandleAnnouncementMicroseconds': { - 'name': 'events.ParallelReplicasHandleAnnouncementMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasHandleRequestMicroseconds': { - 'name': 'events.ParallelReplicasHandleRequestMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasNumRequests': { - 'name': 'events.ParallelReplicasNumRequests', - 'type': 'monotonic_gauge', - }, - 'ParallelReplicasProcessingPartsMicroseconds': { - 'name': 'events.ParallelReplicasProcessingPartsMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasQueryCount': {'name': 'events.ParallelReplicasQueryCount', 'type': 'monotonic_gauge'}, - 'ParallelReplicasReadAssignedForStealingMarks': { - 'name': 'events.ParallelReplicasReadAssignedForStealingMarks', - 'type': 'monotonic_gauge', - }, - 'ParallelReplicasReadAssignedMarks': { - 'name': 'events.ParallelReplicasReadAssignedMarks', - 'type': 'monotonic_gauge', - }, - 'ParallelReplicasReadMarks': {'name': 'events.ParallelReplicasReadMarks', 'type': 'monotonic_gauge'}, - 'ParallelReplicasReadRequestMicroseconds': { - 'name': 'events.ParallelReplicasReadRequestMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasReadUnassignedMarks': { - 'name': 'events.ParallelReplicasReadUnassignedMarks', - 'type': 'monotonic_gauge', - }, - 'ParallelReplicasStealingByHashMicroseconds': { - 'name': 'events.ParallelReplicasStealingByHashMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasStealingLeftoversMicroseconds': { - 'name': 'events.ParallelReplicasStealingLeftoversMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasUnavailableCount': { - 'name': 'events.ParallelReplicasUnavailableCount', - 'type': 'monotonic_gauge', - }, - 'ParallelReplicasUsedCount': {'name': 'events.ParallelReplicasUsedCount', 'type': 'monotonic_gauge'}, - 'ParquetDecodingTaskBatches': {'name': 'events.ParquetDecodingTaskBatches', 'type': 'monotonic_gauge'}, - 'ParquetDecodingTasks': {'name': 'events.ParquetDecodingTasks', 'type': 'monotonic_gauge'}, - 'ParquetFetchWaitTimeMicroseconds': { - 'name': 'events.ParquetFetchWaitTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParquetPrunedRowGroups': {'name': 'events.ParquetPrunedRowGroups', 'type': 'monotonic_gauge'}, - 'ParquetReadRowGroups': {'name': 'events.ParquetReadRowGroups', 'type': 'monotonic_gauge'}, - 'PartsLockHoldMicroseconds': { - 'name': 'events.PartsLockHoldMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'PartsLockWaitMicroseconds': { - 'name': 'events.PartsLockWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'PartsWithAppliedMutationsOnFly': {'name': 'events.PartsWithAppliedMutationsOnFly', 'type': 'gauge'}, - 'PatchesAcquireLockMicroseconds': { - 'name': 'events.PatchesAcquireLockMicroseconds', - 'type': 'monotonic_gauge', - }, - 'PatchesAcquireLockTries': {'name': 'events.PatchesAcquireLockTries', 'type': 'monotonic_gauge'}, - 'PatchesAppliedInAllReadTasks': { - 'name': 'events.PatchesAppliedInAllReadTasks', - 'type': 'monotonic_gauge', - }, - 'PatchesJoinAppliedInAllReadTasks': { - 'name': 'events.PatchesJoinAppliedInAllReadTasks', - 'type': 'monotonic_gauge', - }, - 'PatchesMergeAppliedInAllReadTasks': { - 'name': 'events.PatchesMergeAppliedInAllReadTasks', - 'type': 'monotonic_gauge', - }, - 'PatchesReadUncompressedBytes': { - 'name': 'events.PatchesReadUncompressedBytes', - 'type': 'monotonic_gauge', - }, - 'PerfAlignmentFaults': {'name': 'events.PerfAlignmentFaults', 'type': 'monotonic_gauge'}, - 'PerfBranchInstructions': {'name': 'events.PerfBranchInstructions', 'type': 'monotonic_gauge'}, - 'PerfBranchMisses': {'name': 'events.PerfBranchMisses', 'type': 'monotonic_gauge'}, - 'PerfBusCycles': {'name': 'events.PerfBusCycles', 'type': 'monotonic_gauge'}, - 'PerfCPUClock': {'name': 'events.PerfCPUClock', 'type': 'monotonic_gauge'}, - 'PerfCPUCycles': {'name': 'events.PerfCPUCycles', 'type': 'monotonic_gauge'}, - 'PerfCPUMigrations': {'name': 'events.PerfCPUMigrations', 'type': 'monotonic_gauge'}, - 'PerfCacheMisses': {'name': 'events.PerfCacheMisses', 'type': 'monotonic_gauge'}, - 'PerfCacheReferences': {'name': 'events.PerfCacheReferences', 'type': 'monotonic_gauge'}, - 'PerfContextSwitches': {'name': 'events.PerfContextSwitches', 'type': 'monotonic_gauge'}, - 'PerfDataTLBMisses': {'name': 'events.PerfDataTLBMisses', 'type': 'monotonic_gauge'}, - 'PerfDataTLBReferences': {'name': 'events.PerfDataTLBReferences', 'type': 'monotonic_gauge'}, - 'PerfEmulationFaults': {'name': 'events.PerfEmulationFaults', 'type': 'monotonic_gauge'}, - 'PerfInstructionTLBMisses': {'name': 'events.PerfInstructionTLBMisses', 'type': 'monotonic_gauge'}, - 'PerfInstructionTLBReferences': { - 'name': 'events.PerfInstructionTLBReferences', - 'type': 'monotonic_gauge', - }, - 'PerfInstructions': {'name': 'events.PerfInstructions', 'type': 'monotonic_gauge'}, - 'PerfLocalMemoryMisses': {'name': 'events.PerfLocalMemoryMisses', 'type': 'monotonic_gauge'}, - 'PerfLocalMemoryReferences': {'name': 'events.PerfLocalMemoryReferences', 'type': 'monotonic_gauge'}, - 'PerfMinEnabledRunningTime': {'name': 'events.PerfMinEnabledRunningTime', 'type': 'monotonic_gauge'}, - 'PerfMinEnabledTime': {'name': 'events.PerfMinEnabledTime', 'type': 'monotonic_gauge'}, - 'PerfRefCPUCycles': {'name': 'events.PerfRefCPUCycles', 'type': 'monotonic_gauge'}, - 'PerfStalledCyclesBackend': {'name': 'events.PerfStalledCyclesBackend', 'type': 'monotonic_gauge'}, - 'PerfStalledCyclesFrontend': {'name': 'events.PerfStalledCyclesFrontend', 'type': 'monotonic_gauge'}, - 'PerfTaskClock': {'name': 'events.PerfTaskClock', 'type': 'monotonic_gauge'}, - 'PolygonsAddedToPool': {'name': 'events.PolygonsAddedToPool', 'type': 'monotonic_gauge'}, - 'PolygonsInPoolAllocatedBytes': { - 'name': 'events.PolygonsInPoolAllocatedBytes', - 'type': 'monotonic_gauge', - }, - 'PreferredWarmedUnmergedParts': { - 'name': 'events.PreferredWarmedUnmergedParts', - 'type': 'monotonic_gauge', - }, - 'PrimaryIndexCacheHits': {'name': 'events.PrimaryIndexCacheHits', 'type': 'monotonic_gauge'}, - 'PrimaryIndexCacheMisses': {'name': 'events.PrimaryIndexCacheMisses', 'type': 'monotonic_gauge'}, - 'QueriesWithSubqueries': {'name': 'events.QueriesWithSubqueries', 'type': 'monotonic_gauge'}, - 'Query': {'name': 'events.Query', 'type': 'monotonic_gauge'}, - 'QueryBackupThrottlerBytes': {'name': 'events.QueryBackupThrottlerBytes', 'type': 'monotonic_gauge'}, - 'QueryBackupThrottlerSleepMicroseconds': { - 'name': 'events.QueryBackupThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'QueryCacheHits': {'name': 'events.QueryCacheHits', 'type': 'monotonic_gauge'}, - 'QueryCacheMisses': {'name': 'events.QueryCacheMisses', 'type': 'monotonic_gauge'}, - 'QueryConditionCacheHits': {'name': 'events.QueryConditionCacheHits', 'type': 'monotonic_gauge'}, - 'QueryConditionCacheMisses': {'name': 'events.QueryConditionCacheMisses', 'type': 'monotonic_gauge'}, - 'QueryLocalReadThrottlerBytes': { - 'name': 'events.QueryLocalReadThrottlerBytes', - 'type': 'monotonic_gauge', - }, - 'QueryLocalReadThrottlerSleepMicroseconds': { - 'name': 'events.QueryLocalReadThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'QueryLocalWriteThrottlerBytes': { - 'name': 'events.QueryLocalWriteThrottlerBytes', - 'type': 'monotonic_gauge', - }, - 'QueryLocalWriteThrottlerSleepMicroseconds': { - 'name': 'events.QueryLocalWriteThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'QueryMaskingRulesMatch': {'name': 'events.QueryMaskingRulesMatch', 'type': 'monotonic_gauge'}, - 'QueryMemoryLimitExceeded': {'name': 'events.QueryMemoryLimitExceeded', 'type': 'monotonic_gauge'}, - 'QueryPreempted': {'name': 'events.QueryPreempted', 'type': 'monotonic_gauge'}, - 'QueryProfilerConcurrencyOverruns': { - 'name': 'events.QueryProfilerConcurrencyOverruns', - 'type': 'monotonic_gauge', - }, - 'QueryProfilerErrors': {'name': 'events.QueryProfilerErrors', 'type': 'monotonic_gauge'}, - 'QueryProfilerRuns': {'name': 'events.QueryProfilerRuns', 'type': 'monotonic_gauge'}, - 'QueryProfilerSignalOverruns': { - 'name': 'events.QueryProfilerSignalOverruns', - 'type': 'monotonic_gauge', - }, - 'QueryRemoteReadThrottlerBytes': { - 'name': 'events.QueryRemoteReadThrottlerBytes', - 'type': 'monotonic_gauge', - }, - 'QueryRemoteReadThrottlerSleepMicroseconds': { - 'name': 'events.QueryRemoteReadThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'QueryRemoteWriteThrottlerBytes': { - 'name': 'events.QueryRemoteWriteThrottlerBytes', - 'type': 'monotonic_gauge', - }, - 'QueryRemoteWriteThrottlerSleepMicroseconds': { - 'name': 'events.QueryRemoteWriteThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'QueryTimeMicroseconds': { - 'name': 'events.QueryTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'RWLockAcquiredReadLocks': {'name': 'events.RWLockAcquiredReadLocks', 'type': 'monotonic_gauge'}, - 'RWLockAcquiredWriteLocks': {'name': 'events.RWLockAcquiredWriteLocks', 'type': 'monotonic_gauge'}, - 'RWLockReadersWaitMilliseconds': { - 'name': 'events.RWLockReadersWaitMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'RWLockWritersWaitMilliseconds': { - 'name': 'events.RWLockWritersWaitMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'ReadBackoff': {'name': 'events.ReadBackoff', 'type': 'monotonic_gauge'}, - 'ReadBufferFromAzureBytes': {'name': 'events.ReadBufferFromAzureBytes', 'type': 'monotonic_gauge'}, - 'ReadBufferFromAzureInitMicroseconds': { - 'name': 'events.ReadBufferFromAzureInitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ReadBufferFromAzureMicroseconds': { - 'name': 'events.ReadBufferFromAzureMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ReadBufferFromAzureRequestsErrors': { - 'name': 'events.ReadBufferFromAzureRequestsErrors', - 'type': 'monotonic_gauge', - }, - 'ReadBufferFromFileDescriptorRead': { - 'name': 'events.ReadBufferFromFileDescriptorRead', - 'type': 'monotonic_gauge', - }, - 'ReadBufferFromFileDescriptorReadBytes': { - 'name': 'events.ReadBufferFromFileDescriptorReadBytes', - 'type': 'monotonic_gauge', - }, - 'ReadBufferFromFileDescriptorReadFailed': { - 'name': 'events.ReadBufferFromFileDescriptorReadFailed', - 'type': 'monotonic_gauge', - }, - 'ReadBufferFromS3Bytes': {'name': 'events.ReadBufferFromS3Bytes', 'type': 'monotonic_gauge'}, - 'ReadBufferFromS3InitMicroseconds': { - 'name': 'events.ReadBufferFromS3InitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ReadBufferFromS3Microseconds': { - 'name': 'events.ReadBufferFromS3Microseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ReadBufferFromS3RequestsErrors': { - 'name': 'events.ReadBufferFromS3RequestsErrors', - 'type': 'monotonic_gauge', - }, - 'ReadBufferSeekCancelConnection': { - 'name': 'events.ReadBufferSeekCancelConnection', - 'type': 'monotonic_gauge', - }, - 'ReadCompressedBytes': {'name': 'events.ReadCompressedBytes', 'type': 'monotonic_gauge'}, - 'ReadPatchesMicroseconds': {'name': 'events.ReadPatchesMicroseconds', 'type': 'monotonic_gauge'}, - 'ReadTaskRequestsReceived': {'name': 'events.ReadTaskRequestsReceived', 'type': 'monotonic_gauge'}, - 'ReadTaskRequestsSent': {'name': 'events.ReadTaskRequestsSent', 'type': 'monotonic_gauge'}, - 'ReadTaskRequestsSentElapsedMicroseconds': { - 'name': 'events.ReadTaskRequestsSentElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ReadTasksWithAppliedMutationsOnFly': { - 'name': 'events.ReadTasksWithAppliedMutationsOnFly', - 'type': 'monotonic_gauge', - }, - 'ReadTasksWithAppliedPatches': { - 'name': 'events.ReadTasksWithAppliedPatches', - 'type': 'monotonic_gauge', - }, - 'ReadWriteBufferFromHTTPBytes': { - 'name': 'events.ReadWriteBufferFromHTTPBytes', - 'type': 'monotonic_gauge', - }, - 'ReadWriteBufferFromHTTPRequestsSent': { - 'name': 'events.ReadWriteBufferFromHTTPRequestsSent', - 'type': 'monotonic_gauge', - }, - 'RealTimeMicroseconds': { - 'name': 'events.RealTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'RefreshableViewLockTableRetry': { - 'name': 'events.RefreshableViewLockTableRetry', - 'type': 'monotonic_gauge', - }, - 'RefreshableViewRefreshFailed': { - 'name': 'events.RefreshableViewRefreshFailed', - 'type': 'monotonic_gauge', - }, - 'RefreshableViewRefreshSuccess': { - 'name': 'events.RefreshableViewRefreshSuccess', - 'type': 'monotonic_gauge', - }, - 'RefreshableViewSyncReplicaRetry': { - 'name': 'events.RefreshableViewSyncReplicaRetry', - 'type': 'monotonic_gauge', - }, - 'RefreshableViewSyncReplicaSuccess': { - 'name': 'events.RefreshableViewSyncReplicaSuccess', - 'type': 'monotonic_gauge', - }, - 'RegexpLocalCacheHit': {'name': 'events.RegexpLocalCacheHit', 'type': 'monotonic_gauge'}, - 'RegexpLocalCacheMiss': {'name': 'events.RegexpLocalCacheMiss', 'type': 'monotonic_gauge'}, - 'RegexpWithMultipleNeedlesCreated': { - 'name': 'events.RegexpWithMultipleNeedlesCreated', - 'type': 'monotonic_gauge', - }, - 'RegexpWithMultipleNeedlesGlobalCacheHit': { - 'name': 'events.RegexpWithMultipleNeedlesGlobalCacheHit', - 'type': 'monotonic_gauge', - }, - 'RegexpWithMultipleNeedlesGlobalCacheMiss': { - 'name': 'events.RegexpWithMultipleNeedlesGlobalCacheMiss', - 'type': 'monotonic_gauge', - }, - 'RejectedInserts': {'name': 'events.RejectedInserts', 'type': 'monotonic_gauge'}, - 'RejectedLightweightUpdates': {'name': 'events.RejectedLightweightUpdates', 'type': 'monotonic_gauge'}, - 'RejectedMutations': {'name': 'events.RejectedMutations', 'type': 'monotonic_gauge'}, - 'RemoteFSBuffers': {'name': 'events.RemoteFSBuffers', 'type': 'monotonic_gauge'}, - 'RemoteFSCancelledPrefetches': { - 'name': 'events.RemoteFSCancelledPrefetches', - 'type': 'monotonic_gauge', - }, - 'RemoteFSLazySeeks': {'name': 'events.RemoteFSLazySeeks', 'type': 'monotonic_gauge'}, - 'RemoteFSPrefetchedBytes': {'name': 'events.RemoteFSPrefetchedBytes', 'type': 'monotonic_gauge'}, - 'RemoteFSPrefetchedReads': {'name': 'events.RemoteFSPrefetchedReads', 'type': 'monotonic_gauge'}, - 'RemoteFSPrefetches': {'name': 'events.RemoteFSPrefetches', 'type': 'monotonic_gauge'}, - 'RemoteFSSeeks': {'name': 'events.RemoteFSSeeks', 'type': 'monotonic_gauge'}, - 'RemoteFSSeeksWithReset': {'name': 'events.RemoteFSSeeksWithReset', 'type': 'monotonic_gauge'}, - 'RemoteFSUnprefetchedBytes': {'name': 'events.RemoteFSUnprefetchedBytes', 'type': 'monotonic_gauge'}, - 'RemoteFSUnprefetchedReads': {'name': 'events.RemoteFSUnprefetchedReads', 'type': 'monotonic_gauge'}, - 'RemoteFSUnusedPrefetches': {'name': 'events.RemoteFSUnusedPrefetches', 'type': 'monotonic_gauge'}, - 'RemoteReadThrottlerBytes': {'name': 'events.RemoteReadThrottlerBytes', 'type': 'monotonic_gauge'}, - 'RemoteReadThrottlerSleepMicroseconds': { - 'name': 'events.RemoteReadThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'RemoteWriteThrottlerBytes': {'name': 'events.RemoteWriteThrottlerBytes', 'type': 'monotonic_gauge'}, - 'RemoteWriteThrottlerSleepMicroseconds': { - 'name': 'events.RemoteWriteThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ReplacingSortedMilliseconds': { - 'name': 'events.ReplacingSortedMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'ReplicaPartialShutdown': {'name': 'events.ReplicaPartialShutdown', 'type': 'monotonic_gauge'}, - 'ReplicatedCoveredPartsInZooKeeperOnStart': { - 'name': 'events.ReplicatedCoveredPartsInZooKeeperOnStart', - 'type': 'monotonic_gauge', - }, - 'ReplicatedDataLoss': {'name': 'events.ReplicatedDataLoss', 'type': 'monotonic_gauge'}, - 'ReplicatedPartChecks': {'name': 'events.ReplicatedPartChecks', 'type': 'monotonic_gauge'}, - 'ReplicatedPartChecksFailed': {'name': 'events.ReplicatedPartChecksFailed', 'type': 'monotonic_gauge'}, - 'ReplicatedPartFailedFetches': { - 'name': 'events.ReplicatedPartFailedFetches', - 'type': 'monotonic_gauge', - }, - 'ReplicatedPartFetches': {'name': 'events.ReplicatedPartFetches', 'type': 'monotonic_gauge'}, - 'ReplicatedPartFetchesOfMerged': { - 'name': 'events.ReplicatedPartFetchesOfMerged', - 'type': 'monotonic_gauge', - }, - 'ReplicatedPartMerges': {'name': 'events.ReplicatedPartMerges', 'type': 'monotonic_gauge'}, - 'ReplicatedPartMutations': {'name': 'events.ReplicatedPartMutations', 'type': 'monotonic_gauge'}, - 'RestorePartsSkippedBytes': {'name': 'events.RestorePartsSkippedBytes', 'type': 'monotonic_gauge'}, - 'RestorePartsSkippedFiles': {'name': 'events.RestorePartsSkippedFiles', 'type': 'monotonic_gauge'}, - 'RowsReadByMainReader': {'name': 'events.RowsReadByMainReader', 'type': 'monotonic_gauge'}, - 'RowsReadByPrewhereReaders': {'name': 'events.RowsReadByPrewhereReaders', 'type': 'monotonic_gauge'}, - 'S3AbortMultipartUpload': {'name': 'events.S3AbortMultipartUpload', 'type': 'monotonic_gauge'}, - 'S3Clients': {'name': 'events.S3Clients', 'type': 'monotonic_gauge'}, - 'S3CompleteMultipartUpload': {'name': 'events.S3CompleteMultipartUpload', 'type': 'monotonic_gauge'}, - 'S3CopyObject': {'name': 'events.S3CopyObject', 'type': 'monotonic_gauge'}, - 'S3CreateMultipartUpload': {'name': 'events.S3CreateMultipartUpload', 'type': 'monotonic_gauge'}, - 'S3DeleteObjects': {'name': 'events.S3DeleteObjects', 'type': 'monotonic_gauge'}, - 'S3GetObject': {'name': 'events.S3GetObject', 'type': 'monotonic_gauge'}, - 'S3GetObjectAttributes': {'name': 'events.S3GetObjectAttributes', 'type': 'monotonic_gauge'}, - 'S3GetRequestThrottlerCount': {'name': 'events.S3GetRequestThrottlerCount', 'type': 'monotonic_gauge'}, - 'S3GetRequestThrottlerSleepMicroseconds': { - 'name': 'events.S3GetRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'S3HeadObject': {'name': 'events.S3HeadObject', 'type': 'monotonic_gauge'}, - 'S3ListObjects': {'name': 'events.S3ListObjects', 'type': 'monotonic_gauge'}, - 'S3PutObject': {'name': 'events.S3PutObject', 'type': 'monotonic_gauge'}, - 'S3PutRequestThrottlerCount': {'name': 'events.S3PutRequestThrottlerCount', 'type': 'monotonic_gauge'}, - 'S3PutRequestThrottlerSleepMicroseconds': { - 'name': 'events.S3PutRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'S3QueueSetFileFailedMicroseconds': { - 'name': 'events.S3QueueSetFileFailedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'S3QueueSetFileProcessedMicroseconds': { - 'name': 'events.S3QueueSetFileProcessedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'S3QueueSetFileProcessingMicroseconds': { - 'name': 'events.S3QueueSetFileProcessingMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'S3ReadMicroseconds': { - 'name': 'events.S3ReadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'S3ReadRequestAttempts': {'name': 'events.S3ReadRequestAttempts', 'type': 'monotonic_gauge'}, - 'S3ReadRequestRetryableErrors': { - 'name': 'events.S3ReadRequestRetryableErrors', - 'type': 'monotonic_gauge', - }, - 'S3ReadRequestsCount': {'name': 'events.S3ReadRequestsCount', 'type': 'monotonic_gauge'}, - 'S3ReadRequestsErrors': {'name': 'events.S3ReadRequestsErrors', 'type': 'monotonic_gauge'}, - 'S3ReadRequestsRedirects': {'name': 'events.S3ReadRequestsRedirects', 'type': 'monotonic_gauge'}, - 'S3ReadRequestsThrottling': {'name': 'events.S3ReadRequestsThrottling', 'type': 'monotonic_gauge'}, - 'S3UploadPart': {'name': 'events.S3UploadPart', 'type': 'monotonic_gauge'}, - 'S3UploadPartCopy': {'name': 'events.S3UploadPartCopy', 'type': 'monotonic_gauge'}, - 'S3WriteMicroseconds': { - 'name': 'events.S3WriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'S3WriteRequestAttempts': {'name': 'events.S3WriteRequestAttempts', 'type': 'monotonic_gauge'}, - 'S3WriteRequestRetryableErrors': { - 'name': 'events.S3WriteRequestRetryableErrors', - 'type': 'monotonic_gauge', - }, - 'S3WriteRequestsCount': {'name': 'events.S3WriteRequestsCount', 'type': 'monotonic_gauge'}, - 'S3WriteRequestsErrors': {'name': 'events.S3WriteRequestsErrors', 'type': 'monotonic_gauge'}, - 'S3WriteRequestsRedirects': {'name': 'events.S3WriteRequestsRedirects', 'type': 'monotonic_gauge'}, - 'S3WriteRequestsThrottling': {'name': 'events.S3WriteRequestsThrottling', 'type': 'monotonic_gauge'}, - 'ScalarSubqueriesCacheMiss': {'name': 'events.ScalarSubqueriesCacheMiss', 'type': 'monotonic_gauge'}, - 'ScalarSubqueriesGlobalCacheHit': { - 'name': 'events.ScalarSubqueriesGlobalCacheHit', - 'type': 'monotonic_gauge', - }, - 'ScalarSubqueriesLocalCacheHit': { - 'name': 'events.ScalarSubqueriesLocalCacheHit', - 'type': 'monotonic_gauge', - }, - 'SchedulerIOReadBytes': {'name': 'events.SchedulerIOReadBytes', 'type': 'monotonic_gauge'}, - 'SchedulerIOReadRequests': {'name': 'events.SchedulerIOReadRequests', 'type': 'monotonic_gauge'}, - 'SchedulerIOReadWaitMicroseconds': { - 'name': 'events.SchedulerIOReadWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SchedulerIOWriteBytes': {'name': 'events.SchedulerIOWriteBytes', 'type': 'monotonic_gauge'}, - 'SchedulerIOWriteRequests': {'name': 'events.SchedulerIOWriteRequests', 'type': 'monotonic_gauge'}, - 'SchedulerIOWriteWaitMicroseconds': { - 'name': 'events.SchedulerIOWriteWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SchemaInferenceCacheEvictions': { - 'name': 'events.SchemaInferenceCacheEvictions', - 'type': 'monotonic_gauge', - }, - 'SchemaInferenceCacheHits': {'name': 'events.SchemaInferenceCacheHits', 'type': 'monotonic_gauge'}, - 'SchemaInferenceCacheInvalidations': { - 'name': 'events.SchemaInferenceCacheInvalidations', - 'type': 'monotonic_gauge', - }, - 'SchemaInferenceCacheMisses': {'name': 'events.SchemaInferenceCacheMisses', 'type': 'monotonic_gauge'}, - 'SchemaInferenceCacheNumRowsHits': { - 'name': 'events.SchemaInferenceCacheNumRowsHits', - 'type': 'monotonic_gauge', - }, - 'SchemaInferenceCacheNumRowsMisses': { - 'name': 'events.SchemaInferenceCacheNumRowsMisses', - 'type': 'monotonic_gauge', - }, - 'SchemaInferenceCacheSchemaHits': { - 'name': 'events.SchemaInferenceCacheSchemaHits', - 'type': 'monotonic_gauge', - }, - 'SchemaInferenceCacheSchemaMisses': { - 'name': 'events.SchemaInferenceCacheSchemaMisses', - 'type': 'monotonic_gauge', - }, - 'Seek': {'name': 'events.Seek', 'type': 'monotonic_gauge'}, - 'SelectQueriesWithPrimaryKeyUsage': { - 'name': 'events.SelectQueriesWithPrimaryKeyUsage', - 'type': 'monotonic_gauge', - }, - 'SelectQueriesWithSubqueries': { - 'name': 'events.SelectQueriesWithSubqueries', - 'type': 'monotonic_gauge', - }, - 'SelectQuery': {'name': 'events.SelectQuery', 'type': 'monotonic_gauge'}, - 'SelectQueryTimeMicroseconds': { - 'name': 'events.SelectQueryTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SelectedBytes': {'name': 'events.SelectedBytes', 'type': 'monotonic_gauge'}, - 'SelectedMarks': {'name': 'events.SelectedMarks', 'type': 'monotonic_gauge'}, - 'SelectedMarksTotal': {'name': 'events.SelectedMarksTotal', 'type': 'monotonic_gauge'}, - 'SelectedParts': {'name': 'events.SelectedParts', 'type': 'monotonic_gauge'}, - 'SelectedPartsTotal': {'name': 'events.SelectedPartsTotal', 'type': 'monotonic_gauge'}, - 'SelectedRanges': {'name': 'events.SelectedRanges', 'type': 'monotonic_gauge'}, - 'SelectedRows': {'name': 'events.SelectedRows', 'type': 'monotonic_gauge'}, - 'ServerStartupMilliseconds': { - 'name': 'events.ServerStartupMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'SharedDatabaseCatalogFailedToApplyState': { - 'name': 'events.SharedDatabaseCatalogFailedToApplyState', - 'type': 'monotonic_gauge', - }, - 'SharedDatabaseCatalogStateApplicationMicroseconds': { - 'name': 'events.SharedDatabaseCatalogStateApplicationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SharedMergeTreeCondemnedPartsKillRequest': { - 'name': 'events.SharedMergeTreeCondemnedPartsKillRequest', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeCondemnedPartsLockConfict': { - 'name': 'events.SharedMergeTreeCondemnedPartsLockConfict', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeCondemnedPartsRemoved': { - 'name': 'events.SharedMergeTreeCondemnedPartsRemoved', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeDataPartsFetchAttempt': { - 'name': 'events.SharedMergeTreeDataPartsFetchAttempt', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeDataPartsFetchFromPeer': { - 'name': 'events.SharedMergeTreeDataPartsFetchFromPeer', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeDataPartsFetchFromPeerMicroseconds': { - 'name': 'events.SharedMergeTreeDataPartsFetchFromPeerMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeDataPartsFetchFromS3': { - 'name': 'events.SharedMergeTreeDataPartsFetchFromS3', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeGetPartsBatchToLoadMicroseconds': { - 'name': 'events.SharedMergeTreeGetPartsBatchToLoadMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeHandleBlockingParts': { - 'name': 'events.SharedMergeTreeHandleBlockingParts', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeHandleBlockingPartsMicroseconds': { - 'name': 'events.SharedMergeTreeHandleBlockingPartsMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeHandleFetchPartsMicroseconds': { - 'name': 'events.SharedMergeTreeHandleFetchPartsMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeHandleOutdatedParts': { - 'name': 'events.SharedMergeTreeHandleOutdatedParts', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeHandleOutdatedPartsMicroseconds': { - 'name': 'events.SharedMergeTreeHandleOutdatedPartsMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeLoadChecksumAndIndexesMicroseconds': { - 'name': 'events.SharedMergeTreeLoadChecksumAndIndexesMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMergeMutationAssignmentAttempt': { - 'name': 'events.SharedMergeTreeMergeMutationAssignmentAttempt', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMergeMutationAssignmentFailedWithConflict': { - 'name': 'events.SharedMergeTreeMergeMutationAssignmentFailedWithConflict', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMergeMutationAssignmentFailedWithNothingToDo': { - 'name': 'events.SharedMergeTreeMergeMutationAssignmentFailedWithNothingToDo', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMergeMutationAssignmentSuccessful': { - 'name': 'events.SharedMergeTreeMergeMutationAssignmentSuccessful', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMergePartsMovedToCondemned': { - 'name': 'events.SharedMergeTreeMergePartsMovedToCondemned', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMergePartsMovedToOudated': { - 'name': 'events.SharedMergeTreeMergePartsMovedToOudated', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMergeSelectingTaskMicroseconds': { - 'name': 'events.SharedMergeTreeMergeSelectingTaskMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMetadataCacheHintLoadedFromCache': { - 'name': 'events.SharedMergeTreeMetadataCacheHintLoadedFromCache', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeOptimizeAsync': { - 'name': 'events.SharedMergeTreeOptimizeAsync', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeOptimizeSync': { - 'name': 'events.SharedMergeTreeOptimizeSync', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeOutdatedPartsConfirmationInvocations': { - 'name': 'events.SharedMergeTreeOutdatedPartsConfirmationInvocations', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeOutdatedPartsConfirmationRequest': { - 'name': 'events.SharedMergeTreeOutdatedPartsConfirmationRequest', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeOutdatedPartsHTTPRequest': { - 'name': 'events.SharedMergeTreeOutdatedPartsHTTPRequest', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeOutdatedPartsHTTPResponse': { - 'name': 'events.SharedMergeTreeOutdatedPartsHTTPResponse', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeScheduleDataProcessingJob': { - 'name': 'events.SharedMergeTreeScheduleDataProcessingJob', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeScheduleDataProcessingJobMicroseconds': { - 'name': 'events.SharedMergeTreeScheduleDataProcessingJobMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeScheduleDataProcessingJobNothingToScheduled': { - 'name': 'events.SharedMergeTreeScheduleDataProcessingJobNothingToScheduled', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeTryUpdateDiskMetadataCacheForPartMicroseconds': { - 'name': 'events.SharedMergeTreeTryUpdateDiskMetadataCacheForPartMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdateMicroseconds': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdateMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SharedMergeTreeVirtualPartsUpdates': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdates', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdatesByLeader': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesByLeader', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdatesForMergesOrStatus': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesForMergesOrStatus', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdatesFromPeer': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesFromPeer', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdatesFromPeerMicroseconds': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesFromPeerMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SharedMergeTreeVirtualPartsUpdatesFromZooKeeper': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesFromZooKeeper', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdatesFromZooKeeperMicroseconds': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesFromZooKeeperMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SharedMergeTreeVirtualPartsUpdatesLeaderFailedElection': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesLeaderFailedElection', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdatesLeaderSuccessfulElection': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesLeaderSuccessfulElection', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdatesPeerNotFound': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesPeerNotFound', - 'type': 'monotonic_gauge', - }, - 'SleepFunctionCalls': {'name': 'events.SleepFunctionCalls', 'type': 'monotonic_gauge'}, - 'SleepFunctionElapsedMicroseconds': { - 'name': 'events.SleepFunctionElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SleepFunctionMicroseconds': { - 'name': 'events.SleepFunctionMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SlowRead': {'name': 'events.SlowRead', 'type': 'monotonic_gauge'}, - 'SoftPageFaults': {'name': 'events.SoftPageFaults', 'type': 'monotonic_gauge'}, - 'StorageBufferErrorOnFlush': {'name': 'events.StorageBufferErrorOnFlush', 'type': 'monotonic_gauge'}, - 'StorageBufferFlush': {'name': 'events.StorageBufferFlush', 'type': 'monotonic_gauge'}, - 'StorageBufferLayerLockReadersWaitMilliseconds': { - 'name': 'events.StorageBufferLayerLockReadersWaitMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'StorageBufferLayerLockWritersWaitMilliseconds': { - 'name': 'events.StorageBufferLayerLockWritersWaitMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'StorageBufferPassedAllMinThresholds': { - 'name': 'events.StorageBufferPassedAllMinThresholds', - 'type': 'monotonic_gauge', - }, - 'StorageBufferPassedBytesFlushThreshold': { - 'name': 'events.StorageBufferPassedBytesFlushThreshold', - 'type': 'monotonic_gauge', - }, - 'StorageBufferPassedBytesMaxThreshold': { - 'name': 'events.StorageBufferPassedBytesMaxThreshold', - 'type': 'monotonic_gauge', - }, - 'StorageBufferPassedRowsFlushThreshold': { - 'name': 'events.StorageBufferPassedRowsFlushThreshold', - 'type': 'monotonic_gauge', - }, - 'StorageBufferPassedRowsMaxThreshold': { - 'name': 'events.StorageBufferPassedRowsMaxThreshold', - 'type': 'monotonic_gauge', - }, - 'StorageBufferPassedTimeFlushThreshold': { - 'name': 'events.StorageBufferPassedTimeFlushThreshold', - 'type': 'monotonic_gauge', - }, - 'StorageBufferPassedTimeMaxThreshold': { - 'name': 'events.StorageBufferPassedTimeMaxThreshold', - 'type': 'monotonic_gauge', - }, - 'StorageConnectionsCreated': {'name': 'events.StorageConnectionsCreated', 'type': 'monotonic_gauge'}, - 'StorageConnectionsElapsedMicroseconds': { - 'name': 'events.StorageConnectionsElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'StorageConnectionsErrors': {'name': 'events.StorageConnectionsErrors', 'type': 'monotonic_gauge'}, - 'StorageConnectionsExpired': {'name': 'events.StorageConnectionsExpired', 'type': 'monotonic_gauge'}, - 'StorageConnectionsPreserved': { - 'name': 'events.StorageConnectionsPreserved', - 'type': 'monotonic_gauge', - }, - 'StorageConnectionsReset': {'name': 'events.StorageConnectionsReset', 'type': 'monotonic_gauge'}, - 'StorageConnectionsReused': {'name': 'events.StorageConnectionsReused', 'type': 'monotonic_gauge'}, - 'SummingSortedMilliseconds': { - 'name': 'events.SummingSortedMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'SuspendSendingQueryToShard': {'name': 'events.SuspendSendingQueryToShard', 'type': 'monotonic_gauge'}, - 'SynchronousReadWaitMicroseconds': { - 'name': 'events.SynchronousReadWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SynchronousRemoteReadWaitMicroseconds': { - 'name': 'events.SynchronousRemoteReadWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SystemLogErrorOnFlush': {'name': 'events.SystemLogErrorOnFlush', 'type': 'monotonic_gauge'}, - 'SystemTimeMicroseconds': { - 'name': 'events.SystemTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'TableFunctionExecute': {'name': 'events.TableFunctionExecute', 'type': 'monotonic_gauge'}, - 'ThreadPoolReaderPageCacheHit': { - 'name': 'events.ThreadPoolReaderPageCacheHit', - 'type': 'monotonic_gauge', - }, - 'ThreadPoolReaderPageCacheHitBytes': { - 'name': 'events.ThreadPoolReaderPageCacheHitBytes', - 'type': 'monotonic_gauge', - }, - 'ThreadPoolReaderPageCacheHitElapsedMicroseconds': { - 'name': 'events.ThreadPoolReaderPageCacheHitElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ThreadPoolReaderPageCacheMiss': { - 'name': 'events.ThreadPoolReaderPageCacheMiss', - 'type': 'monotonic_gauge', - }, - 'ThreadPoolReaderPageCacheMissBytes': { - 'name': 'events.ThreadPoolReaderPageCacheMissBytes', - 'type': 'monotonic_gauge', - }, - 'ThreadPoolReaderPageCacheMissElapsedMicroseconds': { - 'name': 'events.ThreadPoolReaderPageCacheMissElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ThreadpoolReaderPrepareMicroseconds': { - 'name': 'events.ThreadpoolReaderPrepareMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ThreadpoolReaderReadBytes': {'name': 'events.ThreadpoolReaderReadBytes', 'type': 'monotonic_gauge'}, - 'ThreadpoolReaderSubmit': {'name': 'events.ThreadpoolReaderSubmit', 'type': 'monotonic_gauge'}, - 'ThreadpoolReaderSubmitLookupInCacheMicroseconds': { - 'name': 'events.ThreadpoolReaderSubmitLookupInCacheMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ThreadpoolReaderSubmitReadSynchronously': { - 'name': 'events.ThreadpoolReaderSubmitReadSynchronously', - 'type': 'monotonic_gauge', - }, - 'ThreadpoolReaderSubmitReadSynchronouslyBytes': { - 'name': 'events.ThreadpoolReaderSubmitReadSynchronouslyBytes', - 'type': 'monotonic_gauge', - }, - 'ThreadpoolReaderSubmitReadSynchronouslyMicroseconds': { - 'name': 'events.ThreadpoolReaderSubmitReadSynchronouslyMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ThreadpoolReaderTaskMicroseconds': { - 'name': 'events.ThreadpoolReaderTaskMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ThrottlerSleepMicroseconds': { - 'name': 'events.ThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'TinyS3Clients': {'name': 'events.TinyS3Clients', 'type': 'monotonic_gauge'}, - 'USearchAddComputedDistances': { - 'name': 'events.USearchAddComputedDistances', - 'type': 'monotonic_gauge', - }, - 'USearchAddCount': {'name': 'events.USearchAddCount', 'type': 'monotonic_gauge'}, - 'USearchAddVisitedMembers': {'name': 'events.USearchAddVisitedMembers', 'type': 'monotonic_gauge'}, - 'USearchSearchComputedDistances': { - 'name': 'events.USearchSearchComputedDistances', - 'type': 'monotonic_gauge', - }, - 'USearchSearchCount': {'name': 'events.USearchSearchCount', 'type': 'monotonic_gauge'}, - 'USearchSearchVisitedMembers': { - 'name': 'events.USearchSearchVisitedMembers', - 'type': 'monotonic_gauge', - }, - 'UncompressedCacheHits': {'name': 'events.UncompressedCacheHits', 'type': 'monotonic_gauge'}, - 'UncompressedCacheMisses': {'name': 'events.UncompressedCacheMisses', 'type': 'monotonic_gauge'}, - 'UncompressedCacheWeightLost': { - 'name': 'events.UncompressedCacheWeightLost', - 'type': 'monotonic_gauge', - }, - 'UserTimeMicroseconds': { - 'name': 'events.UserTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'VectorSimilarityIndexCacheHits': { - 'name': 'events.VectorSimilarityIndexCacheHits', - 'type': 'monotonic_gauge', - }, - 'VectorSimilarityIndexCacheMisses': { - 'name': 'events.VectorSimilarityIndexCacheMisses', - 'type': 'monotonic_gauge', - }, - 'VectorSimilarityIndexCacheWeightLost': { - 'name': 'events.VectorSimilarityIndexCacheWeightLost', - 'type': 'monotonic_gauge', - }, - 'VersionedCollapsingSortedMilliseconds': { - 'name': 'events.VersionedCollapsingSortedMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'WaitMarksLoadMicroseconds': { - 'name': 'events.WaitMarksLoadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'WaitPrefetchTaskMicroseconds': { - 'name': 'events.WaitPrefetchTaskMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'WriteBufferFromFileDescriptorWrite': { - 'name': 'events.WriteBufferFromFileDescriptorWrite', - 'type': 'monotonic_gauge', - }, - 'WriteBufferFromFileDescriptorWriteBytes': { - 'name': 'events.WriteBufferFromFileDescriptorWriteBytes', - 'type': 'monotonic_gauge', - }, - 'WriteBufferFromFileDescriptorWriteFailed': { - 'name': 'events.WriteBufferFromFileDescriptorWriteFailed', - 'type': 'monotonic_gauge', - }, - 'WriteBufferFromHTTPBytes': {'name': 'events.WriteBufferFromHTTPBytes', 'type': 'monotonic_gauge'}, - 'WriteBufferFromHTTPRequestsSent': { - 'name': 'events.WriteBufferFromHTTPRequestsSent', - 'type': 'monotonic_gauge', - }, - 'WriteBufferFromS3Bytes': {'name': 'events.WriteBufferFromS3Bytes', 'type': 'monotonic_gauge'}, - 'WriteBufferFromS3Microseconds': { - 'name': 'events.WriteBufferFromS3Microseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'WriteBufferFromS3RequestsErrors': { - 'name': 'events.WriteBufferFromS3RequestsErrors', - 'type': 'monotonic_gauge', - }, - 'WriteBufferFromS3WaitInflightLimitMicroseconds': { - 'name': 'events.WriteBufferFromS3WaitInflightLimitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ZooKeeperBytesReceived': {'name': 'events.ZooKeeperBytesReceived', 'type': 'monotonic_gauge'}, - 'ZooKeeperBytesSent': {'name': 'events.ZooKeeperBytesSent', 'type': 'monotonic_gauge'}, - 'ZooKeeperCheck': {'name': 'events.ZooKeeperCheck', 'type': 'monotonic_gauge'}, - 'ZooKeeperClose': {'name': 'events.ZooKeeperClose', 'type': 'monotonic_gauge'}, - 'ZooKeeperCreate': {'name': 'events.ZooKeeperCreate', 'type': 'monotonic_gauge'}, - 'ZooKeeperExists': {'name': 'events.ZooKeeperExists', 'type': 'monotonic_gauge'}, - 'ZooKeeperGet': {'name': 'events.ZooKeeperGet', 'type': 'monotonic_gauge'}, - 'ZooKeeperGetACL': {'name': 'events.ZooKeeperGetACL', 'type': 'monotonic_gauge'}, - 'ZooKeeperHardwareExceptions': { - 'name': 'events.ZooKeeperHardwareExceptions', - 'type': 'monotonic_gauge', - }, - 'ZooKeeperInit': {'name': 'events.ZooKeeperInit', 'type': 'monotonic_gauge'}, - 'ZooKeeperList': {'name': 'events.ZooKeeperList', 'type': 'monotonic_gauge'}, - 'ZooKeeperMulti': {'name': 'events.ZooKeeperMulti', 'type': 'monotonic_gauge'}, - 'ZooKeeperMultiRead': {'name': 'events.ZooKeeperMultiRead', 'type': 'monotonic_gauge'}, - 'ZooKeeperMultiWrite': {'name': 'events.ZooKeeperMultiWrite', 'type': 'monotonic_gauge'}, - 'ZooKeeperOtherExceptions': {'name': 'events.ZooKeeperOtherExceptions', 'type': 'monotonic_gauge'}, - 'ZooKeeperReconfig': {'name': 'events.ZooKeeperReconfig', 'type': 'monotonic_gauge'}, - 'ZooKeeperRemove': {'name': 'events.ZooKeeperRemove', 'type': 'monotonic_gauge'}, - 'ZooKeeperSet': {'name': 'events.ZooKeeperSet', 'type': 'monotonic_gauge'}, - 'ZooKeeperSync': {'name': 'events.ZooKeeperSync', 'type': 'monotonic_gauge'}, - 'ZooKeeperTransactions': {'name': 'events.ZooKeeperTransactions', 'type': 'monotonic_gauge'}, - 'ZooKeeperUserExceptions': {'name': 'events.ZooKeeperUserExceptions', 'type': 'monotonic_gauge'}, - 'ZooKeeperWaitMicroseconds': { - 'name': 'events.ZooKeeperWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ZooKeeperWatchResponse': {'name': 'events.ZooKeeperWatchResponse', 'type': 'monotonic_gauge'}, - }, - }, - ], -} diff --git a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_metrics.py b/clickhouse/datadog_checks/clickhouse/advanced_queries/system_metrics.py deleted file mode 100644 index e729780bcd24e..0000000000000 --- a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_metrics.py +++ /dev/null @@ -1,780 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# This file is autogenerated. -# To change this file you should edit scripts/templates/system_metrics.tpl and then run the following command: -# hatch run metrics:generate - -# https://clickhouse.com/docs/operations/system-tables/metrics -SystemMetrics = { - 'name': 'system_metrics', - 'query': 'SELECT value, metric FROM system.metrics', - 'columns': [ - {'name': 'metric_value', 'type': 'source'}, - { - 'name': 'metric_name', - 'type': 'match', - 'source': 'metric_value', - 'items': { - 'ActiveTimersInQueryProfiler': {'name': 'metrics.ActiveTimersInQueryProfiler', 'type': 'gauge'}, - 'AddressesActive': {'name': 'metrics.AddressesActive', 'type': 'gauge'}, - 'AddressesBanned': {'name': 'metrics.AddressesBanned', 'type': 'gauge'}, - 'AggregatorThreads': {'name': 'metrics.AggregatorThreads', 'type': 'gauge'}, - 'AggregatorThreadsActive': {'name': 'metrics.AggregatorThreadsActive', 'type': 'gauge'}, - 'AggregatorThreadsScheduled': {'name': 'metrics.AggregatorThreadsScheduled', 'type': 'gauge'}, - 'AsyncInsertCacheSize': {'name': 'metrics.AsyncInsertCacheSize', 'type': 'gauge'}, - 'AsynchronousInsertQueueBytes': {'name': 'metrics.AsynchronousInsertQueueBytes', 'type': 'gauge'}, - 'AsynchronousInsertQueueSize': {'name': 'metrics.AsynchronousInsertQueueSize', 'type': 'gauge'}, - 'AsynchronousInsertThreads': {'name': 'metrics.AsynchronousInsertThreads', 'type': 'gauge'}, - 'AsynchronousInsertThreadsActive': {'name': 'metrics.AsynchronousInsertThreadsActive', 'type': 'gauge'}, - 'AsynchronousInsertThreadsScheduled': { - 'name': 'metrics.AsynchronousInsertThreadsScheduled', - 'type': 'gauge', - }, - 'AsynchronousReadWait': {'name': 'metrics.AsynchronousReadWait', 'type': 'gauge'}, - 'AttachedDatabase': {'name': 'metrics.AttachedDatabase', 'type': 'gauge'}, - 'AttachedDictionary': {'name': 'metrics.AttachedDictionary', 'type': 'gauge'}, - 'AttachedReplicatedTable': {'name': 'metrics.AttachedReplicatedTable', 'type': 'gauge'}, - 'AttachedTable': {'name': 'metrics.AttachedTable', 'type': 'gauge'}, - 'AttachedView': {'name': 'metrics.AttachedView', 'type': 'gauge'}, - 'AvroSchemaCacheBytes': {'name': 'metrics.AvroSchemaCacheBytes', 'type': 'gauge'}, - 'AvroSchemaCacheCells': {'name': 'metrics.AvroSchemaCacheCells', 'type': 'gauge'}, - 'AvroSchemaRegistryCacheBytes': {'name': 'metrics.AvroSchemaRegistryCacheBytes', 'type': 'gauge'}, - 'AvroSchemaRegistryCacheCells': {'name': 'metrics.AvroSchemaRegistryCacheCells', 'type': 'gauge'}, - 'AzureRequests': {'name': 'metrics.AzureRequests', 'type': 'gauge'}, - 'BackgroundBufferFlushSchedulePoolSize': { - 'name': 'metrics.BackgroundBufferFlushSchedulePoolSize', - 'type': 'gauge', - }, - 'BackgroundBufferFlushSchedulePoolTask': { - 'name': 'metrics.BackgroundBufferFlushSchedulePoolTask', - 'type': 'gauge', - }, - 'BackgroundCommonPoolSize': {'name': 'metrics.BackgroundCommonPoolSize', 'type': 'gauge'}, - 'BackgroundCommonPoolTask': {'name': 'metrics.BackgroundCommonPoolTask', 'type': 'gauge'}, - 'BackgroundDistributedSchedulePoolSize': { - 'name': 'metrics.BackgroundDistributedSchedulePoolSize', - 'type': 'gauge', - }, - 'BackgroundDistributedSchedulePoolTask': { - 'name': 'metrics.BackgroundDistributedSchedulePoolTask', - 'type': 'gauge', - }, - 'BackgroundFetchesPoolSize': {'name': 'metrics.BackgroundFetchesPoolSize', 'type': 'gauge'}, - 'BackgroundFetchesPoolTask': {'name': 'metrics.BackgroundFetchesPoolTask', 'type': 'gauge'}, - 'BackgroundMergesAndMutationsPoolSize': { - 'name': 'metrics.BackgroundMergesAndMutationsPoolSize', - 'type': 'gauge', - }, - 'BackgroundMergesAndMutationsPoolTask': { - 'name': 'metrics.BackgroundMergesAndMutationsPoolTask', - 'type': 'gauge', - }, - 'BackgroundMessageBrokerSchedulePoolSize': { - 'name': 'metrics.BackgroundMessageBrokerSchedulePoolSize', - 'type': 'gauge', - }, - 'BackgroundMessageBrokerSchedulePoolTask': { - 'name': 'metrics.BackgroundMessageBrokerSchedulePoolTask', - 'type': 'gauge', - }, - 'BackgroundMovePoolSize': {'name': 'metrics.BackgroundMovePoolSize', 'type': 'gauge'}, - 'BackgroundMovePoolTask': {'name': 'metrics.BackgroundMovePoolTask', 'type': 'gauge'}, - 'BackgroundSchedulePoolSize': {'name': 'metrics.BackgroundSchedulePoolSize', 'type': 'gauge'}, - 'BackgroundSchedulePoolTask': {'name': 'metrics.BackgroundSchedulePoolTask', 'type': 'gauge'}, - 'BackupsIOThreads': {'name': 'metrics.BackupsIOThreads', 'type': 'gauge'}, - 'BackupsIOThreadsActive': {'name': 'metrics.BackupsIOThreadsActive', 'type': 'gauge'}, - 'BackupsIOThreadsScheduled': {'name': 'metrics.BackupsIOThreadsScheduled', 'type': 'gauge'}, - 'BackupsThreads': {'name': 'metrics.BackupsThreads', 'type': 'gauge'}, - 'BackupsThreadsActive': {'name': 'metrics.BackupsThreadsActive', 'type': 'gauge'}, - 'BackupsThreadsScheduled': {'name': 'metrics.BackupsThreadsScheduled', 'type': 'gauge'}, - 'BrokenDisks': {'name': 'metrics.BrokenDisks', 'type': 'gauge'}, - 'BrokenDistributedBytesToInsert': {'name': 'metrics.BrokenDistributedBytesToInsert', 'type': 'gauge'}, - 'BrokenDistributedFilesToInsert': {'name': 'metrics.BrokenDistributedFilesToInsert', 'type': 'gauge'}, - 'BuildVectorSimilarityIndexThreads': { - 'name': 'metrics.BuildVectorSimilarityIndexThreads', - 'type': 'gauge', - }, - 'BuildVectorSimilarityIndexThreadsActive': { - 'name': 'metrics.BuildVectorSimilarityIndexThreadsActive', - 'type': 'gauge', - }, - 'BuildVectorSimilarityIndexThreadsScheduled': { - 'name': 'metrics.BuildVectorSimilarityIndexThreadsScheduled', - 'type': 'gauge', - }, - 'CacheDetachedFileSegments': {'name': 'metrics.CacheDetachedFileSegments', 'type': 'gauge'}, - 'CacheDictionaryThreads': {'name': 'metrics.CacheDictionaryThreads', 'type': 'gauge'}, - 'CacheDictionaryThreadsActive': {'name': 'metrics.CacheDictionaryThreadsActive', 'type': 'gauge'}, - 'CacheDictionaryThreadsScheduled': {'name': 'metrics.CacheDictionaryThreadsScheduled', 'type': 'gauge'}, - 'CacheDictionaryUpdateQueueBatches': { - 'name': 'metrics.CacheDictionaryUpdateQueueBatches', - 'type': 'gauge', - }, - 'CacheDictionaryUpdateQueueKeys': {'name': 'metrics.CacheDictionaryUpdateQueueKeys', 'type': 'gauge'}, - 'CacheFileSegments': {'name': 'metrics.CacheFileSegments', 'type': 'gauge'}, - 'CacheWarmerBytesInProgress': {'name': 'metrics.CacheWarmerBytesInProgress', 'type': 'gauge'}, - 'CompiledExpressionCacheBytes': {'name': 'metrics.CompiledExpressionCacheBytes', 'type': 'gauge'}, - 'CompiledExpressionCacheCount': {'name': 'metrics.CompiledExpressionCacheCount', 'type': 'gauge'}, - 'Compressing': {'name': 'metrics.Compressing', 'type': 'gauge'}, - 'CompressionThread': {'name': 'metrics.CompressionThread', 'type': 'gauge'}, - 'CompressionThreadActive': {'name': 'metrics.CompressionThreadActive', 'type': 'gauge'}, - 'CompressionThreadScheduled': {'name': 'metrics.CompressionThreadScheduled', 'type': 'gauge'}, - 'ConcurrencyControlAcquired': {'name': 'metrics.ConcurrencyControlAcquired', 'type': 'gauge'}, - 'ConcurrencyControlAcquiredNonCompeting': { - 'name': 'metrics.ConcurrencyControlAcquiredNonCompeting', - 'type': 'gauge', - }, - 'ConcurrencyControlPreempted': {'name': 'metrics.ConcurrencyControlPreempted', 'type': 'gauge'}, - 'ConcurrencyControlScheduled': {'name': 'metrics.ConcurrencyControlScheduled', 'type': 'gauge'}, - 'ConcurrencyControlSoftLimit': {'name': 'metrics.ConcurrencyControlSoftLimit', 'type': 'gauge'}, - 'ConcurrentHashJoinPoolThreads': {'name': 'metrics.ConcurrentHashJoinPoolThreads', 'type': 'gauge'}, - 'ConcurrentHashJoinPoolThreadsActive': { - 'name': 'metrics.ConcurrentHashJoinPoolThreadsActive', - 'type': 'gauge', - }, - 'ConcurrentHashJoinPoolThreadsScheduled': { - 'name': 'metrics.ConcurrentHashJoinPoolThreadsScheduled', - 'type': 'gauge', - }, - 'ConcurrentQueryAcquired': {'name': 'metrics.ConcurrentQueryAcquired', 'type': 'gauge'}, - 'ConcurrentQueryScheduled': {'name': 'metrics.ConcurrentQueryScheduled', 'type': 'gauge'}, - 'ContextLockWait': {'name': 'metrics.ContextLockWait', 'type': 'gauge'}, - 'CoordinatedMergesCoordinatorAssignedMerges': { - 'name': 'metrics.CoordinatedMergesCoordinatorAssignedMerges', - 'type': 'gauge', - }, - 'CoordinatedMergesCoordinatorRunningMerges': { - 'name': 'metrics.CoordinatedMergesCoordinatorRunningMerges', - 'type': 'gauge', - }, - 'CoordinatedMergesWorkerAssignedMerges': { - 'name': 'metrics.CoordinatedMergesWorkerAssignedMerges', - 'type': 'gauge', - }, - 'CreatedTimersInQueryProfiler': {'name': 'metrics.CreatedTimersInQueryProfiler', 'type': 'gauge'}, - 'DDLWorkerThreads': {'name': 'metrics.DDLWorkerThreads', 'type': 'gauge'}, - 'DDLWorkerThreadsActive': {'name': 'metrics.DDLWorkerThreadsActive', 'type': 'gauge'}, - 'DDLWorkerThreadsScheduled': {'name': 'metrics.DDLWorkerThreadsScheduled', 'type': 'gauge'}, - 'DNSAddressesCacheBytes': {'name': 'metrics.DNSAddressesCacheBytes', 'type': 'gauge'}, - 'DNSAddressesCacheSize': {'name': 'metrics.DNSAddressesCacheSize', 'type': 'gauge'}, - 'DNSHostsCacheBytes': {'name': 'metrics.DNSHostsCacheBytes', 'type': 'gauge'}, - 'DNSHostsCacheSize': {'name': 'metrics.DNSHostsCacheSize', 'type': 'gauge'}, - 'DWARFReaderThreads': {'name': 'metrics.DWARFReaderThreads', 'type': 'gauge'}, - 'DWARFReaderThreadsActive': {'name': 'metrics.DWARFReaderThreadsActive', 'type': 'gauge'}, - 'DWARFReaderThreadsScheduled': {'name': 'metrics.DWARFReaderThreadsScheduled', 'type': 'gauge'}, - 'DatabaseBackupThreads': {'name': 'metrics.DatabaseBackupThreads', 'type': 'gauge'}, - 'DatabaseBackupThreadsActive': {'name': 'metrics.DatabaseBackupThreadsActive', 'type': 'gauge'}, - 'DatabaseBackupThreadsScheduled': {'name': 'metrics.DatabaseBackupThreadsScheduled', 'type': 'gauge'}, - 'DatabaseCatalogThreads': {'name': 'metrics.DatabaseCatalogThreads', 'type': 'gauge'}, - 'DatabaseCatalogThreadsActive': {'name': 'metrics.DatabaseCatalogThreadsActive', 'type': 'gauge'}, - 'DatabaseCatalogThreadsScheduled': {'name': 'metrics.DatabaseCatalogThreadsScheduled', 'type': 'gauge'}, - 'DatabaseOnDiskThreads': {'name': 'metrics.DatabaseOnDiskThreads', 'type': 'gauge'}, - 'DatabaseOnDiskThreadsActive': {'name': 'metrics.DatabaseOnDiskThreadsActive', 'type': 'gauge'}, - 'DatabaseOnDiskThreadsScheduled': {'name': 'metrics.DatabaseOnDiskThreadsScheduled', 'type': 'gauge'}, - 'DatabaseReplicatedCreateTablesThreads': { - 'name': 'metrics.DatabaseReplicatedCreateTablesThreads', - 'type': 'gauge', - }, - 'DatabaseReplicatedCreateTablesThreadsActive': { - 'name': 'metrics.DatabaseReplicatedCreateTablesThreadsActive', - 'type': 'gauge', - }, - 'DatabaseReplicatedCreateTablesThreadsScheduled': { - 'name': 'metrics.DatabaseReplicatedCreateTablesThreadsScheduled', - 'type': 'gauge', - }, - 'Decompressing': {'name': 'metrics.Decompressing', 'type': 'gauge'}, - 'DelayedInserts': {'name': 'metrics.DelayedInserts', 'type': 'gauge'}, - 'DestroyAggregatesThreads': {'name': 'metrics.DestroyAggregatesThreads', 'type': 'gauge'}, - 'DestroyAggregatesThreadsActive': {'name': 'metrics.DestroyAggregatesThreadsActive', 'type': 'gauge'}, - 'DestroyAggregatesThreadsScheduled': { - 'name': 'metrics.DestroyAggregatesThreadsScheduled', - 'type': 'gauge', - }, - 'DictCacheRequests': {'name': 'metrics.DictCacheRequests', 'type': 'gauge'}, - 'DiskConnectionsStored': {'name': 'metrics.DiskConnectionsStored', 'type': 'gauge'}, - 'DiskConnectionsTotal': {'name': 'metrics.DiskConnectionsTotal', 'type': 'gauge'}, - 'DiskObjectStorageAsyncThreads': {'name': 'metrics.DiskObjectStorageAsyncThreads', 'type': 'gauge'}, - 'DiskObjectStorageAsyncThreadsActive': { - 'name': 'metrics.DiskObjectStorageAsyncThreadsActive', - 'type': 'gauge', - }, - 'DiskPlainRewritableAzureDirectoryMapSize': { - 'name': 'metrics.DiskPlainRewritableAzureDirectoryMapSize', - 'type': 'gauge', - }, - 'DiskPlainRewritableAzureFileCount': { - 'name': 'metrics.DiskPlainRewritableAzureFileCount', - 'type': 'gauge', - }, - 'DiskPlainRewritableAzureUniqueFileNamesCount': { - 'name': 'metrics.DiskPlainRewritableAzureUniqueFileNamesCount', - 'type': 'gauge', - }, - 'DiskPlainRewritableLocalDirectoryMapSize': { - 'name': 'metrics.DiskPlainRewritableLocalDirectoryMapSize', - 'type': 'gauge', - }, - 'DiskPlainRewritableLocalFileCount': { - 'name': 'metrics.DiskPlainRewritableLocalFileCount', - 'type': 'gauge', - }, - 'DiskPlainRewritableLocalUniqueFileNamesCount': { - 'name': 'metrics.DiskPlainRewritableLocalUniqueFileNamesCount', - 'type': 'gauge', - }, - 'DiskPlainRewritableS3DirectoryMapSize': { - 'name': 'metrics.DiskPlainRewritableS3DirectoryMapSize', - 'type': 'gauge', - }, - 'DiskPlainRewritableS3FileCount': {'name': 'metrics.DiskPlainRewritableS3FileCount', 'type': 'gauge'}, - 'DiskPlainRewritableS3UniqueFileNamesCount': { - 'name': 'metrics.DiskPlainRewritableS3UniqueFileNamesCount', - 'type': 'gauge', - }, - 'DiskS3NoSuchKeyErrors': {'name': 'metrics.DiskS3NoSuchKeyErrors', 'type': 'gauge'}, - 'DiskSpaceReservedForMerge': {'name': 'metrics.DiskSpaceReservedForMerge', 'type': 'gauge'}, - 'DistrCacheAllocatedConnections': {'name': 'metrics.DistrCacheAllocatedConnections', 'type': 'gauge'}, - 'DistrCacheBorrowedConnections': {'name': 'metrics.DistrCacheBorrowedConnections', 'type': 'gauge'}, - 'DistrCacheOpenedConnections': {'name': 'metrics.DistrCacheOpenedConnections', 'type': 'gauge'}, - 'DistrCacheReadRequests': {'name': 'metrics.DistrCacheReadRequests', 'type': 'gauge'}, - 'DistrCacheRegisteredServers': {'name': 'metrics.DistrCacheRegisteredServers', 'type': 'gauge'}, - 'DistrCacheRegisteredServersCurrentAZ': { - 'name': 'metrics.DistrCacheRegisteredServersCurrentAZ', - 'type': 'gauge', - }, - 'DistrCacheServerConnections': {'name': 'metrics.DistrCacheServerConnections', 'type': 'gauge'}, - 'DistrCacheServerRegistryConnections': { - 'name': 'metrics.DistrCacheServerRegistryConnections', - 'type': 'gauge', - }, - 'DistrCacheServerS3CachedClients': {'name': 'metrics.DistrCacheServerS3CachedClients', 'type': 'gauge'}, - 'DistrCacheUsedConnections': {'name': 'metrics.DistrCacheUsedConnections', 'type': 'gauge'}, - 'DistrCacheWriteRequests': {'name': 'metrics.DistrCacheWriteRequests', 'type': 'gauge'}, - 'DistributedBytesToInsert': {'name': 'metrics.DistributedBytesToInsert', 'type': 'gauge'}, - 'DistributedFilesToInsert': {'name': 'metrics.DistributedFilesToInsert', 'type': 'gauge'}, - 'DistributedInsertThreads': {'name': 'metrics.DistributedInsertThreads', 'type': 'gauge'}, - 'DistributedInsertThreadsActive': {'name': 'metrics.DistributedInsertThreadsActive', 'type': 'gauge'}, - 'DistributedInsertThreadsScheduled': { - 'name': 'metrics.DistributedInsertThreadsScheduled', - 'type': 'gauge', - }, - 'DistributedSend': {'name': 'metrics.DistributedSend', 'type': 'gauge'}, - 'DropDistributedCacheThreads': {'name': 'metrics.DropDistributedCacheThreads', 'type': 'gauge'}, - 'DropDistributedCacheThreadsActive': { - 'name': 'metrics.DropDistributedCacheThreadsActive', - 'type': 'gauge', - }, - 'DropDistributedCacheThreadsScheduled': { - 'name': 'metrics.DropDistributedCacheThreadsScheduled', - 'type': 'gauge', - }, - 'EphemeralNode': {'name': 'metrics.EphemeralNode', 'type': 'gauge'}, - 'FilesystemCacheDelayedCleanupElements': { - 'name': 'metrics.FilesystemCacheDelayedCleanupElements', - 'type': 'gauge', - }, - 'FilesystemCacheDownloadQueueElements': { - 'name': 'metrics.FilesystemCacheDownloadQueueElements', - 'type': 'gauge', - }, - 'FilesystemCacheElements': {'name': 'metrics.FilesystemCacheElements', 'type': 'gauge'}, - 'FilesystemCacheHoldFileSegments': {'name': 'metrics.FilesystemCacheHoldFileSegments', 'type': 'gauge'}, - 'FilesystemCacheKeys': {'name': 'metrics.FilesystemCacheKeys', 'type': 'gauge'}, - 'FilesystemCacheReadBuffers': {'name': 'metrics.FilesystemCacheReadBuffers', 'type': 'gauge'}, - 'FilesystemCacheReserveThreads': {'name': 'metrics.FilesystemCacheReserveThreads', 'type': 'gauge'}, - 'FilesystemCacheSize': {'name': 'metrics.FilesystemCacheSize', 'type': 'gauge'}, - 'FilesystemCacheSizeLimit': {'name': 'metrics.FilesystemCacheSizeLimit', 'type': 'gauge'}, - 'FilteringMarksWithPrimaryKey': {'name': 'metrics.FilteringMarksWithPrimaryKey', 'type': 'gauge'}, - 'FilteringMarksWithSecondaryKeys': {'name': 'metrics.FilteringMarksWithSecondaryKeys', 'type': 'gauge'}, - 'FormatParsingThreads': {'name': 'metrics.FormatParsingThreads', 'type': 'gauge'}, - 'FormatParsingThreadsActive': {'name': 'metrics.FormatParsingThreadsActive', 'type': 'gauge'}, - 'FormatParsingThreadsScheduled': {'name': 'metrics.FormatParsingThreadsScheduled', 'type': 'gauge'}, - 'GlobalThread': {'name': 'metrics.GlobalThread', 'type': 'gauge'}, - 'GlobalThreadActive': {'name': 'metrics.GlobalThreadActive', 'type': 'gauge'}, - 'GlobalThreadScheduled': {'name': 'metrics.GlobalThreadScheduled', 'type': 'gauge'}, - 'HTTPConnection': {'name': 'metrics.HTTPConnection', 'type': 'gauge'}, - 'HTTPConnectionsStored': {'name': 'metrics.HTTPConnectionsStored', 'type': 'gauge'}, - 'HTTPConnectionsTotal': {'name': 'metrics.HTTPConnectionsTotal', 'type': 'gauge'}, - 'HashedDictionaryThreads': {'name': 'metrics.HashedDictionaryThreads', 'type': 'gauge'}, - 'HashedDictionaryThreadsActive': {'name': 'metrics.HashedDictionaryThreadsActive', 'type': 'gauge'}, - 'HashedDictionaryThreadsScheduled': { - 'name': 'metrics.HashedDictionaryThreadsScheduled', - 'type': 'gauge', - }, - 'HiveFilesCacheBytes': {'name': 'metrics.HiveFilesCacheBytes', 'type': 'gauge'}, - 'HiveFilesCacheFiles': {'name': 'metrics.HiveFilesCacheFiles', 'type': 'gauge'}, - 'HiveMetadataFilesCacheBytes': {'name': 'metrics.HiveMetadataFilesCacheBytes', 'type': 'gauge'}, - 'HiveMetadataFilesCacheFiles': {'name': 'metrics.HiveMetadataFilesCacheFiles', 'type': 'gauge'}, - 'IDiskCopierThreads': {'name': 'metrics.IDiskCopierThreads', 'type': 'gauge'}, - 'IDiskCopierThreadsActive': {'name': 'metrics.IDiskCopierThreadsActive', 'type': 'gauge'}, - 'IDiskCopierThreadsScheduled': {'name': 'metrics.IDiskCopierThreadsScheduled', 'type': 'gauge'}, - 'IOPrefetchThreads': {'name': 'metrics.IOPrefetchThreads', 'type': 'gauge'}, - 'IOPrefetchThreadsActive': {'name': 'metrics.IOPrefetchThreadsActive', 'type': 'gauge'}, - 'IOPrefetchThreadsScheduled': {'name': 'metrics.IOPrefetchThreadsScheduled', 'type': 'gauge'}, - 'IOThreads': {'name': 'metrics.IOThreads', 'type': 'gauge'}, - 'IOThreadsActive': {'name': 'metrics.IOThreadsActive', 'type': 'gauge'}, - 'IOThreadsScheduled': {'name': 'metrics.IOThreadsScheduled', 'type': 'gauge'}, - 'IOUringInFlightEvents': {'name': 'metrics.IOUringInFlightEvents', 'type': 'gauge'}, - 'IOUringPendingEvents': {'name': 'metrics.IOUringPendingEvents', 'type': 'gauge'}, - 'IOWriterThreads': {'name': 'metrics.IOWriterThreads', 'type': 'gauge'}, - 'IOWriterThreadsActive': {'name': 'metrics.IOWriterThreadsActive', 'type': 'gauge'}, - 'IOWriterThreadsScheduled': {'name': 'metrics.IOWriterThreadsScheduled', 'type': 'gauge'}, - 'IcebergCatalogThreads': {'name': 'metrics.IcebergCatalogThreads', 'type': 'gauge'}, - 'IcebergCatalogThreadsActive': {'name': 'metrics.IcebergCatalogThreadsActive', 'type': 'gauge'}, - 'IcebergCatalogThreadsScheduled': {'name': 'metrics.IcebergCatalogThreadsScheduled', 'type': 'gauge'}, - 'IcebergMetadataFilesCacheBytes': {'name': 'metrics.IcebergMetadataFilesCacheBytes', 'type': 'gauge'}, - 'IcebergMetadataFilesCacheFiles': {'name': 'metrics.IcebergMetadataFilesCacheFiles', 'type': 'gauge'}, - 'IndexMarkCacheBytes': {'name': 'metrics.IndexMarkCacheBytes', 'type': 'gauge'}, - 'IndexMarkCacheFiles': {'name': 'metrics.IndexMarkCacheFiles', 'type': 'gauge'}, - 'IndexUncompressedCacheBytes': {'name': 'metrics.IndexUncompressedCacheBytes', 'type': 'gauge'}, - 'IndexUncompressedCacheCells': {'name': 'metrics.IndexUncompressedCacheCells', 'type': 'gauge'}, - 'InterserverConnection': {'name': 'metrics.InterserverConnection', 'type': 'gauge'}, - 'IsServerShuttingDown': {'name': 'metrics.IsServerShuttingDown', 'type': 'gauge'}, - 'KafkaAssignedPartitions': {'name': 'metrics.KafkaAssignedPartitions', 'type': 'gauge'}, - 'KafkaBackgroundReads': {'name': 'metrics.KafkaBackgroundReads', 'type': 'gauge'}, - 'KafkaConsumers': {'name': 'metrics.KafkaConsumers', 'type': 'gauge'}, - 'KafkaConsumersInUse': {'name': 'metrics.KafkaConsumersInUse', 'type': 'gauge'}, - 'KafkaConsumersWithAssignment': {'name': 'metrics.KafkaConsumersWithAssignment', 'type': 'gauge'}, - 'KafkaLibrdkafkaThreads': {'name': 'metrics.KafkaLibrdkafkaThreads', 'type': 'gauge'}, - 'KafkaProducers': {'name': 'metrics.KafkaProducers', 'type': 'gauge'}, - 'KafkaWrites': {'name': 'metrics.KafkaWrites', 'type': 'gauge'}, - 'KeeperAliveConnections': {'name': 'metrics.KeeperAliveConnections', 'type': 'gauge'}, - 'KeeperOutstandingRequests': {'name': 'metrics.KeeperOutstandingRequests', 'type': 'gauge'}, - 'LicenseRemainingSeconds': {'name': 'metrics.LicenseRemainingSeconds', 'type': 'gauge'}, - 'LocalThread': {'name': 'metrics.LocalThread', 'type': 'gauge'}, - 'LocalThreadActive': {'name': 'metrics.LocalThreadActive', 'type': 'gauge'}, - 'LocalThreadScheduled': {'name': 'metrics.LocalThreadScheduled', 'type': 'gauge'}, - 'MMapCacheCells': {'name': 'metrics.MMapCacheCells', 'type': 'gauge'}, - 'MMappedFileBytes': {'name': 'metrics.MMappedFileBytes', 'type': 'gauge'}, - 'MMappedFiles': {'name': 'metrics.MMappedFiles', 'type': 'gauge'}, - 'MarkCacheBytes': {'name': 'metrics.MarkCacheBytes', 'type': 'gauge'}, - 'MarkCacheFiles': {'name': 'metrics.MarkCacheFiles', 'type': 'gauge'}, - 'MarksLoaderThreads': {'name': 'metrics.MarksLoaderThreads', 'type': 'gauge'}, - 'MarksLoaderThreadsActive': {'name': 'metrics.MarksLoaderThreadsActive', 'type': 'gauge'}, - 'MarksLoaderThreadsScheduled': {'name': 'metrics.MarksLoaderThreadsScheduled', 'type': 'gauge'}, - 'MaxDDLEntryID': {'name': 'metrics.MaxDDLEntryID', 'type': 'gauge'}, - 'MaxPushedDDLEntryID': {'name': 'metrics.MaxPushedDDLEntryID', 'type': 'gauge'}, - 'MemoryTracking': {'name': 'metrics.MemoryTracking', 'type': 'gauge'}, - 'MemoryTrackingUncorrected': {'name': 'metrics.MemoryTrackingUncorrected', 'type': 'gauge'}, - 'Merge': {'name': 'metrics.Merge', 'type': 'gauge'}, - 'MergeJoinBlocksCacheBytes': {'name': 'metrics.MergeJoinBlocksCacheBytes', 'type': 'gauge'}, - 'MergeJoinBlocksCacheCount': {'name': 'metrics.MergeJoinBlocksCacheCount', 'type': 'gauge'}, - 'MergeParts': {'name': 'metrics.MergeParts', 'type': 'gauge'}, - 'MergeTreeAllRangesAnnouncementsSent': { - 'name': 'metrics.MergeTreeAllRangesAnnouncementsSent', - 'type': 'gauge', - }, - 'MergeTreeBackgroundExecutorThreads': { - 'name': 'metrics.MergeTreeBackgroundExecutorThreads', - 'type': 'gauge', - }, - 'MergeTreeBackgroundExecutorThreadsActive': { - 'name': 'metrics.MergeTreeBackgroundExecutorThreadsActive', - 'type': 'gauge', - }, - 'MergeTreeBackgroundExecutorThreadsScheduled': { - 'name': 'metrics.MergeTreeBackgroundExecutorThreadsScheduled', - 'type': 'gauge', - }, - 'MergeTreeDataSelectExecutorThreads': { - 'name': 'metrics.MergeTreeDataSelectExecutorThreads', - 'type': 'gauge', - }, - 'MergeTreeDataSelectExecutorThreadsActive': { - 'name': 'metrics.MergeTreeDataSelectExecutorThreadsActive', - 'type': 'gauge', - }, - 'MergeTreeDataSelectExecutorThreadsScheduled': { - 'name': 'metrics.MergeTreeDataSelectExecutorThreadsScheduled', - 'type': 'gauge', - }, - 'MergeTreeFetchPartitionThreads': {'name': 'metrics.MergeTreeFetchPartitionThreads', 'type': 'gauge'}, - 'MergeTreeFetchPartitionThreadsActive': { - 'name': 'metrics.MergeTreeFetchPartitionThreadsActive', - 'type': 'gauge', - }, - 'MergeTreeFetchPartitionThreadsScheduled': { - 'name': 'metrics.MergeTreeFetchPartitionThreadsScheduled', - 'type': 'gauge', - }, - 'MergeTreeOutdatedPartsLoaderThreads': { - 'name': 'metrics.MergeTreeOutdatedPartsLoaderThreads', - 'type': 'gauge', - }, - 'MergeTreeOutdatedPartsLoaderThreadsActive': { - 'name': 'metrics.MergeTreeOutdatedPartsLoaderThreadsActive', - 'type': 'gauge', - }, - 'MergeTreeOutdatedPartsLoaderThreadsScheduled': { - 'name': 'metrics.MergeTreeOutdatedPartsLoaderThreadsScheduled', - 'type': 'gauge', - }, - 'MergeTreePartsCleanerThreads': {'name': 'metrics.MergeTreePartsCleanerThreads', 'type': 'gauge'}, - 'MergeTreePartsCleanerThreadsActive': { - 'name': 'metrics.MergeTreePartsCleanerThreadsActive', - 'type': 'gauge', - }, - 'MergeTreePartsCleanerThreadsScheduled': { - 'name': 'metrics.MergeTreePartsCleanerThreadsScheduled', - 'type': 'gauge', - }, - 'MergeTreePartsLoaderThreads': {'name': 'metrics.MergeTreePartsLoaderThreads', 'type': 'gauge'}, - 'MergeTreePartsLoaderThreadsActive': { - 'name': 'metrics.MergeTreePartsLoaderThreadsActive', - 'type': 'gauge', - }, - 'MergeTreePartsLoaderThreadsScheduled': { - 'name': 'metrics.MergeTreePartsLoaderThreadsScheduled', - 'type': 'gauge', - }, - 'MergeTreeReadTaskRequestsSent': {'name': 'metrics.MergeTreeReadTaskRequestsSent', 'type': 'gauge'}, - 'MergeTreeSubcolumnsReaderThreads': { - 'name': 'metrics.MergeTreeSubcolumnsReaderThreads', - 'type': 'gauge', - }, - 'MergeTreeSubcolumnsReaderThreadsActive': { - 'name': 'metrics.MergeTreeSubcolumnsReaderThreadsActive', - 'type': 'gauge', - }, - 'MergeTreeSubcolumnsReaderThreadsScheduled': { - 'name': 'metrics.MergeTreeSubcolumnsReaderThreadsScheduled', - 'type': 'gauge', - }, - 'MergeTreeUnexpectedPartsLoaderThreads': { - 'name': 'metrics.MergeTreeUnexpectedPartsLoaderThreads', - 'type': 'gauge', - }, - 'MergeTreeUnexpectedPartsLoaderThreadsActive': { - 'name': 'metrics.MergeTreeUnexpectedPartsLoaderThreadsActive', - 'type': 'gauge', - }, - 'MergeTreeUnexpectedPartsLoaderThreadsScheduled': { - 'name': 'metrics.MergeTreeUnexpectedPartsLoaderThreadsScheduled', - 'type': 'gauge', - }, - 'MergesMutationsMemoryTracking': {'name': 'metrics.MergesMutationsMemoryTracking', 'type': 'gauge'}, - 'MetadataFromKeeperCacheObjects': {'name': 'metrics.MetadataFromKeeperCacheObjects', 'type': 'gauge'}, - 'Move': {'name': 'metrics.Move', 'type': 'gauge'}, - 'MySQLConnection': {'name': 'metrics.MySQLConnection', 'type': 'gauge'}, - 'NetworkReceive': {'name': 'metrics.NetworkReceive', 'type': 'gauge'}, - 'NetworkSend': {'name': 'metrics.NetworkSend', 'type': 'gauge'}, - 'ObjectStorageAzureThreads': {'name': 'metrics.ObjectStorageAzureThreads', 'type': 'gauge'}, - 'ObjectStorageAzureThreadsActive': {'name': 'metrics.ObjectStorageAzureThreadsActive', 'type': 'gauge'}, - 'ObjectStorageAzureThreadsScheduled': { - 'name': 'metrics.ObjectStorageAzureThreadsScheduled', - 'type': 'gauge', - }, - 'ObjectStorageQueueRegisteredServers': { - 'name': 'metrics.ObjectStorageQueueRegisteredServers', - 'type': 'gauge', - }, - 'ObjectStorageQueueShutdownThreads': { - 'name': 'metrics.ObjectStorageQueueShutdownThreads', - 'type': 'gauge', - }, - 'ObjectStorageQueueShutdownThreadsActive': { - 'name': 'metrics.ObjectStorageQueueShutdownThreadsActive', - 'type': 'gauge', - }, - 'ObjectStorageQueueShutdownThreadsScheduled': { - 'name': 'metrics.ObjectStorageQueueShutdownThreadsScheduled', - 'type': 'gauge', - }, - 'ObjectStorageS3Threads': {'name': 'metrics.ObjectStorageS3Threads', 'type': 'gauge'}, - 'ObjectStorageS3ThreadsActive': {'name': 'metrics.ObjectStorageS3ThreadsActive', 'type': 'gauge'}, - 'ObjectStorageS3ThreadsScheduled': {'name': 'metrics.ObjectStorageS3ThreadsScheduled', 'type': 'gauge'}, - 'OpenFileForRead': {'name': 'metrics.OpenFileForRead', 'type': 'gauge'}, - 'OpenFileForWrite': {'name': 'metrics.OpenFileForWrite', 'type': 'gauge'}, - 'OutdatedPartsLoadingThreads': {'name': 'metrics.OutdatedPartsLoadingThreads', 'type': 'gauge'}, - 'OutdatedPartsLoadingThreadsActive': { - 'name': 'metrics.OutdatedPartsLoadingThreadsActive', - 'type': 'gauge', - }, - 'OutdatedPartsLoadingThreadsScheduled': { - 'name': 'metrics.OutdatedPartsLoadingThreadsScheduled', - 'type': 'gauge', - }, - 'PageCacheBytes': {'name': 'metrics.PageCacheBytes', 'type': 'gauge'}, - 'PageCacheCells': {'name': 'metrics.PageCacheCells', 'type': 'gauge'}, - 'ParallelCompressedWriteBufferThreads': { - 'name': 'metrics.ParallelCompressedWriteBufferThreads', - 'type': 'gauge', - }, - 'ParallelCompressedWriteBufferWait': { - 'name': 'metrics.ParallelCompressedWriteBufferWait', - 'type': 'gauge', - }, - 'ParallelFormattingOutputFormatThreads': { - 'name': 'metrics.ParallelFormattingOutputFormatThreads', - 'type': 'gauge', - }, - 'ParallelFormattingOutputFormatThreadsActive': { - 'name': 'metrics.ParallelFormattingOutputFormatThreadsActive', - 'type': 'gauge', - }, - 'ParallelFormattingOutputFormatThreadsScheduled': { - 'name': 'metrics.ParallelFormattingOutputFormatThreadsScheduled', - 'type': 'gauge', - }, - 'ParallelParsingInputFormatThreads': { - 'name': 'metrics.ParallelParsingInputFormatThreads', - 'type': 'gauge', - }, - 'ParallelParsingInputFormatThreadsActive': { - 'name': 'metrics.ParallelParsingInputFormatThreadsActive', - 'type': 'gauge', - }, - 'ParallelParsingInputFormatThreadsScheduled': { - 'name': 'metrics.ParallelParsingInputFormatThreadsScheduled', - 'type': 'gauge', - }, - 'ParallelWithQueryActiveThreads': {'name': 'metrics.ParallelWithQueryActiveThreads', 'type': 'gauge'}, - 'ParallelWithQueryScheduledThreads': { - 'name': 'metrics.ParallelWithQueryScheduledThreads', - 'type': 'gauge', - }, - 'ParallelWithQueryThreads': {'name': 'metrics.ParallelWithQueryThreads', 'type': 'gauge'}, - 'ParquetDecoderIOThreads': {'name': 'metrics.ParquetDecoderIOThreads', 'type': 'gauge'}, - 'ParquetDecoderIOThreadsActive': {'name': 'metrics.ParquetDecoderIOThreadsActive', 'type': 'gauge'}, - 'ParquetDecoderIOThreadsScheduled': { - 'name': 'metrics.ParquetDecoderIOThreadsScheduled', - 'type': 'gauge', - }, - 'ParquetDecoderThreads': {'name': 'metrics.ParquetDecoderThreads', 'type': 'gauge'}, - 'ParquetDecoderThreadsActive': {'name': 'metrics.ParquetDecoderThreadsActive', 'type': 'gauge'}, - 'ParquetDecoderThreadsScheduled': {'name': 'metrics.ParquetDecoderThreadsScheduled', 'type': 'gauge'}, - 'ParquetEncoderThreads': {'name': 'metrics.ParquetEncoderThreads', 'type': 'gauge'}, - 'ParquetEncoderThreadsActive': {'name': 'metrics.ParquetEncoderThreadsActive', 'type': 'gauge'}, - 'ParquetEncoderThreadsScheduled': {'name': 'metrics.ParquetEncoderThreadsScheduled', 'type': 'gauge'}, - 'PartMutation': {'name': 'metrics.PartMutation', 'type': 'gauge'}, - 'PartsActive': {'name': 'metrics.PartsActive', 'type': 'gauge'}, - 'PartsCommitted': {'name': 'metrics.PartsCommitted', 'type': 'gauge'}, - 'PartsCompact': {'name': 'metrics.PartsCompact', 'type': 'gauge'}, - 'PartsDeleteOnDestroy': {'name': 'metrics.PartsDeleteOnDestroy', 'type': 'gauge'}, - 'PartsDeleting': {'name': 'metrics.PartsDeleting', 'type': 'gauge'}, - 'PartsOutdated': {'name': 'metrics.PartsOutdated', 'type': 'gauge'}, - 'PartsPreActive': {'name': 'metrics.PartsPreActive', 'type': 'gauge'}, - 'PartsPreCommitted': {'name': 'metrics.PartsPreCommitted', 'type': 'gauge'}, - 'PartsTemporary': {'name': 'metrics.PartsTemporary', 'type': 'gauge'}, - 'PartsWide': {'name': 'metrics.PartsWide', 'type': 'gauge'}, - 'PendingAsyncInsert': {'name': 'metrics.PendingAsyncInsert', 'type': 'gauge'}, - 'PolygonDictionaryThreads': {'name': 'metrics.PolygonDictionaryThreads', 'type': 'gauge'}, - 'PolygonDictionaryThreadsActive': {'name': 'metrics.PolygonDictionaryThreadsActive', 'type': 'gauge'}, - 'PolygonDictionaryThreadsScheduled': { - 'name': 'metrics.PolygonDictionaryThreadsScheduled', - 'type': 'gauge', - }, - 'PostgreSQLConnection': {'name': 'metrics.PostgreSQLConnection', 'type': 'gauge'}, - 'PrimaryIndexCacheBytes': {'name': 'metrics.PrimaryIndexCacheBytes', 'type': 'gauge'}, - 'PrimaryIndexCacheFiles': {'name': 'metrics.PrimaryIndexCacheFiles', 'type': 'gauge'}, - 'Query': {'name': 'metrics.Query', 'type': 'gauge'}, - 'QueryCacheBytes': {'name': 'metrics.QueryCacheBytes', 'type': 'gauge'}, - 'QueryCacheEntries': {'name': 'metrics.QueryCacheEntries', 'type': 'gauge'}, - 'QueryConditionCacheBytes': {'name': 'metrics.QueryConditionCacheBytes', 'type': 'gauge'}, - 'QueryConditionCacheEntries': {'name': 'metrics.QueryConditionCacheEntries', 'type': 'gauge'}, - 'QueryPipelineExecutorThreads': {'name': 'metrics.QueryPipelineExecutorThreads', 'type': 'gauge'}, - 'QueryPipelineExecutorThreadsActive': { - 'name': 'metrics.QueryPipelineExecutorThreadsActive', - 'type': 'gauge', - }, - 'QueryPipelineExecutorThreadsScheduled': { - 'name': 'metrics.QueryPipelineExecutorThreadsScheduled', - 'type': 'gauge', - }, - 'QueryPreempted': {'name': 'metrics.QueryPreempted', 'type': 'gauge'}, - 'QueryThread': {'name': 'metrics.QueryThread', 'type': 'gauge'}, - 'RWLockActiveReaders': {'name': 'metrics.RWLockActiveReaders', 'type': 'gauge'}, - 'RWLockActiveWriters': {'name': 'metrics.RWLockActiveWriters', 'type': 'gauge'}, - 'RWLockWaitingReaders': {'name': 'metrics.RWLockWaitingReaders', 'type': 'gauge'}, - 'RWLockWaitingWriters': {'name': 'metrics.RWLockWaitingWriters', 'type': 'gauge'}, - 'Read': {'name': 'metrics.Read', 'type': 'gauge'}, - 'ReadTaskRequestsSent': {'name': 'metrics.ReadTaskRequestsSent', 'type': 'gauge'}, - 'ReadonlyDisks': {'name': 'metrics.ReadonlyDisks', 'type': 'gauge'}, - 'ReadonlyReplica': {'name': 'metrics.ReadonlyReplica', 'type': 'gauge'}, - 'RefreshableViews': {'name': 'metrics.RefreshableViews', 'type': 'gauge'}, - 'RefreshingViews': {'name': 'metrics.RefreshingViews', 'type': 'gauge'}, - 'RemoteRead': {'name': 'metrics.RemoteRead', 'type': 'gauge'}, - 'ReplicatedChecks': {'name': 'metrics.ReplicatedChecks', 'type': 'gauge'}, - 'ReplicatedFetch': {'name': 'metrics.ReplicatedFetch', 'type': 'gauge'}, - 'ReplicatedSend': {'name': 'metrics.ReplicatedSend', 'type': 'gauge'}, - 'RestartReplicaThreads': {'name': 'metrics.RestartReplicaThreads', 'type': 'gauge'}, - 'RestartReplicaThreadsActive': {'name': 'metrics.RestartReplicaThreadsActive', 'type': 'gauge'}, - 'RestartReplicaThreadsScheduled': {'name': 'metrics.RestartReplicaThreadsScheduled', 'type': 'gauge'}, - 'RestoreThreads': {'name': 'metrics.RestoreThreads', 'type': 'gauge'}, - 'RestoreThreadsActive': {'name': 'metrics.RestoreThreadsActive', 'type': 'gauge'}, - 'RestoreThreadsScheduled': {'name': 'metrics.RestoreThreadsScheduled', 'type': 'gauge'}, - 'Revision': {'name': 'metrics.Revision', 'type': 'gauge'}, - 'S3Requests': {'name': 'metrics.S3Requests', 'type': 'gauge'}, - 'SchedulerIOReadScheduled': {'name': 'metrics.SchedulerIOReadScheduled', 'type': 'gauge'}, - 'SchedulerIOWriteScheduled': {'name': 'metrics.SchedulerIOWriteScheduled', 'type': 'gauge'}, - 'SendExternalTables': {'name': 'metrics.SendExternalTables', 'type': 'gauge'}, - 'SendScalars': {'name': 'metrics.SendScalars', 'type': 'gauge'}, - 'SharedCatalogDropDetachLocalTablesErrors': { - 'name': 'metrics.SharedCatalogDropDetachLocalTablesErrors', - 'type': 'gauge', - }, - 'SharedCatalogDropLocalThreads': {'name': 'metrics.SharedCatalogDropLocalThreads', 'type': 'gauge'}, - 'SharedCatalogDropLocalThreadsActive': { - 'name': 'metrics.SharedCatalogDropLocalThreadsActive', - 'type': 'gauge', - }, - 'SharedCatalogDropLocalThreadsScheduled': { - 'name': 'metrics.SharedCatalogDropLocalThreadsScheduled', - 'type': 'gauge', - }, - 'SharedCatalogDropZooKeeperThreads': { - 'name': 'metrics.SharedCatalogDropZooKeeperThreads', - 'type': 'gauge', - }, - 'SharedCatalogDropZooKeeperThreadsActive': { - 'name': 'metrics.SharedCatalogDropZooKeeperThreadsActive', - 'type': 'gauge', - }, - 'SharedCatalogDropZooKeeperThreadsScheduled': { - 'name': 'metrics.SharedCatalogDropZooKeeperThreadsScheduled', - 'type': 'gauge', - }, - 'SharedCatalogNumberOfObjectsInState': { - 'name': 'metrics.SharedCatalogNumberOfObjectsInState', - 'type': 'gauge', - }, - 'SharedCatalogStateApplicationThreads': { - 'name': 'metrics.SharedCatalogStateApplicationThreads', - 'type': 'gauge', - }, - 'SharedCatalogStateApplicationThreadsActive': { - 'name': 'metrics.SharedCatalogStateApplicationThreadsActive', - 'type': 'gauge', - }, - 'SharedCatalogStateApplicationThreadsScheduled': { - 'name': 'metrics.SharedCatalogStateApplicationThreadsScheduled', - 'type': 'gauge', - }, - 'SharedDatabaseCatalogTablesInLocalDropDetachQueue': { - 'name': 'metrics.SharedDatabaseCatalogTablesInLocalDropDetachQueue', - 'type': 'gauge', - }, - 'SharedMergeTreeAssignedCurrentParts': { - 'name': 'metrics.SharedMergeTreeAssignedCurrentParts', - 'type': 'gauge', - }, - 'SharedMergeTreeCondemnedPartsInKeeper': { - 'name': 'metrics.SharedMergeTreeCondemnedPartsInKeeper', - 'type': 'gauge', - }, - 'SharedMergeTreeFetch': {'name': 'metrics.SharedMergeTreeFetch', 'type': 'gauge'}, - 'SharedMergeTreeOutdatedPartsInKeeper': { - 'name': 'metrics.SharedMergeTreeOutdatedPartsInKeeper', - 'type': 'gauge', - }, - 'SharedMergeTreeThreads': {'name': 'metrics.SharedMergeTreeThreads', 'type': 'gauge'}, - 'SharedMergeTreeThreadsActive': {'name': 'metrics.SharedMergeTreeThreadsActive', 'type': 'gauge'}, - 'SharedMergeTreeThreadsScheduled': {'name': 'metrics.SharedMergeTreeThreadsScheduled', 'type': 'gauge'}, - 'StartupScriptsExecutionState': {'name': 'metrics.StartupScriptsExecutionState', 'type': 'gauge'}, - 'StartupSystemTablesThreads': {'name': 'metrics.StartupSystemTablesThreads', 'type': 'gauge'}, - 'StartupSystemTablesThreadsActive': { - 'name': 'metrics.StartupSystemTablesThreadsActive', - 'type': 'gauge', - }, - 'StartupSystemTablesThreadsScheduled': { - 'name': 'metrics.StartupSystemTablesThreadsScheduled', - 'type': 'gauge', - }, - 'StatelessWorkerThreads': {'name': 'metrics.StatelessWorkerThreads', 'type': 'gauge'}, - 'StatelessWorkerThreadsActive': {'name': 'metrics.StatelessWorkerThreadsActive', 'type': 'gauge'}, - 'StatelessWorkerThreadsScheduled': {'name': 'metrics.StatelessWorkerThreadsScheduled', 'type': 'gauge'}, - 'StorageBufferBytes': {'name': 'metrics.StorageBufferBytes', 'type': 'gauge'}, - 'StorageBufferFlushThreads': {'name': 'metrics.StorageBufferFlushThreads', 'type': 'gauge'}, - 'StorageBufferFlushThreadsActive': {'name': 'metrics.StorageBufferFlushThreadsActive', 'type': 'gauge'}, - 'StorageBufferFlushThreadsScheduled': { - 'name': 'metrics.StorageBufferFlushThreadsScheduled', - 'type': 'gauge', - }, - 'StorageBufferRows': {'name': 'metrics.StorageBufferRows', 'type': 'gauge'}, - 'StorageConnectionsStored': {'name': 'metrics.StorageConnectionsStored', 'type': 'gauge'}, - 'StorageConnectionsTotal': {'name': 'metrics.StorageConnectionsTotal', 'type': 'gauge'}, - 'StorageDistributedThreads': {'name': 'metrics.StorageDistributedThreads', 'type': 'gauge'}, - 'StorageDistributedThreadsActive': {'name': 'metrics.StorageDistributedThreadsActive', 'type': 'gauge'}, - 'StorageDistributedThreadsScheduled': { - 'name': 'metrics.StorageDistributedThreadsScheduled', - 'type': 'gauge', - }, - 'StorageHiveThreads': {'name': 'metrics.StorageHiveThreads', 'type': 'gauge'}, - 'StorageHiveThreadsActive': {'name': 'metrics.StorageHiveThreadsActive', 'type': 'gauge'}, - 'StorageHiveThreadsScheduled': {'name': 'metrics.StorageHiveThreadsScheduled', 'type': 'gauge'}, - 'StorageObjectStorageThreads': {'name': 'metrics.StorageObjectStorageThreads', 'type': 'gauge'}, - 'StorageObjectStorageThreadsActive': { - 'name': 'metrics.StorageObjectStorageThreadsActive', - 'type': 'gauge', - }, - 'StorageObjectStorageThreadsScheduled': { - 'name': 'metrics.StorageObjectStorageThreadsScheduled', - 'type': 'gauge', - }, - 'StorageS3Threads': {'name': 'metrics.StorageS3Threads', 'type': 'gauge'}, - 'StorageS3ThreadsActive': {'name': 'metrics.StorageS3ThreadsActive', 'type': 'gauge'}, - 'StorageS3ThreadsScheduled': {'name': 'metrics.StorageS3ThreadsScheduled', 'type': 'gauge'}, - 'SystemReplicasThreads': {'name': 'metrics.SystemReplicasThreads', 'type': 'gauge'}, - 'SystemReplicasThreadsActive': {'name': 'metrics.SystemReplicasThreadsActive', 'type': 'gauge'}, - 'SystemReplicasThreadsScheduled': {'name': 'metrics.SystemReplicasThreadsScheduled', 'type': 'gauge'}, - 'TCPConnection': {'name': 'metrics.TCPConnection', 'type': 'gauge'}, - 'TablesLoaderBackgroundThreads': {'name': 'metrics.TablesLoaderBackgroundThreads', 'type': 'gauge'}, - 'TablesLoaderBackgroundThreadsActive': { - 'name': 'metrics.TablesLoaderBackgroundThreadsActive', - 'type': 'gauge', - }, - 'TablesLoaderBackgroundThreadsScheduled': { - 'name': 'metrics.TablesLoaderBackgroundThreadsScheduled', - 'type': 'gauge', - }, - 'TablesLoaderForegroundThreads': {'name': 'metrics.TablesLoaderForegroundThreads', 'type': 'gauge'}, - 'TablesLoaderForegroundThreadsActive': { - 'name': 'metrics.TablesLoaderForegroundThreadsActive', - 'type': 'gauge', - }, - 'TablesLoaderForegroundThreadsScheduled': { - 'name': 'metrics.TablesLoaderForegroundThreadsScheduled', - 'type': 'gauge', - }, - 'TablesToDropQueueSize': {'name': 'metrics.TablesToDropQueueSize', 'type': 'gauge'}, - 'TaskTrackerThreads': {'name': 'metrics.TaskTrackerThreads', 'type': 'gauge'}, - 'TaskTrackerThreadsActive': {'name': 'metrics.TaskTrackerThreadsActive', 'type': 'gauge'}, - 'TaskTrackerThreadsScheduled': {'name': 'metrics.TaskTrackerThreadsScheduled', 'type': 'gauge'}, - 'TemporaryFilesForAggregation': {'name': 'metrics.TemporaryFilesForAggregation', 'type': 'gauge'}, - 'TemporaryFilesForJoin': {'name': 'metrics.TemporaryFilesForJoin', 'type': 'gauge'}, - 'TemporaryFilesForMerge': {'name': 'metrics.TemporaryFilesForMerge', 'type': 'gauge'}, - 'TemporaryFilesForSort': {'name': 'metrics.TemporaryFilesForSort', 'type': 'gauge'}, - 'TemporaryFilesUnknown': {'name': 'metrics.TemporaryFilesUnknown', 'type': 'gauge'}, - 'ThreadPoolFSReaderThreads': {'name': 'metrics.ThreadPoolFSReaderThreads', 'type': 'gauge'}, - 'ThreadPoolFSReaderThreadsActive': {'name': 'metrics.ThreadPoolFSReaderThreadsActive', 'type': 'gauge'}, - 'ThreadPoolFSReaderThreadsScheduled': { - 'name': 'metrics.ThreadPoolFSReaderThreadsScheduled', - 'type': 'gauge', - }, - 'ThreadPoolRemoteFSReaderThreads': {'name': 'metrics.ThreadPoolRemoteFSReaderThreads', 'type': 'gauge'}, - 'ThreadPoolRemoteFSReaderThreadsActive': { - 'name': 'metrics.ThreadPoolRemoteFSReaderThreadsActive', - 'type': 'gauge', - }, - 'ThreadPoolRemoteFSReaderThreadsScheduled': { - 'name': 'metrics.ThreadPoolRemoteFSReaderThreadsScheduled', - 'type': 'gauge', - }, - 'ThreadsInOvercommitTracker': {'name': 'metrics.ThreadsInOvercommitTracker', 'type': 'gauge'}, - 'TotalTemporaryFiles': {'name': 'metrics.TotalTemporaryFiles', 'type': 'gauge'}, - 'UncompressedCacheBytes': {'name': 'metrics.UncompressedCacheBytes', 'type': 'gauge'}, - 'UncompressedCacheCells': {'name': 'metrics.UncompressedCacheCells', 'type': 'gauge'}, - 'VectorSimilarityIndexCacheBytes': {'name': 'metrics.VectorSimilarityIndexCacheBytes', 'type': 'gauge'}, - 'VectorSimilarityIndexCacheCells': {'name': 'metrics.VectorSimilarityIndexCacheCells', 'type': 'gauge'}, - 'VersionInteger': {'name': 'metrics.VersionInteger', 'type': 'gauge'}, - 'Write': {'name': 'metrics.Write', 'type': 'gauge'}, - 'ZooKeeperRequest': {'name': 'metrics.ZooKeeperRequest', 'type': 'gauge'}, - 'ZooKeeperSession': {'name': 'metrics.ZooKeeperSession', 'type': 'gauge'}, - 'ZooKeeperWatch': {'name': 'metrics.ZooKeeperWatch', 'type': 'gauge'}, - }, - }, - ], -} diff --git a/clickhouse/datadog_checks/clickhouse/clickhouse.py b/clickhouse/datadog_checks/clickhouse/clickhouse.py index e25a7992aed90..3f5e66416995d 100644 --- a/clickhouse/datadog_checks/clickhouse/clickhouse.py +++ b/clickhouse/datadog_checks/clickhouse/clickhouse.py @@ -69,6 +69,7 @@ def __init__(self, name, init_config, instances): self._error_sanitizer = ErrorSanitizer(self._config.password) self.check_initializations.append(self.validate_config) + self.check_initializations.append(advanced_queries.warm_cache) # Submit health event with config validation result # Tags are now available so health events will include them diff --git a/clickhouse/datadog_checks/clickhouse/data/system_async_metrics.json b/clickhouse/datadog_checks/clickhouse/data/system_async_metrics.json new file mode 100644 index 0000000000000..e23cf217e9c15 --- /dev/null +++ b/clickhouse/datadog_checks/clickhouse/data/system_async_metrics.json @@ -0,0 +1,135 @@ +{ + "name": "system_asynchronous_metrics", + "query": "SELECT value, metric FROM system.asynchronous_metrics", + "value_column": "metric_value", + "match_column": "metric_name", + "prefix": "asynchronous_metrics", + "items": { + "gauge": [ + "AsynchronousHeavyMetricsCalculationTimeSpent", + "AsynchronousHeavyMetricsUpdateInterval", + "AsynchronousMetricsCalculationTimeSpent", + "AsynchronousMetricsUpdateInterval", + "CGroupMaxCPU", + "CGroupMemoryTotal", + "CGroupMemoryUsed", + "CGroupSystemTime", + "CGroupSystemTimeNormalized", + "CGroupUserTime", + "CGroupUserTimeNormalized", + "CompiledExpressionCacheBytes", + "CompiledExpressionCacheCount", + "DictionaryTotalFailedUpdates", + "FilesystemCacheBytes", + "FilesystemCacheCapacity", + "FilesystemCacheFiles", + "FilesystemLogsPathAvailableBytes", + "FilesystemLogsPathAvailableINodes", + "FilesystemLogsPathTotalBytes", + "FilesystemLogsPathTotalINodes", + "FilesystemLogsPathUsedBytes", + "FilesystemLogsPathUsedINodes", + "FilesystemMainPathAvailableBytes", + "FilesystemMainPathAvailableINodes", + "FilesystemMainPathTotalBytes", + "FilesystemMainPathTotalINodes", + "FilesystemMainPathUsedBytes", + "FilesystemMainPathUsedINodes", + "HashTableStatsCacheEntries", + "HashTableStatsCacheHits", + "HashTableStatsCacheMisses", + "IndexMarkCacheBytes", + "IndexMarkCacheFiles", + "IndexUncompressedCacheBytes", + "IndexUncompressedCacheCells", + "Jitter", + "LoadAverage1", + "LoadAverage15", + "LoadAverage5", + "MMapCacheCells", + "MarkCacheBytes", + "MarkCacheFiles", + "MaxPartCountForPartition", + "MemoryCode", + "MemoryDataAndStack", + "MemoryResident", + "MemoryResidentMax", + "MemoryShared", + "MemoryVirtual", + "NetworkTCPReceiveQueue", + "NetworkTCPSocketRemoteAddresses", + "NetworkTCPSockets", + "NetworkTCPTransmitQueue", + "NetworkTCPUnrecoveredRetransmits", + "NumberOfDatabases", + "NumberOfDetachedByUserParts", + "NumberOfDetachedParts", + "NumberOfPendingMutations", + "NumberOfPendingMutationsOverExecutionTime", + "NumberOfStuckMutations", + "NumberOfTables", + "NumberOfTablesSystem", + "OSCPUOverload", + "OSContextSwitches", + "OSGuestNiceTimeNormalized", + "OSGuestTimeNormalized", + "OSIOWaitTimeNormalized", + "OSIdleTimeNormalized", + "OSInterrupts", + "OSIrqTimeNormalized", + "OSMemoryAvailable", + "OSMemoryBuffers", + "OSMemoryCached", + "OSMemoryFreePlusCached", + "OSMemoryFreeWithoutCached", + "OSMemorySwapCached", + "OSMemoryTotal", + "OSNiceTimeNormalized", + "OSOpenFiles", + "OSProcessesBlocked", + "OSProcessesCreated", + "OSProcessesRunning", + "OSSoftIrqTimeNormalized", + "OSStealTimeNormalized", + "OSSystemTimeNormalized", + "OSThreadsRunnable", + "OSThreadsTotal", + "OSUptime", + "OSUserTimeNormalized", + "PageCacheBytes", + "PageCacheCells", + "PageCacheMaxBytes", + "PageCachePinnedBytes", + "PrimaryIndexCacheBytes", + "PrimaryIndexCacheFiles", + "QueryCacheBytes", + "QueryCacheEntries", + "ReplicasMaxAbsoluteDelay", + "ReplicasMaxInsertsInQueue", + "ReplicasMaxMergesInQueue", + "ReplicasMaxQueueSize", + "ReplicasMaxRelativeDelay", + "ReplicasSumInsertsInQueue", + "ReplicasSumMergesInQueue", + "ReplicasSumQueueSize", + "TotalBytesOfMergeTreeTables", + "TotalBytesOfMergeTreeTablesSystem", + "TotalIndexGranularityBytesInMemory", + "TotalIndexGranularityBytesInMemoryAllocated", + "TotalPartsOfMergeTreeTables", + "TotalPartsOfMergeTreeTablesSystem", + "TotalPrimaryKeyBytesInMemory", + "TotalPrimaryKeyBytesInMemoryAllocated", + "TotalRowsOfMergeTreeTables", + "TotalRowsOfMergeTreeTablesSystem", + "TrackedMemory", + "UncompressedCacheBytes", + "UncompressedCacheCells", + "UnreclaimableRSS", + "Uptime", + "VMMaxMapCount", + "VMNumMaps", + "jemalloc.epoch" + ] + } +} diff --git a/clickhouse/datadog_checks/clickhouse/data/system_events.json b/clickhouse/datadog_checks/clickhouse/data/system_events.json new file mode 100644 index 0000000000000..ca6aa30ffbb35 --- /dev/null +++ b/clickhouse/datadog_checks/clickhouse/data/system_events.json @@ -0,0 +1,1052 @@ +{ + "name": "system_events", + "query": "SELECT value, event FROM system.events", + "value_column": "metric_value", + "match_column": "metric_name", + "prefix": "events", + "items": { + "gauge": [ + "DistrCacheGetClient", + "DistrCacheHoldConnections", + "MutationsAppliedOnFlyInAllParts", + "PageCacheBytesUnpinnedRoundedToHugePages", + "PageCacheBytesUnpinnedRoundedToPages", + "PageCacheChunkDataHits", + "PageCacheChunkDataMisses", + "PageCacheChunkDataPartialHits", + "PageCacheChunkMisses", + "PageCacheChunkShared", + "PartsWithAppliedMutationsOnFly" + ], + "monotonic_gauge": [ + "AIORead", + "AIOReadBytes", + "AIOWrite", + "AIOWriteBytes", + "AddressesDiscovered", + "AddressesExpired", + "AddressesMarkedAsFailed", + "AggregationHashTablesInitializedAsTwoLevel", + "AggregationOptimizedEqualRangesOfKeys", + "AggregationPreallocatedElementsInHashTables", + "AnalyzePatchRangesMicroseconds", + "ApplyPatchesMicroseconds", + "ArenaAllocBytes", + "ArenaAllocChunks", + "AsyncInsertBytes", + "AsyncInsertCacheHits", + "AsyncInsertQuery", + "AsyncInsertRows", + "AsyncLoggingConsoleDroppedMessages", + "AsyncLoggingConsoleTotalMessages", + "AsyncLoggingErrorFileLogDroppedMessages", + "AsyncLoggingErrorFileLogTotalMessages", + "AsyncLoggingFileLogDroppedMessages", + "AsyncLoggingFileLogTotalMessages", + "AsyncLoggingSyslogDroppedMessages", + "AsyncLoggingSyslogTotalMessages", + "AsyncLoggingTextLogDroppedMessages", + "AsyncLoggingTextLogTotalMessages", + "AsynchronousReaderIgnoredBytes", + "AzureCommitBlockList", + "AzureCopyObject", + "AzureCreateContainer", + "AzureDeleteObjects", + "AzureGetObject", + "AzureGetProperties", + "AzureGetRequestThrottlerCount", + "AzureListObjects", + "AzurePutRequestThrottlerCount", + "AzureReadRequestsCount", + "AzureReadRequestsErrors", + "AzureReadRequestsRedirects", + "AzureReadRequestsThrottling", + "AzureStageBlock", + "AzureUpload", + "AzureWriteRequestsCount", + "AzureWriteRequestsErrors", + "AzureWriteRequestsRedirects", + "AzureWriteRequestsThrottling", + "BackgroundLoadingMarksTasks", + "BackupLockFileReads", + "BackupReadLocalBytesToCalculateChecksums", + "BackupReadLocalFilesToCalculateChecksums", + "BackupReadRemoteBytesToCalculateChecksums", + "BackupReadRemoteFilesToCalculateChecksums", + "BackupThrottlerBytes", + "BackupsOpenedForRead", + "BackupsOpenedForUnlock", + "BackupsOpenedForWrite", + "BuildPatchesJoinMicroseconds", + "BuildPatchesMergeMicroseconds", + "CacheWarmerBytesDownloaded", + "CacheWarmerDataPartsDownloaded", + "CachedReadBufferCacheWriteBytes", + "CachedReadBufferPredownloadedBytes", + "CachedReadBufferReadFromCacheBytes", + "CachedReadBufferReadFromCacheHits", + "CachedReadBufferReadFromCacheMisses", + "CachedReadBufferReadFromSourceBytes", + "CachedWriteBufferCacheWriteBytes", + "CannotRemoveEphemeralNode", + "CannotWriteToWriteBufferDiscard", + "CompileExpressionsBytes", + "CompileFunction", + "CompiledFunctionExecute", + "CompressedReadBufferBlocks", + "CompressedReadBufferBytes", + "CompressedReadBufferChecksumDoesntMatch", + "CompressedReadBufferChecksumDoesntMatchSingleBitMismatch", + "ConcurrencyControlDownscales", + "ConcurrencyControlPreemptions", + "ConcurrencyControlQueriesDelayed", + "ConcurrencyControlSlotsAcquired", + "ConcurrencyControlSlotsAcquiredNonCompeting", + "ConcurrencyControlSlotsDelayed", + "ConcurrencyControlSlotsGranted", + "ConcurrencyControlUpscales", + "ConcurrentQuerySlotsAcquired", + "ContextLock", + "CoordinatedMergesMergeAssignmentRequest", + "CoordinatedMergesMergeAssignmentResponse", + "CoordinatedMergesMergeCoordinatorLockStateExclusivelyCount", + "CoordinatedMergesMergeCoordinatorLockStateForShareCount", + "CoordinatedMergesMergeCoordinatorUpdateCount", + "CoordinatedMergesMergeWorkerUpdateCount", + "CreatedLogEntryForMerge", + "CreatedLogEntryForMutation", + "CreatedReadBufferDirectIO", + "CreatedReadBufferDirectIOFailed", + "CreatedReadBufferMMap", + "CreatedReadBufferMMapFailed", + "CreatedReadBufferOrdinary", + "DNSError", + "DataAfterMutationDiffersFromReplica", + "DefaultImplementationForNullsRows", + "DefaultImplementationForNullsRowsWithNulls", + "DelayedInserts", + "DelayedMutations", + "DeltaLakePartitionPrunedFiles", + "DictCacheKeysExpired", + "DictCacheKeysHit", + "DictCacheKeysNotFound", + "DictCacheKeysRequested", + "DictCacheKeysRequestedFound", + "DictCacheKeysRequestedMiss", + "DictCacheRequests", + "DirectorySync", + "DiskAzureCommitBlockList", + "DiskAzureCopyObject", + "DiskAzureCreateContainer", + "DiskAzureDeleteObjects", + "DiskAzureGetObject", + "DiskAzureGetProperties", + "DiskAzureGetRequestThrottlerCount", + "DiskAzureListObjects", + "DiskAzurePutRequestThrottlerCount", + "DiskAzureReadRequestsCount", + "DiskAzureReadRequestsErrors", + "DiskAzureReadRequestsRedirects", + "DiskAzureReadRequestsThrottling", + "DiskAzureStageBlock", + "DiskAzureUpload", + "DiskAzureWriteRequestsCount", + "DiskAzureWriteRequestsErrors", + "DiskAzureWriteRequestsRedirects", + "DiskAzureWriteRequestsThrottling", + "DiskConnectionsCreated", + "DiskConnectionsErrors", + "DiskConnectionsExpired", + "DiskConnectionsPreserved", + "DiskConnectionsReset", + "DiskConnectionsReused", + "DiskPlainRewritableAzureDirectoryCreated", + "DiskPlainRewritableAzureDirectoryRemoved", + "DiskPlainRewritableLegacyLayoutDiskCount", + "DiskPlainRewritableLocalDirectoryCreated", + "DiskPlainRewritableLocalDirectoryRemoved", + "DiskPlainRewritableS3DirectoryCreated", + "DiskPlainRewritableS3DirectoryRemoved", + "DiskS3AbortMultipartUpload", + "DiskS3CompleteMultipartUpload", + "DiskS3CopyObject", + "DiskS3CreateMultipartUpload", + "DiskS3DeleteObjects", + "DiskS3GetObject", + "DiskS3GetObjectAttributes", + "DiskS3GetRequestThrottlerCount", + "DiskS3HeadObject", + "DiskS3ListObjects", + "DiskS3PutObject", + "DiskS3PutRequestThrottlerCount", + "DiskS3ReadRequestAttempts", + "DiskS3ReadRequestRetryableErrors", + "DiskS3ReadRequestsCount", + "DiskS3ReadRequestsErrors", + "DiskS3ReadRequestsRedirects", + "DiskS3ReadRequestsThrottling", + "DiskS3UploadPart", + "DiskS3UploadPartCopy", + "DiskS3WriteRequestAttempts", + "DiskS3WriteRequestRetryableErrors", + "DiskS3WriteRequestsCount", + "DiskS3WriteRequestsErrors", + "DiskS3WriteRequestsRedirects", + "DiskS3WriteRequestsThrottling", + "DistrCacheConnectAttempts", + "DistrCacheDataPacketsBytes", + "DistrCacheHashRingRebuilds", + "DistrCacheIgnoredBytesWhileWaitingProfileEvents", + "DistrCacheMakeRequestErrors", + "DistrCacheOpenedConnections", + "DistrCacheOpenedConnectionsBypassingPool", + "DistrCachePackets", + "DistrCachePacketsBytes", + "DistrCacheRangeChange", + "DistrCacheRangeResetBackward", + "DistrCacheRangeResetForward", + "DistrCacheReadBytesFromCache", + "DistrCacheReadBytesFromFallbackBuffer", + "DistrCacheReadErrors", + "DistrCacheReceiveResponseErrors", + "DistrCacheReconnectsAfterTimeout", + "DistrCacheRegistryUpdates", + "DistrCacheReusedConnections", + "DistrCacheServerAckRequestPackets", + "DistrCacheServerCachedReadBufferCacheHits", + "DistrCacheServerCachedReadBufferCacheMisses", + "DistrCacheServerContinueRequestPackets", + "DistrCacheServerCredentialsRefresh", + "DistrCacheServerEndRequestPackets", + "DistrCacheServerNewS3CachedClients", + "DistrCacheServerReceivedCredentialsRefreshPackets", + "DistrCacheServerReusedS3CachedClients", + "DistrCacheServerStartRequestPackets", + "DistrCacheServerSwitches", + "DistrCacheServerUpdates", + "DistrCacheUnusedDataPacketsBytes", + "DistrCacheUnusedPackets", + "DistrCacheUnusedPacketsBufferAllocations", + "DistrCacheUnusedPacketsBytes", + "DistributedAsyncInsertionFailures", + "DistributedConnectionFailAtAll", + "DistributedConnectionFailTry", + "DistributedConnectionMissingTable", + "DistributedConnectionReconnectCount", + "DistributedConnectionSkipReadOnlyReplica", + "DistributedConnectionStaleReplica", + "DistributedConnectionTries", + "DistributedConnectionUsable", + "DistributedDelayedInserts", + "DistributedRejectedInserts", + "DistributedSyncInsertionTimeoutExceeded", + "DuplicatedInsertedBlocks", + "EngineFileLikeReadFiles", + "ExecuteShellCommand", + "ExternalAggregationCompressedBytes", + "ExternalAggregationMerge", + "ExternalAggregationUncompressedBytes", + "ExternalAggregationWritePart", + "ExternalDataSourceLocalCacheReadBytes", + "ExternalJoinCompressedBytes", + "ExternalJoinMerge", + "ExternalJoinUncompressedBytes", + "ExternalJoinWritePart", + "ExternalProcessingCompressedBytesTotal", + "ExternalProcessingFilesTotal", + "ExternalProcessingUncompressedBytesTotal", + "ExternalSortCompressedBytes", + "ExternalSortMerge", + "ExternalSortUncompressedBytes", + "ExternalSortWritePart", + "FailedAsyncInsertQuery", + "FailedInsertQuery", + "FailedQuery", + "FailedSelectQuery", + "FileOpen", + "FileSegmentFailToIncreasePriority", + "FileSegmentUsedBytes", + "FileSync", + "FilesystemCacheBackgroundDownloadQueuePush", + "FilesystemCacheBackgroundEvictedBytes", + "FilesystemCacheBackgroundEvictedFileSegments", + "FilesystemCacheCreatedKeyDirectories", + "FilesystemCacheEvictedBytes", + "FilesystemCacheEvictedFileSegments", + "FilesystemCacheEvictedFileSegmentsDuringPriorityIncrease", + "FilesystemCacheEvictionReusedIterator", + "FilesystemCacheEvictionSkippedEvictingFileSegments", + "FilesystemCacheEvictionSkippedFileSegments", + "FilesystemCacheEvictionTries", + "FilesystemCacheFailToReserveSpaceBecauseOfCacheResize", + "FilesystemCacheFailToReserveSpaceBecauseOfLockContention", + "FilesystemCacheFailedEvictionCandidates", + "FilesystemCacheFreeSpaceKeepingThreadRun", + "FilesystemCacheHoldFileSegments", + "FilesystemCacheReserveAttempts", + "FilesystemCacheUnusedHoldFileSegments", + "FilterTransformPassedBytes", + "FilterTransformPassedRows", + "FunctionExecute", + "GWPAsanAllocateFailed", + "GWPAsanAllocateSuccess", + "GWPAsanFree", + "GatheredColumns", + "GlobalThreadPoolExpansions", + "GlobalThreadPoolJobs", + "GlobalThreadPoolShrinks", + "HTTPConnectionsCreated", + "HTTPConnectionsErrors", + "HTTPConnectionsExpired", + "HTTPConnectionsPreserved", + "HTTPConnectionsReset", + "HTTPConnectionsReused", + "HTTPServerConnectionsClosed", + "HTTPServerConnectionsCreated", + "HTTPServerConnectionsExpired", + "HTTPServerConnectionsPreserved", + "HTTPServerConnectionsReset", + "HTTPServerConnectionsReused", + "HardPageFaults", + "HashJoinPreallocatedElementsInHashTables", + "HedgedRequestsChangeReplica", + "IOBufferAllocBytes", + "IOBufferAllocs", + "IOUringCQEsCompleted", + "IOUringCQEsFailed", + "IOUringSQEsResubmitsAsync", + "IOUringSQEsResubmitsSync", + "IOUringSQEsSubmitted", + "IcebergMetadataFilesCacheHits", + "IcebergMetadataFilesCacheMisses", + "IcebergMetadataFilesCacheWeightLost", + "IcebergMetadataReturnedObjectInfos", + "IcebergMinMaxIndexPrunedFiles", + "IcebergPartitionPrunedFiles", + "IcebergPartitionPrunnedFiles", + "IcebergTrivialCountOptimizationApplied", + "IcebergVersionHintUsed", + "IgnoredColdParts", + "IndexBinarySearchAlgorithm", + "IndexGenericExclusionSearchAlgorithm", + "InitialQuery", + "InsertQueriesWithSubqueries", + "InsertQuery", + "InsertedBytes", + "InsertedCompactParts", + "InsertedRows", + "InsertedWideParts", + "InterfaceHTTPReceiveBytes", + "InterfaceHTTPSendBytes", + "InterfaceInterserverReceiveBytes", + "InterfaceInterserverSendBytes", + "InterfaceMySQLReceiveBytes", + "InterfaceMySQLSendBytes", + "InterfaceNativeReceiveBytes", + "InterfaceNativeSendBytes", + "InterfacePostgreSQLReceiveBytes", + "InterfacePostgreSQLSendBytes", + "InterfacePrometheusReceiveBytes", + "InterfacePrometheusSendBytes", + "JoinBuildTableRowCount", + "JoinProbeTableRowCount", + "JoinResultRowCount", + "KafkaBackgroundReads", + "KafkaCommitFailures", + "KafkaCommits", + "KafkaConsumerErrors", + "KafkaDirectReads", + "KafkaMessagesFailed", + "KafkaMessagesPolled", + "KafkaMessagesProduced", + "KafkaMessagesRead", + "KafkaProducerErrors", + "KafkaProducerFlushes", + "KafkaRebalanceAssignments", + "KafkaRebalanceErrors", + "KafkaRebalanceRevocations", + "KafkaRowsRead", + "KafkaRowsRejected", + "KafkaRowsWritten", + "KafkaWrites", + "KeeperBatchMaxCount", + "KeeperBatchMaxTotalSize", + "KeeperCheckRequest", + "KeeperCommits", + "KeeperCommitsFailed", + "KeeperCreateRequest", + "KeeperExistsRequest", + "KeeperGetRequest", + "KeeperListRequest", + "KeeperLogsEntryReadFromCommitCache", + "KeeperLogsEntryReadFromFile", + "KeeperLogsEntryReadFromLatestCache", + "KeeperLogsPrefetchedEntries", + "KeeperMultiReadRequest", + "KeeperMultiRequest", + "KeeperPacketsReceived", + "KeeperPacketsSent", + "KeeperReadSnapshot", + "KeeperReconfigRequest", + "KeeperRemoveRequest", + "KeeperRequestRejectedDueToSoftMemoryLimitCount", + "KeeperRequestTotal", + "KeeperSaveSnapshot", + "KeeperSetRequest", + "KeeperSnapshotApplys", + "KeeperSnapshotApplysFailed", + "KeeperSnapshotCreations", + "KeeperSnapshotCreationsFailed", + "LoadedDataParts", + "LoadedMarksCount", + "LoadedMarksFiles", + "LoadedMarksMemoryBytes", + "LoadedPrimaryIndexBytes", + "LoadedPrimaryIndexFiles", + "LoadedPrimaryIndexRows", + "LoadingMarksTasksCanceled", + "LocalReadThrottlerBytes", + "LocalThreadPoolExpansions", + "LocalThreadPoolShrinks", + "LocalWriteThrottlerBytes", + "LogDebug", + "LogError", + "LogFatal", + "LogInfo", + "LogTest", + "LogTrace", + "LogWarning", + "MMappedFileCacheHits", + "MMappedFileCacheMisses", + "MainConfigLoads", + "MarkCacheEvictedBytes", + "MarkCacheEvictedFiles", + "MarkCacheEvictedMarks", + "MarkCacheHits", + "MarkCacheMisses", + "MemoryAllocatorPurge", + "MemoryWorkerRun", + "Merge", + "MergeSourceParts", + "MergeTreeAllRangesAnnouncementsSent", + "MergeTreeDataProjectionWriterBlocks", + "MergeTreeDataProjectionWriterBlocksAlreadySorted", + "MergeTreeDataProjectionWriterCompressedBytes", + "MergeTreeDataProjectionWriterRows", + "MergeTreeDataProjectionWriterUncompressedBytes", + "MergeTreeDataWriterBlocks", + "MergeTreeDataWriterBlocksAlreadySorted", + "MergeTreeDataWriterCompressedBytes", + "MergeTreeDataWriterRows", + "MergeTreeDataWriterUncompressedBytes", + "MergeTreeReadTaskRequestsReceived", + "MergeTreeReadTaskRequestsSent", + "MergedColumns", + "MergedIntoCompactParts", + "MergedIntoWideParts", + "MergedRows", + "MergedUncompressedBytes", + "MergerMutatorPartsInRangesForMergeCount", + "MergerMutatorRangesForMergeCount", + "MergerMutatorSelectRangePartsCount", + "MergesThrottlerBytes", + "MetadataFromKeeperBackgroundCleanupErrors", + "MetadataFromKeeperBackgroundCleanupObjects", + "MetadataFromKeeperBackgroundCleanupTransactions", + "MetadataFromKeeperCacheHit", + "MetadataFromKeeperCacheMiss", + "MetadataFromKeeperCleanupTransactionCommit", + "MetadataFromKeeperCleanupTransactionCommitRetry", + "MetadataFromKeeperIndividualOperations", + "MetadataFromKeeperOperations", + "MetadataFromKeeperReconnects", + "MetadataFromKeeperTransactionCommit", + "MetadataFromKeeperTransactionCommitRetry", + "MetadataFromKeeperUpdateCacheOneLevel", + "MutatedRows", + "MutatedUncompressedBytes", + "MutationAffectedRowsUpperBound", + "MutationAllPartColumns", + "MutationCreatedEmptyParts", + "MutationSomePartColumns", + "MutationTotalParts", + "MutationUntouchedParts", + "MutationsAppliedOnFlyInAllReadTasks", + "MutationsThrottlerBytes", + "NetworkReceiveBytes", + "NetworkSendBytes", + "NotCreatedLogEntryForMerge", + "NotCreatedLogEntryForMutation", + "OSReadBytes", + "OSReadChars", + "OSWriteBytes", + "OSWriteChars", + "ObjectStorageQueueCancelledFiles", + "ObjectStorageQueueCommitRequests", + "ObjectStorageQueueExceptionsDuringInsert", + "ObjectStorageQueueExceptionsDuringRead", + "ObjectStorageQueueFailedFiles", + "ObjectStorageQueueFailedToBatchSetProcessing", + "ObjectStorageQueueFilteredFiles", + "ObjectStorageQueueInsertIterations", + "ObjectStorageQueueListedFiles", + "ObjectStorageQueueProcessedFiles", + "ObjectStorageQueueProcessedRows", + "ObjectStorageQueueReadBytes", + "ObjectStorageQueueReadFiles", + "ObjectStorageQueueReadRows", + "ObjectStorageQueueRemovedObjects", + "ObjectStorageQueueSuccessfulCommits", + "ObjectStorageQueueTrySetProcessingFailed", + "ObjectStorageQueueTrySetProcessingRequests", + "ObjectStorageQueueTrySetProcessingSucceeded", + "ObjectStorageQueueUnsuccessfulCommits", + "ObsoleteReplicatedParts", + "OpenedFileCacheHits", + "OpenedFileCacheMisses", + "OverflowAny", + "OverflowBreak", + "OverflowThrow", + "PageCacheHits", + "PageCacheMisses", + "PageCacheOvercommitResize", + "PageCacheReadBytes", + "PageCacheResized", + "PageCacheWeightLost", + "ParallelReplicasAvailableCount", + "ParallelReplicasDeniedRequests", + "ParallelReplicasNumRequests", + "ParallelReplicasQueryCount", + "ParallelReplicasReadAssignedForStealingMarks", + "ParallelReplicasReadAssignedMarks", + "ParallelReplicasReadMarks", + "ParallelReplicasReadUnassignedMarks", + "ParallelReplicasUnavailableCount", + "ParallelReplicasUsedCount", + "ParquetDecodingTaskBatches", + "ParquetDecodingTasks", + "ParquetPrunedRowGroups", + "ParquetReadRowGroups", + "PatchesAcquireLockMicroseconds", + "PatchesAcquireLockTries", + "PatchesAppliedInAllReadTasks", + "PatchesJoinAppliedInAllReadTasks", + "PatchesMergeAppliedInAllReadTasks", + "PatchesReadUncompressedBytes", + "PerfAlignmentFaults", + "PerfBranchInstructions", + "PerfBranchMisses", + "PerfBusCycles", + "PerfCPUClock", + "PerfCPUCycles", + "PerfCPUMigrations", + "PerfCacheMisses", + "PerfCacheReferences", + "PerfContextSwitches", + "PerfDataTLBMisses", + "PerfDataTLBReferences", + "PerfEmulationFaults", + "PerfInstructionTLBMisses", + "PerfInstructionTLBReferences", + "PerfInstructions", + "PerfLocalMemoryMisses", + "PerfLocalMemoryReferences", + "PerfMinEnabledRunningTime", + "PerfMinEnabledTime", + "PerfRefCPUCycles", + "PerfStalledCyclesBackend", + "PerfStalledCyclesFrontend", + "PerfTaskClock", + "PolygonsAddedToPool", + "PolygonsInPoolAllocatedBytes", + "PreferredWarmedUnmergedParts", + "PrimaryIndexCacheHits", + "PrimaryIndexCacheMisses", + "QueriesWithSubqueries", + "Query", + "QueryBackupThrottlerBytes", + "QueryCacheHits", + "QueryCacheMisses", + "QueryConditionCacheHits", + "QueryConditionCacheMisses", + "QueryLocalReadThrottlerBytes", + "QueryLocalWriteThrottlerBytes", + "QueryMaskingRulesMatch", + "QueryMemoryLimitExceeded", + "QueryPreempted", + "QueryProfilerConcurrencyOverruns", + "QueryProfilerErrors", + "QueryProfilerRuns", + "QueryProfilerSignalOverruns", + "QueryRemoteReadThrottlerBytes", + "QueryRemoteWriteThrottlerBytes", + "RWLockAcquiredReadLocks", + "RWLockAcquiredWriteLocks", + "ReadBackoff", + "ReadBufferFromAzureBytes", + "ReadBufferFromAzureRequestsErrors", + "ReadBufferFromFileDescriptorRead", + "ReadBufferFromFileDescriptorReadBytes", + "ReadBufferFromFileDescriptorReadFailed", + "ReadBufferFromS3Bytes", + "ReadBufferFromS3RequestsErrors", + "ReadBufferSeekCancelConnection", + "ReadCompressedBytes", + "ReadPatchesMicroseconds", + "ReadTaskRequestsReceived", + "ReadTaskRequestsSent", + "ReadTasksWithAppliedMutationsOnFly", + "ReadTasksWithAppliedPatches", + "ReadWriteBufferFromHTTPBytes", + "ReadWriteBufferFromHTTPRequestsSent", + "RefreshableViewLockTableRetry", + "RefreshableViewRefreshFailed", + "RefreshableViewRefreshSuccess", + "RefreshableViewSyncReplicaRetry", + "RefreshableViewSyncReplicaSuccess", + "RegexpLocalCacheHit", + "RegexpLocalCacheMiss", + "RegexpWithMultipleNeedlesCreated", + "RegexpWithMultipleNeedlesGlobalCacheHit", + "RegexpWithMultipleNeedlesGlobalCacheMiss", + "RejectedInserts", + "RejectedLightweightUpdates", + "RejectedMutations", + "RemoteFSBuffers", + "RemoteFSCancelledPrefetches", + "RemoteFSLazySeeks", + "RemoteFSPrefetchedBytes", + "RemoteFSPrefetchedReads", + "RemoteFSPrefetches", + "RemoteFSSeeks", + "RemoteFSSeeksWithReset", + "RemoteFSUnprefetchedBytes", + "RemoteFSUnprefetchedReads", + "RemoteFSUnusedPrefetches", + "RemoteReadThrottlerBytes", + "RemoteWriteThrottlerBytes", + "ReplicaPartialShutdown", + "ReplicatedCoveredPartsInZooKeeperOnStart", + "ReplicatedDataLoss", + "ReplicatedPartChecks", + "ReplicatedPartChecksFailed", + "ReplicatedPartFailedFetches", + "ReplicatedPartFetches", + "ReplicatedPartFetchesOfMerged", + "ReplicatedPartMerges", + "ReplicatedPartMutations", + "RestorePartsSkippedBytes", + "RestorePartsSkippedFiles", + "RowsReadByMainReader", + "RowsReadByPrewhereReaders", + "S3AbortMultipartUpload", + "S3Clients", + "S3CompleteMultipartUpload", + "S3CopyObject", + "S3CreateMultipartUpload", + "S3DeleteObjects", + "S3GetObject", + "S3GetObjectAttributes", + "S3GetRequestThrottlerCount", + "S3HeadObject", + "S3ListObjects", + "S3PutObject", + "S3PutRequestThrottlerCount", + "S3ReadRequestAttempts", + "S3ReadRequestRetryableErrors", + "S3ReadRequestsCount", + "S3ReadRequestsErrors", + "S3ReadRequestsRedirects", + "S3ReadRequestsThrottling", + "S3UploadPart", + "S3UploadPartCopy", + "S3WriteRequestAttempts", + "S3WriteRequestRetryableErrors", + "S3WriteRequestsCount", + "S3WriteRequestsErrors", + "S3WriteRequestsRedirects", + "S3WriteRequestsThrottling", + "ScalarSubqueriesCacheMiss", + "ScalarSubqueriesGlobalCacheHit", + "ScalarSubqueriesLocalCacheHit", + "SchedulerIOReadBytes", + "SchedulerIOReadRequests", + "SchedulerIOWriteBytes", + "SchedulerIOWriteRequests", + "SchemaInferenceCacheEvictions", + "SchemaInferenceCacheHits", + "SchemaInferenceCacheInvalidations", + "SchemaInferenceCacheMisses", + "SchemaInferenceCacheNumRowsHits", + "SchemaInferenceCacheNumRowsMisses", + "SchemaInferenceCacheSchemaHits", + "SchemaInferenceCacheSchemaMisses", + "Seek", + "SelectQueriesWithPrimaryKeyUsage", + "SelectQueriesWithSubqueries", + "SelectQuery", + "SelectedBytes", + "SelectedMarks", + "SelectedMarksTotal", + "SelectedParts", + "SelectedPartsTotal", + "SelectedRanges", + "SelectedRows", + "SharedDatabaseCatalogFailedToApplyState", + "SharedMergeTreeCondemnedPartsKillRequest", + "SharedMergeTreeCondemnedPartsLockConfict", + "SharedMergeTreeCondemnedPartsRemoved", + "SharedMergeTreeDataPartsFetchAttempt", + "SharedMergeTreeDataPartsFetchFromPeer", + "SharedMergeTreeDataPartsFetchFromPeerMicroseconds", + "SharedMergeTreeDataPartsFetchFromS3", + "SharedMergeTreeGetPartsBatchToLoadMicroseconds", + "SharedMergeTreeHandleBlockingParts", + "SharedMergeTreeHandleBlockingPartsMicroseconds", + "SharedMergeTreeHandleFetchPartsMicroseconds", + "SharedMergeTreeHandleOutdatedParts", + "SharedMergeTreeHandleOutdatedPartsMicroseconds", + "SharedMergeTreeLoadChecksumAndIndexesMicroseconds", + "SharedMergeTreeMergeMutationAssignmentAttempt", + "SharedMergeTreeMergeMutationAssignmentFailedWithConflict", + "SharedMergeTreeMergeMutationAssignmentFailedWithNothingToDo", + "SharedMergeTreeMergeMutationAssignmentSuccessful", + "SharedMergeTreeMergePartsMovedToCondemned", + "SharedMergeTreeMergePartsMovedToOudated", + "SharedMergeTreeMergeSelectingTaskMicroseconds", + "SharedMergeTreeMetadataCacheHintLoadedFromCache", + "SharedMergeTreeOptimizeAsync", + "SharedMergeTreeOptimizeSync", + "SharedMergeTreeOutdatedPartsConfirmationInvocations", + "SharedMergeTreeOutdatedPartsConfirmationRequest", + "SharedMergeTreeOutdatedPartsHTTPRequest", + "SharedMergeTreeOutdatedPartsHTTPResponse", + "SharedMergeTreeScheduleDataProcessingJob", + "SharedMergeTreeScheduleDataProcessingJobMicroseconds", + "SharedMergeTreeScheduleDataProcessingJobNothingToScheduled", + "SharedMergeTreeTryUpdateDiskMetadataCacheForPartMicroseconds", + "SharedMergeTreeVirtualPartsUpdates", + "SharedMergeTreeVirtualPartsUpdatesByLeader", + "SharedMergeTreeVirtualPartsUpdatesForMergesOrStatus", + "SharedMergeTreeVirtualPartsUpdatesFromPeer", + "SharedMergeTreeVirtualPartsUpdatesFromZooKeeper", + "SharedMergeTreeVirtualPartsUpdatesLeaderFailedElection", + "SharedMergeTreeVirtualPartsUpdatesLeaderSuccessfulElection", + "SharedMergeTreeVirtualPartsUpdatesPeerNotFound", + "SleepFunctionCalls", + "SlowRead", + "SoftPageFaults", + "StorageBufferErrorOnFlush", + "StorageBufferFlush", + "StorageBufferPassedAllMinThresholds", + "StorageBufferPassedBytesFlushThreshold", + "StorageBufferPassedBytesMaxThreshold", + "StorageBufferPassedRowsFlushThreshold", + "StorageBufferPassedRowsMaxThreshold", + "StorageBufferPassedTimeFlushThreshold", + "StorageBufferPassedTimeMaxThreshold", + "StorageConnectionsCreated", + "StorageConnectionsErrors", + "StorageConnectionsExpired", + "StorageConnectionsPreserved", + "StorageConnectionsReset", + "StorageConnectionsReused", + "SuspendSendingQueryToShard", + "SystemLogErrorOnFlush", + "TableFunctionExecute", + "ThreadPoolReaderPageCacheHit", + "ThreadPoolReaderPageCacheHitBytes", + "ThreadPoolReaderPageCacheMiss", + "ThreadPoolReaderPageCacheMissBytes", + "ThreadpoolReaderReadBytes", + "ThreadpoolReaderSubmit", + "ThreadpoolReaderSubmitReadSynchronously", + "ThreadpoolReaderSubmitReadSynchronouslyBytes", + "TinyS3Clients", + "USearchAddComputedDistances", + "USearchAddCount", + "USearchAddVisitedMembers", + "USearchSearchComputedDistances", + "USearchSearchCount", + "USearchSearchVisitedMembers", + "UncompressedCacheHits", + "UncompressedCacheMisses", + "UncompressedCacheWeightLost", + "VectorSimilarityIndexCacheHits", + "VectorSimilarityIndexCacheMisses", + "VectorSimilarityIndexCacheWeightLost", + "WriteBufferFromFileDescriptorWrite", + "WriteBufferFromFileDescriptorWriteBytes", + "WriteBufferFromFileDescriptorWriteFailed", + "WriteBufferFromHTTPBytes", + "WriteBufferFromHTTPRequestsSent", + "WriteBufferFromS3Bytes", + "WriteBufferFromS3RequestsErrors", + "ZooKeeperBytesReceived", + "ZooKeeperBytesSent", + "ZooKeeperCheck", + "ZooKeeperClose", + "ZooKeeperCreate", + "ZooKeeperExists", + "ZooKeeperGet", + "ZooKeeperGetACL", + "ZooKeeperHardwareExceptions", + "ZooKeeperInit", + "ZooKeeperList", + "ZooKeeperMulti", + "ZooKeeperMultiRead", + "ZooKeeperMultiWrite", + "ZooKeeperOtherExceptions", + "ZooKeeperReconfig", + "ZooKeeperRemove", + "ZooKeeperSet", + "ZooKeeperSync", + "ZooKeeperTransactions", + "ZooKeeperUserExceptions", + "ZooKeeperWatchResponse" + ], + "temporal_percent": { + "AggregatingSortedMilliseconds": "millisecond", + "AsyncLoaderWaitMicroseconds": "microsecond", + "AsynchronousReadWaitMicroseconds": "microsecond", + "AsynchronousRemoteReadWaitMicroseconds": "microsecond", + "AzureGetRequestThrottlerSleepMicroseconds": "microsecond", + "AzurePutRequestThrottlerSleepMicroseconds": "microsecond", + "AzureReadMicroseconds": "microsecond", + "AzureWriteMicroseconds": "microsecond", + "BackupEntriesCollectorForTablesDataMicroseconds": "microsecond", + "BackupEntriesCollectorMicroseconds": "microsecond", + "BackupEntriesCollectorRunPostTasksMicroseconds": "microsecond", + "BackupPreparingFileInfosMicroseconds": "microsecond", + "BackupReadMetadataMicroseconds": "microsecond", + "BackupThrottlerSleepMicroseconds": "microsecond", + "BackupWriteMetadataMicroseconds": "microsecond", + "CachedReadBufferCacheWriteMicroseconds": "microsecond", + "CachedReadBufferCreateBufferMicroseconds": "microsecond", + "CachedReadBufferReadFromCacheMicroseconds": "microsecond", + "CachedReadBufferReadFromSourceMicroseconds": "microsecond", + "CachedReadBufferWaitReadBufferMicroseconds": "microsecond", + "CachedWriteBufferCacheWriteMicroseconds": "microsecond", + "CoalescingSortedMilliseconds": "millisecond", + "CollapsingSortedMilliseconds": "millisecond", + "CommonBackgroundExecutorTaskCancelMicroseconds": "microsecond", + "CommonBackgroundExecutorTaskExecuteStepMicroseconds": "microsecond", + "CommonBackgroundExecutorTaskResetMicroseconds": "microsecond", + "CommonBackgroundExecutorWaitMicroseconds": "microsecond", + "CompileExpressionsMicroseconds": "microsecond", + "CompressedReadBufferChecksumDoesntMatchMicroseconds": "microsecond", + "ConcurrencyControlPreemptedMicroseconds": "microsecond", + "ConcurrencyControlWaitMicroseconds": "microsecond", + "ConcurrentQueryWaitMicroseconds": "microsecond", + "ConnectionPoolIsFullMicroseconds": "microsecond", + "ContextLockWaitMicroseconds": "microsecond", + "CoordinatedMergesMergeAssignmentRequestMicroseconds": "microsecond", + "CoordinatedMergesMergeAssignmentResponseMicroseconds": "microsecond", + "CoordinatedMergesMergeCoordinatorFetchMetadataMicroseconds": "microsecond", + "CoordinatedMergesMergeCoordinatorFilterMicroseconds": "microsecond", + "CoordinatedMergesMergeCoordinatorLockStateExclusivelyMicroseconds": "microsecond", + "CoordinatedMergesMergeCoordinatorLockStateForShareMicroseconds": "microsecond", + "CoordinatedMergesMergeCoordinatorSelectMergesMicroseconds": "microsecond", + "CoordinatedMergesMergeCoordinatorUpdateMicroseconds": "microsecond", + "CoordinatedMergesMergeWorkerUpdateMicroseconds": "microsecond", + "DelayedInsertsMilliseconds": "millisecond", + "DelayedMutationsMilliseconds": "millisecond", + "DictCacheLockReadNs": "nanosecond", + "DictCacheLockWriteNs": "nanosecond", + "DictCacheRequestTimeNs": "nanosecond", + "DirectorySyncElapsedMicroseconds": "microsecond", + "DiskAzureGetRequestThrottlerSleepMicroseconds": "microsecond", + "DiskAzurePutRequestThrottlerSleepMicroseconds": "microsecond", + "DiskAzureReadMicroseconds": "microsecond", + "DiskAzureWriteMicroseconds": "microsecond", + "DiskConnectionsElapsedMicroseconds": "microsecond", + "DiskReadElapsedMicroseconds": "microsecond", + "DiskS3GetRequestThrottlerSleepMicroseconds": "microsecond", + "DiskS3PutRequestThrottlerSleepMicroseconds": "microsecond", + "DiskS3ReadMicroseconds": "microsecond", + "DiskS3WriteMicroseconds": "microsecond", + "DiskWriteElapsedMicroseconds": "microsecond", + "DistrCacheConnectMicroseconds": "microsecond", + "DistrCacheFallbackReadMicroseconds": "microsecond", + "DistrCacheGetClientMicroseconds": "microsecond", + "DistrCacheGetResponseMicroseconds": "microsecond", + "DistrCacheLockRegistryMicroseconds": "microsecond", + "DistrCacheNextImplMicroseconds": "microsecond", + "DistrCachePrecomputeRangesMicroseconds": "microsecond", + "DistrCacheReadMicroseconds": "microsecond", + "DistrCacheRegistryUpdateMicroseconds": "microsecond", + "DistrCacheServerProcessRequestMicroseconds": "microsecond", + "DistrCacheStartRangeMicroseconds": "microsecond", + "DistributedDelayedInsertsMilliseconds": "millisecond", + "FetchBackgroundExecutorTaskCancelMicroseconds": "microsecond", + "FetchBackgroundExecutorTaskExecuteStepMicroseconds": "microsecond", + "FetchBackgroundExecutorTaskResetMicroseconds": "microsecond", + "FetchBackgroundExecutorWaitMicroseconds": "microsecond", + "FileSegmentCacheWriteMicroseconds": "microsecond", + "FileSegmentCompleteMicroseconds": "microsecond", + "FileSegmentHolderCompleteMicroseconds": "microsecond", + "FileSegmentLockMicroseconds": "microsecond", + "FileSegmentPredownloadMicroseconds": "microsecond", + "FileSegmentReadMicroseconds": "microsecond", + "FileSegmentRemoveMicroseconds": "microsecond", + "FileSegmentUseMicroseconds": "microsecond", + "FileSegmentWaitMicroseconds": "microsecond", + "FileSegmentWaitReadBufferMicroseconds": "microsecond", + "FileSegmentWriteMicroseconds": "microsecond", + "FileSyncElapsedMicroseconds": "microsecond", + "FilesystemCacheEvictMicroseconds": "microsecond", + "FilesystemCacheFreeSpaceKeepingThreadWorkMilliseconds": "millisecond", + "FilesystemCacheGetMicroseconds": "microsecond", + "FilesystemCacheGetOrSetMicroseconds": "microsecond", + "FilesystemCacheLoadMetadataMicroseconds": "microsecond", + "FilesystemCacheLockCacheMicroseconds": "microsecond", + "FilesystemCacheLockKeyMicroseconds": "microsecond", + "FilesystemCacheLockMetadataMicroseconds": "microsecond", + "FilesystemCacheReserveMicroseconds": "microsecond", + "FilteringMarksWithPrimaryKeyMicroseconds": "microsecond", + "FilteringMarksWithSecondaryKeysMicroseconds": "microsecond", + "GatheringColumnMilliseconds": "millisecond", + "GlobalThreadPoolJobWaitTimeMicroseconds": "microsecond", + "GlobalThreadPoolLockWaitMicroseconds": "microsecond", + "GlobalThreadPoolThreadCreationMicroseconds": "microsecond", + "HTTPConnectionsElapsedMicroseconds": "microsecond", + "IcebergIteratorInitializationMicroseconds": "microsecond", + "IcebergMetadataReadWaitTimeMicroseconds": "microsecond", + "IcebergMetadataUpdateMicroseconds": "microsecond", + "InsertQueryTimeMicroseconds": "microsecond", + "KeeperCommitWaitElapsedMicroseconds": "microsecond", + "KeeperLatency": "millisecond", + "KeeperPreprocessElapsedMicroseconds": "microsecond", + "KeeperProcessElapsedMicroseconds": "microsecond", + "KeeperStorageLockWaitMicroseconds": "microsecond", + "KeeperTotalElapsedMicroseconds": "microsecond", + "LoadedDataPartsMicroseconds": "microsecond", + "LocalReadThrottlerSleepMicroseconds": "microsecond", + "LocalThreadPoolBusyMicroseconds": "microsecond", + "LocalThreadPoolJobWaitTimeMicroseconds": "microsecond", + "LocalThreadPoolJobs": "microsecond", + "LocalThreadPoolLockWaitMicroseconds": "microsecond", + "LocalThreadPoolThreadCreationMicroseconds": "microsecond", + "LocalWriteThrottlerSleepMicroseconds": "microsecond", + "LoggerElapsedNanoseconds": "nanosecond", + "MemoryAllocatorPurgeTimeMicroseconds": "microsecond", + "MemoryOvercommitWaitTimeMicroseconds": "microsecond", + "MemoryWorkerRunElapsedMicroseconds": "microsecond", + "MergeExecuteMilliseconds": "millisecond", + "MergeHorizontalStageExecuteMilliseconds": "millisecond", + "MergeHorizontalStageTotalMilliseconds": "millisecond", + "MergeMutateBackgroundExecutorTaskCancelMicroseconds": "microsecond", + "MergeMutateBackgroundExecutorTaskExecuteStepMicroseconds": "microsecond", + "MergeMutateBackgroundExecutorTaskResetMicroseconds": "microsecond", + "MergeMutateBackgroundExecutorWaitMicroseconds": "microsecond", + "MergePrewarmStageExecuteMilliseconds": "millisecond", + "MergePrewarmStageTotalMilliseconds": "millisecond", + "MergeProjectionStageExecuteMilliseconds": "millisecond", + "MergeProjectionStageTotalMilliseconds": "millisecond", + "MergeTotalMilliseconds": "millisecond", + "MergeTreeAllRangesAnnouncementsSentElapsedMicroseconds": "microsecond", + "MergeTreeDataProjectionWriterMergingBlocksMicroseconds": "microsecond", + "MergeTreeDataProjectionWriterSortingBlocksMicroseconds": "microsecond", + "MergeTreeDataWriterMergingBlocksMicroseconds": "microsecond", + "MergeTreeDataWriterProjectionsCalculationMicroseconds": "microsecond", + "MergeTreeDataWriterSkipIndicesCalculationMicroseconds": "microsecond", + "MergeTreeDataWriterSortingBlocksMicroseconds": "microsecond", + "MergeTreeDataWriterStatisticsCalculationMicroseconds": "microsecond", + "MergeTreePrefetchedReadPoolInit": "microsecond", + "MergeTreeReadTaskRequestsSentElapsedMicroseconds": "microsecond", + "MergeVerticalStageExecuteMilliseconds": "millisecond", + "MergeVerticalStageTotalMilliseconds": "millisecond", + "MergerMutatorPrepareRangesForMergeElapsedMicroseconds": "microsecond", + "MergerMutatorSelectPartsForMergeElapsedMicroseconds": "microsecond", + "MergerMutatorsGetPartsForMergeElapsedMicroseconds": "microsecond", + "MergesThrottlerSleepMicroseconds": "microsecond", + "MergingSortedMilliseconds": "millisecond", + "MetadataFromKeeperCacheUpdateMicroseconds": "microsecond", + "MetadataFromKeeperIndividualOperationsMicroseconds": "microsecond", + "MoveBackgroundExecutorTaskCancelMicroseconds": "microsecond", + "MoveBackgroundExecutorTaskExecuteStepMicroseconds": "microsecond", + "MoveBackgroundExecutorTaskResetMicroseconds": "microsecond", + "MoveBackgroundExecutorWaitMicroseconds": "microsecond", + "MutateTaskProjectionsCalculationMicroseconds": "microsecond", + "MutationExecuteMilliseconds": "millisecond", + "MutationTotalMilliseconds": "millisecond", + "MutationsThrottlerSleepMicroseconds": "microsecond", + "NetworkReceiveElapsedMicroseconds": "microsecond", + "NetworkSendElapsedMicroseconds": "microsecond", + "OSCPUVirtualTimeMicroseconds": "microsecond", + "OSCPUWaitMicroseconds": "microsecond", + "OSIOWaitMicroseconds": "microsecond", + "ObjectStorageQueueCleanupMaxSetSizeOrTTLMicroseconds": "microsecond", + "ObjectStorageQueueLockLocalFileStatusesMicroseconds": "microsecond", + "ObjectStorageQueuePullMicroseconds": "microsecond", + "OpenedFileCacheMicroseconds": "microsecond", + "OtherQueryTimeMicroseconds": "microsecond", + "ParallelReplicasAnnouncementMicroseconds": "microsecond", + "ParallelReplicasCollectingOwnedSegmentsMicroseconds": "microsecond", + "ParallelReplicasHandleAnnouncementMicroseconds": "microsecond", + "ParallelReplicasHandleRequestMicroseconds": "microsecond", + "ParallelReplicasProcessingPartsMicroseconds": "microsecond", + "ParallelReplicasReadRequestMicroseconds": "microsecond", + "ParallelReplicasStealingByHashMicroseconds": "microsecond", + "ParallelReplicasStealingLeftoversMicroseconds": "microsecond", + "ParquetFetchWaitTimeMicroseconds": "microsecond", + "PartsLockHoldMicroseconds": "microsecond", + "PartsLockWaitMicroseconds": "microsecond", + "QueryBackupThrottlerSleepMicroseconds": "microsecond", + "QueryLocalReadThrottlerSleepMicroseconds": "microsecond", + "QueryLocalWriteThrottlerSleepMicroseconds": "microsecond", + "QueryRemoteReadThrottlerSleepMicroseconds": "microsecond", + "QueryRemoteWriteThrottlerSleepMicroseconds": "microsecond", + "QueryTimeMicroseconds": "microsecond", + "RWLockReadersWaitMilliseconds": "millisecond", + "RWLockWritersWaitMilliseconds": "millisecond", + "ReadBufferFromAzureInitMicroseconds": "microsecond", + "ReadBufferFromAzureMicroseconds": "microsecond", + "ReadBufferFromS3InitMicroseconds": "microsecond", + "ReadBufferFromS3Microseconds": "microsecond", + "ReadTaskRequestsSentElapsedMicroseconds": "microsecond", + "RealTimeMicroseconds": "microsecond", + "RemoteReadThrottlerSleepMicroseconds": "microsecond", + "RemoteWriteThrottlerSleepMicroseconds": "microsecond", + "ReplacingSortedMilliseconds": "millisecond", + "S3GetRequestThrottlerSleepMicroseconds": "microsecond", + "S3PutRequestThrottlerSleepMicroseconds": "microsecond", + "S3QueueSetFileFailedMicroseconds": "microsecond", + "S3QueueSetFileProcessedMicroseconds": "microsecond", + "S3QueueSetFileProcessingMicroseconds": "microsecond", + "S3ReadMicroseconds": "microsecond", + "S3WriteMicroseconds": "microsecond", + "SchedulerIOReadWaitMicroseconds": "microsecond", + "SchedulerIOWriteWaitMicroseconds": "microsecond", + "SelectQueryTimeMicroseconds": "microsecond", + "ServerStartupMilliseconds": "millisecond", + "SharedDatabaseCatalogStateApplicationMicroseconds": "microsecond", + "SharedMergeTreeVirtualPartsUpdateMicroseconds": "microsecond", + "SharedMergeTreeVirtualPartsUpdatesFromPeerMicroseconds": "microsecond", + "SharedMergeTreeVirtualPartsUpdatesFromZooKeeperMicroseconds": "microsecond", + "SleepFunctionElapsedMicroseconds": "microsecond", + "SleepFunctionMicroseconds": "microsecond", + "StorageBufferLayerLockReadersWaitMilliseconds": "millisecond", + "StorageBufferLayerLockWritersWaitMilliseconds": "millisecond", + "StorageConnectionsElapsedMicroseconds": "microsecond", + "SummingSortedMilliseconds": "millisecond", + "SynchronousReadWaitMicroseconds": "microsecond", + "SynchronousRemoteReadWaitMicroseconds": "microsecond", + "SystemTimeMicroseconds": "microsecond", + "ThreadPoolReaderPageCacheHitElapsedMicroseconds": "microsecond", + "ThreadPoolReaderPageCacheMissElapsedMicroseconds": "microsecond", + "ThreadpoolReaderPrepareMicroseconds": "microsecond", + "ThreadpoolReaderSubmitLookupInCacheMicroseconds": "microsecond", + "ThreadpoolReaderSubmitReadSynchronouslyMicroseconds": "microsecond", + "ThreadpoolReaderTaskMicroseconds": "microsecond", + "ThrottlerSleepMicroseconds": "microsecond", + "UserTimeMicroseconds": "microsecond", + "VersionedCollapsingSortedMilliseconds": "millisecond", + "WaitMarksLoadMicroseconds": "microsecond", + "WaitPrefetchTaskMicroseconds": "microsecond", + "WriteBufferFromS3Microseconds": "microsecond", + "WriteBufferFromS3WaitInflightLimitMicroseconds": "microsecond", + "ZooKeeperWaitMicroseconds": "microsecond" + } + } +} diff --git a/clickhouse/datadog_checks/clickhouse/data/system_metrics.json b/clickhouse/datadog_checks/clickhouse/data/system_metrics.json new file mode 100644 index 0000000000000..116fc08c266d0 --- /dev/null +++ b/clickhouse/datadog_checks/clickhouse/data/system_metrics.json @@ -0,0 +1,441 @@ +{ + "name": "system_metrics", + "query": "SELECT value, metric FROM system.metrics", + "value_column": "metric_value", + "match_column": "metric_name", + "prefix": "metrics", + "items": { + "gauge": [ + "ActiveTimersInQueryProfiler", + "AddressesActive", + "AddressesBanned", + "AggregatorThreads", + "AggregatorThreadsActive", + "AggregatorThreadsScheduled", + "AsyncInsertCacheSize", + "AsynchronousInsertQueueBytes", + "AsynchronousInsertQueueSize", + "AsynchronousInsertThreads", + "AsynchronousInsertThreadsActive", + "AsynchronousInsertThreadsScheduled", + "AsynchronousReadWait", + "AttachedDatabase", + "AttachedDictionary", + "AttachedReplicatedTable", + "AttachedTable", + "AttachedView", + "AvroSchemaCacheBytes", + "AvroSchemaCacheCells", + "AvroSchemaRegistryCacheBytes", + "AvroSchemaRegistryCacheCells", + "AzureRequests", + "BackgroundBufferFlushSchedulePoolSize", + "BackgroundBufferFlushSchedulePoolTask", + "BackgroundCommonPoolSize", + "BackgroundCommonPoolTask", + "BackgroundDistributedSchedulePoolSize", + "BackgroundDistributedSchedulePoolTask", + "BackgroundFetchesPoolSize", + "BackgroundFetchesPoolTask", + "BackgroundMergesAndMutationsPoolSize", + "BackgroundMergesAndMutationsPoolTask", + "BackgroundMessageBrokerSchedulePoolSize", + "BackgroundMessageBrokerSchedulePoolTask", + "BackgroundMovePoolSize", + "BackgroundMovePoolTask", + "BackgroundSchedulePoolSize", + "BackgroundSchedulePoolTask", + "BackupsIOThreads", + "BackupsIOThreadsActive", + "BackupsIOThreadsScheduled", + "BackupsThreads", + "BackupsThreadsActive", + "BackupsThreadsScheduled", + "BrokenDisks", + "BrokenDistributedBytesToInsert", + "BrokenDistributedFilesToInsert", + "BuildVectorSimilarityIndexThreads", + "BuildVectorSimilarityIndexThreadsActive", + "BuildVectorSimilarityIndexThreadsScheduled", + "CacheDetachedFileSegments", + "CacheDictionaryThreads", + "CacheDictionaryThreadsActive", + "CacheDictionaryThreadsScheduled", + "CacheDictionaryUpdateQueueBatches", + "CacheDictionaryUpdateQueueKeys", + "CacheFileSegments", + "CacheWarmerBytesInProgress", + "CompiledExpressionCacheBytes", + "CompiledExpressionCacheCount", + "Compressing", + "CompressionThread", + "CompressionThreadActive", + "CompressionThreadScheduled", + "ConcurrencyControlAcquired", + "ConcurrencyControlAcquiredNonCompeting", + "ConcurrencyControlPreempted", + "ConcurrencyControlScheduled", + "ConcurrencyControlSoftLimit", + "ConcurrentHashJoinPoolThreads", + "ConcurrentHashJoinPoolThreadsActive", + "ConcurrentHashJoinPoolThreadsScheduled", + "ConcurrentQueryAcquired", + "ConcurrentQueryScheduled", + "ContextLockWait", + "CoordinatedMergesCoordinatorAssignedMerges", + "CoordinatedMergesCoordinatorRunningMerges", + "CoordinatedMergesWorkerAssignedMerges", + "CreatedTimersInQueryProfiler", + "DDLWorkerThreads", + "DDLWorkerThreadsActive", + "DDLWorkerThreadsScheduled", + "DNSAddressesCacheBytes", + "DNSAddressesCacheSize", + "DNSHostsCacheBytes", + "DNSHostsCacheSize", + "DWARFReaderThreads", + "DWARFReaderThreadsActive", + "DWARFReaderThreadsScheduled", + "DatabaseBackupThreads", + "DatabaseBackupThreadsActive", + "DatabaseBackupThreadsScheduled", + "DatabaseCatalogThreads", + "DatabaseCatalogThreadsActive", + "DatabaseCatalogThreadsScheduled", + "DatabaseOnDiskThreads", + "DatabaseOnDiskThreadsActive", + "DatabaseOnDiskThreadsScheduled", + "DatabaseReplicatedCreateTablesThreads", + "DatabaseReplicatedCreateTablesThreadsActive", + "DatabaseReplicatedCreateTablesThreadsScheduled", + "Decompressing", + "DelayedInserts", + "DestroyAggregatesThreads", + "DestroyAggregatesThreadsActive", + "DestroyAggregatesThreadsScheduled", + "DictCacheRequests", + "DiskConnectionsStored", + "DiskConnectionsTotal", + "DiskObjectStorageAsyncThreads", + "DiskObjectStorageAsyncThreadsActive", + "DiskPlainRewritableAzureDirectoryMapSize", + "DiskPlainRewritableAzureFileCount", + "DiskPlainRewritableAzureUniqueFileNamesCount", + "DiskPlainRewritableLocalDirectoryMapSize", + "DiskPlainRewritableLocalFileCount", + "DiskPlainRewritableLocalUniqueFileNamesCount", + "DiskPlainRewritableS3DirectoryMapSize", + "DiskPlainRewritableS3FileCount", + "DiskPlainRewritableS3UniqueFileNamesCount", + "DiskS3NoSuchKeyErrors", + "DiskSpaceReservedForMerge", + "DistrCacheAllocatedConnections", + "DistrCacheBorrowedConnections", + "DistrCacheOpenedConnections", + "DistrCacheReadRequests", + "DistrCacheRegisteredServers", + "DistrCacheRegisteredServersCurrentAZ", + "DistrCacheServerConnections", + "DistrCacheServerRegistryConnections", + "DistrCacheServerS3CachedClients", + "DistrCacheUsedConnections", + "DistrCacheWriteRequests", + "DistributedBytesToInsert", + "DistributedFilesToInsert", + "DistributedInsertThreads", + "DistributedInsertThreadsActive", + "DistributedInsertThreadsScheduled", + "DistributedSend", + "DropDistributedCacheThreads", + "DropDistributedCacheThreadsActive", + "DropDistributedCacheThreadsScheduled", + "EphemeralNode", + "FilesystemCacheDelayedCleanupElements", + "FilesystemCacheDownloadQueueElements", + "FilesystemCacheElements", + "FilesystemCacheHoldFileSegments", + "FilesystemCacheKeys", + "FilesystemCacheReadBuffers", + "FilesystemCacheReserveThreads", + "FilesystemCacheSize", + "FilesystemCacheSizeLimit", + "FilteringMarksWithPrimaryKey", + "FilteringMarksWithSecondaryKeys", + "FormatParsingThreads", + "FormatParsingThreadsActive", + "FormatParsingThreadsScheduled", + "GlobalThread", + "GlobalThreadActive", + "GlobalThreadScheduled", + "HTTPConnection", + "HTTPConnectionsStored", + "HTTPConnectionsTotal", + "HashedDictionaryThreads", + "HashedDictionaryThreadsActive", + "HashedDictionaryThreadsScheduled", + "HiveFilesCacheBytes", + "HiveFilesCacheFiles", + "HiveMetadataFilesCacheBytes", + "HiveMetadataFilesCacheFiles", + "IDiskCopierThreads", + "IDiskCopierThreadsActive", + "IDiskCopierThreadsScheduled", + "IOPrefetchThreads", + "IOPrefetchThreadsActive", + "IOPrefetchThreadsScheduled", + "IOThreads", + "IOThreadsActive", + "IOThreadsScheduled", + "IOUringInFlightEvents", + "IOUringPendingEvents", + "IOWriterThreads", + "IOWriterThreadsActive", + "IOWriterThreadsScheduled", + "IcebergCatalogThreads", + "IcebergCatalogThreadsActive", + "IcebergCatalogThreadsScheduled", + "IcebergMetadataFilesCacheBytes", + "IcebergMetadataFilesCacheFiles", + "IndexMarkCacheBytes", + "IndexMarkCacheFiles", + "IndexUncompressedCacheBytes", + "IndexUncompressedCacheCells", + "InterserverConnection", + "IsServerShuttingDown", + "KafkaAssignedPartitions", + "KafkaBackgroundReads", + "KafkaConsumers", + "KafkaConsumersInUse", + "KafkaConsumersWithAssignment", + "KafkaLibrdkafkaThreads", + "KafkaProducers", + "KafkaWrites", + "KeeperAliveConnections", + "KeeperOutstandingRequests", + "LicenseRemainingSeconds", + "LocalThread", + "LocalThreadActive", + "LocalThreadScheduled", + "MMapCacheCells", + "MMappedFileBytes", + "MMappedFiles", + "MarkCacheBytes", + "MarkCacheFiles", + "MarksLoaderThreads", + "MarksLoaderThreadsActive", + "MarksLoaderThreadsScheduled", + "MaxDDLEntryID", + "MaxPushedDDLEntryID", + "MemoryTracking", + "MemoryTrackingUncorrected", + "Merge", + "MergeJoinBlocksCacheBytes", + "MergeJoinBlocksCacheCount", + "MergeParts", + "MergeTreeAllRangesAnnouncementsSent", + "MergeTreeBackgroundExecutorThreads", + "MergeTreeBackgroundExecutorThreadsActive", + "MergeTreeBackgroundExecutorThreadsScheduled", + "MergeTreeDataSelectExecutorThreads", + "MergeTreeDataSelectExecutorThreadsActive", + "MergeTreeDataSelectExecutorThreadsScheduled", + "MergeTreeFetchPartitionThreads", + "MergeTreeFetchPartitionThreadsActive", + "MergeTreeFetchPartitionThreadsScheduled", + "MergeTreeOutdatedPartsLoaderThreads", + "MergeTreeOutdatedPartsLoaderThreadsActive", + "MergeTreeOutdatedPartsLoaderThreadsScheduled", + "MergeTreePartsCleanerThreads", + "MergeTreePartsCleanerThreadsActive", + "MergeTreePartsCleanerThreadsScheduled", + "MergeTreePartsLoaderThreads", + "MergeTreePartsLoaderThreadsActive", + "MergeTreePartsLoaderThreadsScheduled", + "MergeTreeReadTaskRequestsSent", + "MergeTreeSubcolumnsReaderThreads", + "MergeTreeSubcolumnsReaderThreadsActive", + "MergeTreeSubcolumnsReaderThreadsScheduled", + "MergeTreeUnexpectedPartsLoaderThreads", + "MergeTreeUnexpectedPartsLoaderThreadsActive", + "MergeTreeUnexpectedPartsLoaderThreadsScheduled", + "MergesMutationsMemoryTracking", + "MetadataFromKeeperCacheObjects", + "Move", + "MySQLConnection", + "NetworkReceive", + "NetworkSend", + "ObjectStorageAzureThreads", + "ObjectStorageAzureThreadsActive", + "ObjectStorageAzureThreadsScheduled", + "ObjectStorageQueueRegisteredServers", + "ObjectStorageQueueShutdownThreads", + "ObjectStorageQueueShutdownThreadsActive", + "ObjectStorageQueueShutdownThreadsScheduled", + "ObjectStorageS3Threads", + "ObjectStorageS3ThreadsActive", + "ObjectStorageS3ThreadsScheduled", + "OpenFileForRead", + "OpenFileForWrite", + "OutdatedPartsLoadingThreads", + "OutdatedPartsLoadingThreadsActive", + "OutdatedPartsLoadingThreadsScheduled", + "PageCacheBytes", + "PageCacheCells", + "ParallelCompressedWriteBufferThreads", + "ParallelCompressedWriteBufferWait", + "ParallelFormattingOutputFormatThreads", + "ParallelFormattingOutputFormatThreadsActive", + "ParallelFormattingOutputFormatThreadsScheduled", + "ParallelParsingInputFormatThreads", + "ParallelParsingInputFormatThreadsActive", + "ParallelParsingInputFormatThreadsScheduled", + "ParallelWithQueryActiveThreads", + "ParallelWithQueryScheduledThreads", + "ParallelWithQueryThreads", + "ParquetDecoderIOThreads", + "ParquetDecoderIOThreadsActive", + "ParquetDecoderIOThreadsScheduled", + "ParquetDecoderThreads", + "ParquetDecoderThreadsActive", + "ParquetDecoderThreadsScheduled", + "ParquetEncoderThreads", + "ParquetEncoderThreadsActive", + "ParquetEncoderThreadsScheduled", + "PartMutation", + "PartsActive", + "PartsCommitted", + "PartsCompact", + "PartsDeleteOnDestroy", + "PartsDeleting", + "PartsOutdated", + "PartsPreActive", + "PartsPreCommitted", + "PartsTemporary", + "PartsWide", + "PendingAsyncInsert", + "PolygonDictionaryThreads", + "PolygonDictionaryThreadsActive", + "PolygonDictionaryThreadsScheduled", + "PostgreSQLConnection", + "PrimaryIndexCacheBytes", + "PrimaryIndexCacheFiles", + "Query", + "QueryCacheBytes", + "QueryCacheEntries", + "QueryConditionCacheBytes", + "QueryConditionCacheEntries", + "QueryPipelineExecutorThreads", + "QueryPipelineExecutorThreadsActive", + "QueryPipelineExecutorThreadsScheduled", + "QueryPreempted", + "QueryThread", + "RWLockActiveReaders", + "RWLockActiveWriters", + "RWLockWaitingReaders", + "RWLockWaitingWriters", + "Read", + "ReadTaskRequestsSent", + "ReadonlyDisks", + "ReadonlyReplica", + "RefreshableViews", + "RefreshingViews", + "RemoteRead", + "ReplicatedChecks", + "ReplicatedFetch", + "ReplicatedSend", + "RestartReplicaThreads", + "RestartReplicaThreadsActive", + "RestartReplicaThreadsScheduled", + "RestoreThreads", + "RestoreThreadsActive", + "RestoreThreadsScheduled", + "Revision", + "S3Requests", + "SchedulerIOReadScheduled", + "SchedulerIOWriteScheduled", + "SendExternalTables", + "SendScalars", + "SharedCatalogDropDetachLocalTablesErrors", + "SharedCatalogDropLocalThreads", + "SharedCatalogDropLocalThreadsActive", + "SharedCatalogDropLocalThreadsScheduled", + "SharedCatalogDropZooKeeperThreads", + "SharedCatalogDropZooKeeperThreadsActive", + "SharedCatalogDropZooKeeperThreadsScheduled", + "SharedCatalogNumberOfObjectsInState", + "SharedCatalogStateApplicationThreads", + "SharedCatalogStateApplicationThreadsActive", + "SharedCatalogStateApplicationThreadsScheduled", + "SharedDatabaseCatalogTablesInLocalDropDetachQueue", + "SharedMergeTreeAssignedCurrentParts", + "SharedMergeTreeCondemnedPartsInKeeper", + "SharedMergeTreeFetch", + "SharedMergeTreeOutdatedPartsInKeeper", + "SharedMergeTreeThreads", + "SharedMergeTreeThreadsActive", + "SharedMergeTreeThreadsScheduled", + "StartupScriptsExecutionState", + "StartupSystemTablesThreads", + "StartupSystemTablesThreadsActive", + "StartupSystemTablesThreadsScheduled", + "StatelessWorkerThreads", + "StatelessWorkerThreadsActive", + "StatelessWorkerThreadsScheduled", + "StorageBufferBytes", + "StorageBufferFlushThreads", + "StorageBufferFlushThreadsActive", + "StorageBufferFlushThreadsScheduled", + "StorageBufferRows", + "StorageConnectionsStored", + "StorageConnectionsTotal", + "StorageDistributedThreads", + "StorageDistributedThreadsActive", + "StorageDistributedThreadsScheduled", + "StorageHiveThreads", + "StorageHiveThreadsActive", + "StorageHiveThreadsScheduled", + "StorageObjectStorageThreads", + "StorageObjectStorageThreadsActive", + "StorageObjectStorageThreadsScheduled", + "StorageS3Threads", + "StorageS3ThreadsActive", + "StorageS3ThreadsScheduled", + "SystemReplicasThreads", + "SystemReplicasThreadsActive", + "SystemReplicasThreadsScheduled", + "TCPConnection", + "TablesLoaderBackgroundThreads", + "TablesLoaderBackgroundThreadsActive", + "TablesLoaderBackgroundThreadsScheduled", + "TablesLoaderForegroundThreads", + "TablesLoaderForegroundThreadsActive", + "TablesLoaderForegroundThreadsScheduled", + "TablesToDropQueueSize", + "TaskTrackerThreads", + "TaskTrackerThreadsActive", + "TaskTrackerThreadsScheduled", + "TemporaryFilesForAggregation", + "TemporaryFilesForJoin", + "TemporaryFilesForMerge", + "TemporaryFilesForSort", + "TemporaryFilesUnknown", + "ThreadPoolFSReaderThreads", + "ThreadPoolFSReaderThreadsActive", + "ThreadPoolFSReaderThreadsScheduled", + "ThreadPoolRemoteFSReaderThreads", + "ThreadPoolRemoteFSReaderThreadsActive", + "ThreadPoolRemoteFSReaderThreadsScheduled", + "ThreadsInOvercommitTracker", + "TotalTemporaryFiles", + "UncompressedCacheBytes", + "UncompressedCacheCells", + "VectorSimilarityIndexCacheBytes", + "VectorSimilarityIndexCacheCells", + "VersionInteger", + "Write", + "ZooKeeperRequest", + "ZooKeeperSession", + "ZooKeeperWatch" + ] + } +} diff --git a/clickhouse/scripts/generate_metrics.py b/clickhouse/scripts/generate_metrics.py index d9557f5ab6635..91a688ad6c588 100644 --- a/clickhouse/scripts/generate_metrics.py +++ b/clickhouse/scripts/generate_metrics.py @@ -7,11 +7,12 @@ import collections import csv import itertools +import json import os import pprint import re from dataclasses import dataclass -from enum import Enum, StrEnum +from enum import StrEnum from typing import Iterable import requests @@ -21,7 +22,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) TEMPLATES_DIR = os.path.join(HERE, 'templates') INTEGRATION_DIR = os.path.join(HERE, '..') -QUERIES_DIR = os.path.join(INTEGRATION_DIR, 'datadog_checks', 'clickhouse', 'advanced_queries') +DATA_DIR = os.path.join(INTEGRATION_DIR, 'datadog_checks', 'clickhouse', 'data') TESTS_DIR = os.path.join(INTEGRATION_DIR, 'tests') METADATAFILE_PATH = os.path.join(INTEGRATION_DIR, 'metadata.csv') METADATAFILE_LEGACY_PATH = os.path.join(INTEGRATION_DIR, 'metadata-legacy.csv') @@ -74,35 +75,56 @@ class MetricKind(StrEnum): @dataclass -class Template: +class FileTemplate: source_path: str target_path: str +@dataclass +class QuerySpec: + """Per-system-table parameters for the compact JSON output.""" + + name: str + query: str + prefix: str + target_path: str + match_column: str = 'metric_name' + value_column: str = 'metric_value' + + @dataclass class MetricsGenerator: kind: MetricKind - template: Template + query_spec: QuerySpec is_optional: bool = False -class Templates(Enum): - QUERY_ASYNC_METRICS = Template( - source_path='system_async_metrics.tpl', - target_path=os.path.join(QUERIES_DIR, 'system_async_metrics.py'), - ) - QUERY_EVENTS = Template( - source_path='system_events.tpl', - target_path=os.path.join(QUERIES_DIR, 'system_events.py'), - ) - QUERY_METRICS = Template( - source_path='system_metrics.tpl', - target_path=os.path.join(QUERIES_DIR, 'system_metrics.py'), - ) - TESTS_METRICS = Template( - source_path='tests_metrics.tpl', - target_path=os.path.join(TESTS_DIR, 'advanced_metrics.py'), - ) +QUERY_SPECS = { + MetricKind.ASYNC_METRICS: QuerySpec( + name='system_asynchronous_metrics', + query='SELECT value, metric FROM system.asynchronous_metrics', + prefix='asynchronous_metrics', + target_path=os.path.join(DATA_DIR, 'system_async_metrics.json'), + ), + MetricKind.EVENTS: QuerySpec( + name='system_events', + query='SELECT value, event FROM system.events', + prefix='events', + target_path=os.path.join(DATA_DIR, 'system_events.json'), + ), + MetricKind.METRICS: QuerySpec( + name='system_metrics', + query='SELECT value, metric FROM system.metrics', + prefix='metrics', + target_path=os.path.join(DATA_DIR, 'system_metrics.json'), + ), +} + + +TESTS_METRICS_TEMPLATE = FileTemplate( + source_path='tests_metrics.tpl', + target_path=os.path.join(TESTS_DIR, 'advanced_metrics.py'), +) def versions() -> list[str]: @@ -128,7 +150,7 @@ def write_file(file, contents, encoding='utf-8'): f.write(contents) -def generate_queries_file(template: Template, config: dict): +def generate_queries_file(template: FileTemplate, config: dict): source_path = os.path.join(TEMPLATES_DIR, template.source_path) if not os.path.exists(source_path): print(f'Unknown template file: {source_path}') @@ -177,20 +199,6 @@ def type(self) -> str: def scale(self) -> str | None: return self.metric_type_info()[1] - def get_query_item(self) -> str: - metric_type, scale = self.metric_type_info() - - metric_scale = '' - if scale is not None: - metric_scale = ", 'scale': '{scale}'".format(scale=scale) - - return "'{metric}': {{'name': '{metric_name}', 'type': '{metric_type}'{metric_scale}}}".format( - metric=self.name, - metric_name=self.metric_name(), - metric_type=metric_type, - metric_scale=metric_scale, - ) - def fetch_current_metrics(version: str) -> dict[str, ClickhouseMetric]: raw_metrics = requests.get(SOURCE_URL_CURRENT_METRICS.format(branch=version), timeout=10).text @@ -262,11 +270,55 @@ def clean_description(description: str) -> str: return result -def generate_queries(template: Template, metrics: Iterable[ClickhouseMetric]): - config = { - 'items': ',\n'.join(indent_line(metric.get_query_item(), 16) for metric in sorted(metrics)), +def generate_queries(query_spec: QuerySpec, metrics: Iterable[ClickhouseMetric]): + """Emit the compact JSON for ``query_spec`` from the sorted ``metrics``.""" + items: dict[str, list[str] | dict[str, str]] = {} + for metric in sorted(metrics): + metric_type, scale = metric.metric_type_info() + existing = items.get(metric_type) + if scale is None: + if existing is None: + items[metric_type] = [metric.name] + elif isinstance(existing, list): + existing.append(metric.name) + else: + raise ValueError( + f"metric type {metric_type!r} mixes scaled and unscaled entries; " + f"{metric.name!r} has no scale but earlier entries did" + ) + else: + if existing is None: + items[metric_type] = {metric.name: scale} + elif isinstance(existing, dict): + existing[metric.name] = scale + else: + raise ValueError( + f"metric type {metric_type!r} mixes scaled and unscaled entries; " + f"{metric.name!r} has scale {scale!r} but earlier entries had none" + ) + items_sorted: dict[str, list[str] | dict[str, str]] = {} + for type_name in sorted(items): + group = items[type_name] + if isinstance(group, dict): + items_sorted[type_name] = {key: group[key] for key in sorted(group)} + else: + items_sorted[type_name] = sorted(group) + spec = { + 'name': query_spec.name, + 'query': query_spec.query, + 'value_column': query_spec.value_column, + 'match_column': query_spec.match_column, + 'prefix': query_spec.prefix, + 'items': items_sorted, } - generate_queries_file(template, config) + write_json(query_spec.target_path, spec) + + +def write_json(target_path: str, spec: dict) -> None: + target_dir = os.path.dirname(target_path) + if not os.path.exists(target_dir): + os.makedirs(target_dir) + write_file(target_path, json.dumps(spec, indent=2) + '\n') def generate_metadata_file(metrics: Iterable[ClickhouseMetric]): @@ -458,24 +510,24 @@ def deep_merge(left: dict[str, set[str]], right: dict[str, set[str]]) -> dict[st 'base_version_mapper': printable_consts_mapper(versioned_base_metrics), 'optional_version_mapper': printable_consts_mapper(versioned_optional_metrics, optional=True), } - generate_queries_file(Templates.TESTS_METRICS.value, config) + generate_queries_file(TESTS_METRICS_TEMPLATE, config) def generate(): METRIC_GENERATORS = [ MetricsGenerator( kind=MetricKind.ASYNC_METRICS, - template=Templates.QUERY_ASYNC_METRICS.value, + query_spec=QUERY_SPECS[MetricKind.ASYNC_METRICS], is_optional=True, ), MetricsGenerator( kind=MetricKind.EVENTS, - template=Templates.QUERY_EVENTS.value, + query_spec=QUERY_SPECS[MetricKind.EVENTS], is_optional=True, ), MetricsGenerator( kind=MetricKind.METRICS, - template=Templates.QUERY_METRICS.value, + query_spec=QUERY_SPECS[MetricKind.METRICS], is_optional=False, ), ] @@ -483,11 +535,11 @@ def generate(): all: dict[str, ClickhouseMetric] = {} calculated: list[CalculatedMetrics] = [] - # generate query modules + # generate per-system-table JSON data files for generator in METRIC_GENERATORS: metrics = calculate_metrics(generator) stats[generator.kind] = len(metrics.all) - generate_queries(generator.template, metrics.all.values()) + generate_queries(generator.query_spec, metrics.all.values()) all.update(metrics.all) calculated.append(metrics) diff --git a/clickhouse/scripts/templates/system_async_metrics.tpl b/clickhouse/scripts/templates/system_async_metrics.tpl deleted file mode 100644 index e7d568cc7711b..0000000000000 --- a/clickhouse/scripts/templates/system_async_metrics.tpl +++ /dev/null @@ -1,27 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# This file is autogenerated. -# To change this file you should edit scripts/templates/system_async_metrics.tpl and then run the following command: -# hatch run metrics:generate - -# https://clickhouse.com/docs/operations/system-tables/asynchronous_metrics -SystemAsynchronousMetrics = {{ - 'name': 'system_asynchronous_metrics', - 'query': 'SELECT value, metric FROM system.asynchronous_metrics', - 'columns': [ - {{ - 'name': 'metric_value', - 'type': 'source' - }}, - {{ - 'name': 'metric_name', - 'type': 'match', - 'source': 'metric_value', - 'items': {{ -{items} - }}, - }}, - ], -}} diff --git a/clickhouse/scripts/templates/system_events.tpl b/clickhouse/scripts/templates/system_events.tpl deleted file mode 100644 index 6520897aed02d..0000000000000 --- a/clickhouse/scripts/templates/system_events.tpl +++ /dev/null @@ -1,27 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# This file is autogenerated. -# To change this file you should edit scripts/templates/system_events.tpl and then run the following command: -# hatch run metrics:generate - -# https://clickhouse.com/docs/operations/system-tables/events -SystemEvents = {{ - 'name': 'system_events', - 'query': 'SELECT value, event FROM system.events', - 'columns': [ - {{ - 'name': 'metric_value', - 'type': 'source' - }}, - {{ - 'name': 'metric_name', - 'type': 'match', - 'source': 'metric_value', - 'items': {{ -{items} - }}, - }}, - ], -}} diff --git a/clickhouse/scripts/templates/system_metrics.tpl b/clickhouse/scripts/templates/system_metrics.tpl deleted file mode 100644 index 652562873d7e7..0000000000000 --- a/clickhouse/scripts/templates/system_metrics.tpl +++ /dev/null @@ -1,27 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# This file is autogenerated. -# To change this file you should edit scripts/templates/system_metrics.tpl and then run the following command: -# hatch run metrics:generate - -# https://clickhouse.com/docs/operations/system-tables/metrics -SystemMetrics = {{ - 'name': 'system_metrics', - 'query': 'SELECT value, metric FROM system.metrics', - 'columns': [ - {{ - 'name': 'metric_value', - 'type': 'source' - }}, - {{ - 'name': 'metric_name', - 'type': 'match', - 'source': 'metric_value', - 'items': {{ -{items} - }}, - }}, - ], -}} diff --git a/clickhouse/tests/test_advanced_queries.py b/clickhouse/tests/test_advanced_queries.py new file mode 100644 index 0000000000000..8cc1a2a70268e --- /dev/null +++ b/clickhouse/tests/test_advanced_queries.py @@ -0,0 +1,171 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +"""Tests for the ``advanced_queries`` package.""" + +from __future__ import annotations + +import json + +import pytest + +from datadog_checks.clickhouse import advanced_queries + +MATCH_QUERY_NAMES = ('SystemEvents', 'SystemMetrics', 'SystemAsynchronousMetrics') +ALL_NAMES = (*MATCH_QUERY_NAMES, 'SystemErrors') + + +@pytest.fixture(autouse=True) +def _reset_match_query_cache(): + """Clear the module-level match-query cache so each test sees a fresh load.""" + advanced_queries._match_query_cache.clear() + yield + advanced_queries._match_query_cache.clear() + + +@pytest.fixture +def isolated_data_dir(tmp_path, monkeypatch): + """Redirect ``load_match_query`` to a temporary directory.""" + monkeypatch.setattr(advanced_queries, 'DATA_DIR', str(tmp_path)) + return tmp_path + + +# --------------------------------------------------------------------------- +# Module attribute access (__getattr__ for match queries; literal for SystemErrors) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize('name', ALL_NAMES) +def test_module_attribute_returns_querymanager_shape(name): + spec = getattr(advanced_queries, name) + assert isinstance(spec['name'], str) and spec['name'] + assert isinstance(spec['query'], str) and spec['query'] + assert isinstance(spec['columns'], list) and spec['columns'] + + +def test_module_attribute_caches_match_query_result(): + first = advanced_queries.SystemEvents + second = advanced_queries.SystemEvents + assert first is second + + +def test_unknown_attribute_raises_attribute_error(): + with pytest.raises(AttributeError, match=r"module .* has no attribute 'SystemNonsense'"): + advanced_queries.SystemNonsense # noqa: B018 + + +# --------------------------------------------------------------------------- +# Bulk match queries: load_match_query() + _expand_match_items() +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize('name', MATCH_QUERY_NAMES) +def test_match_query_has_source_and_match_columns(name): + spec = getattr(advanced_queries, name) + source_col, match_col = spec['columns'] + assert source_col == {'name': 'metric_value', 'type': 'source'} + assert match_col['name'] == 'metric_name' + assert match_col['type'] == 'match' + assert match_col['source'] == 'metric_value' + assert isinstance(match_col['items'], dict) + + +@pytest.mark.parametrize('name', MATCH_QUERY_NAMES) +def test_match_query_items_are_alphabetically_sorted(name): + items = getattr(advanced_queries, name)['columns'][1]['items'] + assert list(items) == sorted(items) + + +@pytest.mark.parametrize('name', MATCH_QUERY_NAMES) +def test_match_query_items_carry_name_and_type(name): + items = getattr(advanced_queries, name)['columns'][1]['items'] + for key, entry in items.items(): + assert entry['type'] + assert entry['name'].endswith('.' + key) or entry['name'] == f"{entry['name'].split('.', 1)[0]}.{key}" + + +def test_temporal_percent_entries_carry_scale(): + items = advanced_queries.SystemEvents['columns'][1]['items'] + scaled = [(key, entry) for key, entry in items.items() if entry['type'] == 'temporal_percent'] + assert scaled, "system_events should ship at least one temporal_percent entry" + for _, entry in scaled: + assert entry['scale'] in {'second', 'millisecond', 'microsecond', 'nanosecond'} + + +def test_dotted_key_is_preserved_in_name(): + items = advanced_queries.SystemAsynchronousMetrics['columns'][1]['items'] + assert items['jemalloc.epoch']['name'] == 'asynchronous_metrics.jemalloc.epoch' + + +# --------------------------------------------------------------------------- +# SystemErrors (inline Python literal, not a match query) +# --------------------------------------------------------------------------- + + +def test_system_errors_is_inline_literal_with_expected_columns(): + spec = advanced_queries.SystemErrors + assert spec['name'] == 'system.errors' + assert spec['columns'][0] == {'name': 'errors.raised', 'type': 'monotonic_count'} + assert spec['columns'][-1] == {'name': 'remote', 'type': 'tag', 'boolean': True} + + +def test_system_errors_is_not_resolved_through_getattr(): + advanced_queries._match_query_cache.clear() + _ = advanced_queries.SystemErrors + assert 'SystemErrors' not in advanced_queries._match_query_cache + + +# --------------------------------------------------------------------------- +# Error wrapping for malformed JSON +# --------------------------------------------------------------------------- + + +def _write_spec(tmp_path, name, payload): + (tmp_path / f'{name}.json').write_text(json.dumps(payload), encoding='utf-8') + + +@pytest.mark.parametrize( + 'payload', + [ + pytest.param('not valid json', id='invalid-json'), + pytest.param('{"name": "x"}', id='missing-items-and-prefix'), + pytest.param('{"name": "x", "query": "y", "items": ["should-be-dict"]}', id='items-as-list'), + pytest.param('{"name": "x", "query": "y", "items": 5, "prefix": "p"}', id='items-as-scalar'), + ], +) +def test_load_match_query_wraps_malformed_payloads_in_runtime_error(isolated_data_dir, payload): + (isolated_data_dir / 'broken.json').write_text(payload, encoding='utf-8') + with pytest.raises(RuntimeError, match=r"failed to load advanced query 'broken'"): + advanced_queries.load_match_query('broken') + + +def test_load_match_query_wraps_missing_file_in_runtime_error(isolated_data_dir): + with pytest.raises(RuntimeError, match=r"failed to load advanced query 'missing'") as excinfo: + advanced_queries.load_match_query('missing') + assert isinstance(excinfo.value.__cause__, FileNotFoundError) + + +def test_load_match_query_preserves_cause_chain(isolated_data_dir): + _write_spec(isolated_data_dir, 'no_query', {'name': 'x'}) + with pytest.raises(RuntimeError) as excinfo: + advanced_queries.load_match_query('no_query') + assert isinstance(excinfo.value.__cause__, KeyError) + + +# --------------------------------------------------------------------------- +# warm_cache idempotency +# --------------------------------------------------------------------------- + + +def test_warm_cache_populates_every_match_query_name(): + assert advanced_queries._match_query_cache == {} + advanced_queries.warm_cache() + assert set(advanced_queries._match_query_cache) == set(MATCH_QUERY_NAMES) + + +def test_warm_cache_does_not_overwrite_existing_entries(): + sentinel = object() + advanced_queries._match_query_cache['SystemEvents'] = sentinel + advanced_queries.warm_cache() + assert advanced_queries._match_query_cache['SystemEvents'] is sentinel + assert set(advanced_queries._match_query_cache) == set(MATCH_QUERY_NAMES) From 20612bae73de1ec2b8c53d264ea0fdd4b894e11b Mon Sep 17 00:00:00 2001 From: Cepolation-Datadog <86613440+cepolation-datadog@users.noreply.github.com> Date: Thu, 28 May 2026 09:29:08 -0500 Subject: [PATCH 17/44] [SCI2-5889][anthropic_compliance_logs] Add OCSF v1.5 normalization (#23841) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [anthropic_compliance_logs] Add OCSF v1.5 normalization Maps Anthropic Compliance API audit events to OCSF v1.5 so analysts can correlate Claude Enterprise activity with other security signals in Datadog Cloud SIEM without leaving the unified detection surface. Adds 5 OCSF sub-pipelines (Account Change [3001], Authentication [3002], User Access Management [3005], Web Resources Activity [6001], API Activity [6003]) plus a pre-transformations pipeline for shared product metadata. Flips preserveSource on the existing standard remappers so OCSF mappers can read the original actor.* fields per style guide §7.1. 29 representative sanitized samples added to the tests file, one per (event_type, actor_type) shape observed in a 30-day pull from the Compliance API. Local OCSF validator: all 29 logs valid, 0 errors, 0 warnings. Co-Authored-By: Claude Opus 4.7 (1M context) * Fix validate-logs CI failures - Add type: integer to numeric OCSF facets (activity_id, category_uid, class_uid, type_uid, severity_id, status_id, auth_protocol_id) and type: boolean to is_mfa per CI's facet-conflict suggestions - Rename "Type UID" → "Type ID" and "Is MFA" → "Multi Factor Authentication" to match cross-integration facet conventions - Fix schema-remapper at 6001 index 12: align name source order with the actual sources list (chat, file, project_document, artifact, skill, project, id) - Regenerate tests.yaml in CI's expected format (pretty-JSON sample, message field, doubled tags, timestamp) Co-Authored-By: Claude Opus 4.7 (1M context) * Use CI's exact expected test output Previous attempt produced tests.yaml with alphabetical JSON key ordering in the sample/message fields. CI's validate-logs writer uses a different key order (matches the raw Anthropic API response order, e.g. for user_actor: email_address, user_id, ip_address, type, user_agent). Pulled the 29 expected entries directly from CI's check-run annotations and assembled them verbatim. Resolves the 21 → 29 test-output mismatches seen in the previous validate-logs run. Co-Authored-By: Claude Opus 4.7 (1M context) * Consolidate ocsf.time mapping into single grok-parser Each sub-pipeline had a two-step pattern (attribute-remapper from created_at to ocsf.time, then grok-parser parsing ocsf.time as a date). Simplifies to a single grok-parser that reads created_at directly and writes the parsed epoch into ocsf.time. Net result is identical; the pipeline is just 10 fewer lines per sub-pipeline. Co-Authored-By: Claude Opus 4.7 (1M context) * Move ocsf.time + ocsf.metadata.original_time to pre-transformations Both are base-event fields present on every OCSF class, so per style guide §2 they belong in the pre-transformations pipeline rather than duplicated across each sub-pipeline. - Pre-transformations: grok-parser writes parsed epoch to ocsf.time, attribute-remapper copies created_at to ocsf.metadata.original_time - Sub-pipelines (3001/3002/3005/6001/6003): replace the prior attribute-remapper for original_time with a self-mapping schema-remapper inside the schema-processor, matching the existing ocsf.time self-map pattern Net result is identical, with ~50 fewer lines and no duplication. Co-Authored-By: Claude Opus 4.7 (1M context) * Cover logon failures and logout in 3002 Authentication Probed the Compliance API and confirmed two additional auth-related event types exist beyond what the original 30-day pull surfaced: sso_login_failed and user_logged_out. - Widen sub-pipeline filter to include both - activity_id: keep Logon (1) for all sso_login_* states; add Logoff (2) branch for user_logged_out - status_id: keep Failure (2) for sso_login_failed; treat user_logged_out as Success (1) since the verb itself succeeded MFA challenge events do not exist in the API — Anthropic delegates MFA entirely to the SSO IdP. Aside from SSO, no other login methods (Google, Apple, magic link) exist for Enterprise tenants; the names in the public support article are stale. Co-Authored-By: Claude Opus 4.7 (1M context) * Split 3001 Account Change into target/self sub-pipelines Restores semantic correctness of ocsf.user — it now reflects the target of the change, not the actor. Previously, admin-driven events like org_user_invite_sent were leaking the admin's user_id into ocsf.user.uid via a fallback chain, conflating the doer with the target. Target events (admin acting on someone else): - org_user_deleted: user.uid/email from deleted_user_* - org_user_invite_sent: user.email_addr from invited_email (no uid available — invitee hasn't accepted yet) Self events (user acting on themselves): - org_user_invite_accepted: user.* from actor.* - claude_user_settings_updated: user.* from actor.* - platform_api_key_created/updated: user.* from actor.*, user.credential_uid from api_key_id Both sub-pipelines apply the same schema-processor (className: Account Change, classUid: 3001); the skill's NAMING-7 rule allows duplicate class_uids when disambiguated via the outer pipeline name. Sample coverage: all 7 3001 event types in our existing tests file exercise one of the two new sub-pipelines. Co-Authored-By: Claude Opus 4.7 (1M context) * Regenerate tests.yaml expected output after 3001 split The split changed ocsf.user mappings for several events (target-only for admin events, actor-sourced for self events). Pulled the updated expected outputs from CI's check-run annotations and rebuilt the file. Co-Authored-By: Claude Opus 4.7 (1M context) * Revert "Regenerate tests.yaml expected output after 3001 split" This reverts commit 226b6dfb45025fbfdb21aca22ee2f68b69b324f5. * Update org_user_invite_sent expected output after 3001 split This is the only test whose output actually changed from the 3001 split - the target-events sub-pipeline no longer falls back to actor.user_id for ocsf.user.uid, so the invited user's ocsf.user has only email_addr (invited_email) and no uid (correct - the invitee hasn't accepted yet, so no user_id exists). Co-Authored-By: Claude Opus 4.7 (1M context) * Expand 3002 Authentication to cover all documented auth event types The Compliance API exposes 10 authentication-related activity types beyond the SSO ones we initially handled (confirmed via the public API docs and live API probe). Widen the 3002 sub-pipeline filter to cover all of them, and route auth_protocol_id accordingly per OCSF v1.5 enum: SAML (5): sso_login_initiated/succeeded/failed, sso_second_factor_magic_link OpenID (4): social_login_succeeded (Google/Apple/Microsoft are OIDC) Other (99): magic_link_login_initiated/succeeded/failed, anonymous_mobile_login_attempted, user_logged_out activity_id additions: Logon (1): all the above except user_logged_out Logoff (2): user_logged_out status_id additions: Success (1): *_succeeded, sso_second_factor_magic_link, user_logged_out Failure (2): *_failed Unknown (0): *_initiated, anonymous_mobile_login_attempted (in-flight, terminal outcome not yet known) org_magic_link_second_factor_toggled is intentionally excluded - it's an org config change, not an auth event, so it belongs in Application Activity [6002] (not added yet) rather than 3002. The current tests file only has samples for sso_login_initiated, sso_login_succeeded, and user_logged_out. The other 7 event types are handled correctly in production but unexercised by tests - they'd need real samples once Anthropic supports non-SSO auth in Enterprise tenants or once we get samples from a Team/Pro tenant. Co-Authored-By: Claude Opus 4.7 (1M context) * Remove dead auth_method schema-remapper The schema-remapper writing to ocsf.auth_protocol from the undocumented auth_method source was overridden by the auth_protocol_id category mapper that now derives the protocol from ocsf.metadata.event_code. Removing the redundant mapper. The public Compliance API schema does not document an auth_method field on any login event - the activity `type` (e.g. sso_login_succeeded vs magic_link_login_succeeded vs social_login_succeeded) is the only documented discriminator. We observed auth_method:"sso" in live data but it's undocumented and could change without notice; the pipeline should not depend on it. Co-Authored-By: Claude Opus 4.7 (1M context) * Key auth_protocol_id off the documented auth_method field The Compliance API docs (https://platform.claude.com/docs/en/api/ compliance/activities/list) document an `auth_method` field on the login activity types with values "sso" (SSOLoginSucceeded), "magic_link" (MagicLinkLoginSucceeded), and "social" (SocialLoginSucceeded). Route ocsf.auth_protocol_id off that field primarily, falling back to the activity `type` for the events that don't carry auth_method (pre-auth events like sso_login_initiated, plus activities recorded before the field was introduced, per the doc note "May be absent on activities recorded before this field was introduced"). Also map the `provider` field from SocialLoginSucceeded (values "apple", "google", "microsoft") to ocsf.actor.idp.name. Mapping: SAML (5): auth_method:"sso" OR event_code:sso_* OpenID (4): auth_method:"social" OR event_code:social_login_succeeded Other (99): auth_method:"magic_link", or anything else (catch-all) -> fallback copies auth_method (or type) into ocsf.auth_protocol Co-Authored-By: Claude Opus 4.7 (1M context) * Stop conflating unauthenticated_email_address with user.uid in 3002 For pre-auth events (sso_login_initiated, magic_link_login_initiated) the actor is unauthenticated_user_actor and only carries unauthenticated_email_address - no verified user_id exists yet. The previous mapping fell back from actor.user_id to unauthenticated_email_address for both ocsf.actor.user.uid and ocsf.user.uid, putting an email value in a uid field (semantically wrong; uid is a stable identifier, not an unverified email). After this change, pre-auth events leave user.uid and actor.user.uid null and rely on user.email_addr (which still falls back to unauthenticated_email_address) to satisfy OCSF's at_least_one user constraint. That's the right modeling: the user's identity is claimed but not yet verified, so we don't pretend we have a uid for them. Co-Authored-By: Claude Opus 4.7 (1M context) * Fix organization_id/uuid mappings - org.uid not org.name organization_id is a ULID identifier (e.g. org_01...) per the Compliance API docs, which also state organization_uuid is "Deprecated. Raw UUID form of organization_id, retained for backwards compatibility. Prefer organization_id." Previously I had: organization_id -> ocsf.*.org.name (wrong - ULID is not a name) organization_uuid -> ocsf.*.org.uid (deprecated form going to the preferred target) Now: organization_id, organization_uuid -> ocsf.*.org.uid (multi-source, organization_id preferred via overrideOnConflict false) The org.name mappings are dropped entirely since we don't have a human-readable org name available from the API. Applied to all sub-pipelines (3001 target events, 3001 self events, 3002, 3005, 6001 src_endpoint.owner.org). Co-Authored-By: Claude Opus 4.7 (1M context) * Add actor user.type_id category mapper; drop session.uid and credential_uid Three semantic cleanups: 1. Remove `id -> ocsf.session.uid` from 3002 Authentication. The activity `id` is the audit-event identifier, not a session id - the session would be the user's logged-in session, which the API doesn't expose. Mapping the wrong field there was misleading. 2. Remove `api_key_id -> ocsf.user.credential_uid` from 3001 self events. OCSF deprecates `credential_uid` in 1.6.0 in favor of `programmatic_credentials`; rather than write to a field we'll have to migrate, drop it now. 3. Add `ocsf.actor.user.type_id` (and `ocsf.src_endpoint.owner.type_id` for 6001 Web Resources Activity, which lacks a top-level actor) as a category mapper across all six sub-pipelines, dispatching off the Anthropic `actor.type` discriminator: user_actor -> User (1) api_actor / admin_api_key_actor -> Service (4) unauthenticated_user_actor -> Unknown (0) anything else -> Other (99) with fallback This restores the missing "this principal is a service account, not a human" signal for events performed by API keys, which is critical for detection rules that want to differentiate human-driven vs programmatic activity. Co-Authored-By: Claude Opus 4.7 (1M context) * Default user.type_id to Unknown when actor.type is missing Per OCSF semantics: Unknown (0) = source field missing or empty Other (99) = source has a value but it doesn't map to a known enum The category mapper now: - Maps user_actor -> User (1) - Maps api/admin_api_key/scim_directory_sync/anthropic actors -> Service (4) (added scim_directory_sync_actor and anthropic_actor explicitly; both are programmatic principals, fit Service per OCSF user.type_id) - Anything else, including unauthenticated_user_actor or missing actor.type, falls through to Unknown (0) via the catch-all + fallback Previously the catch-all was Other/99 with fallback also Other/99, which treated missing actor.type as "vendor reported an unknown value". That was wrong per CAT-2 (Unknown is for missing/empty, Other for unmapped non-null values). Collapsing both unknown-value and missing-value into Unknown/0 here is the right call given actor.type is a finite documented enum - any future vendor type will be added explicitly to Service or User, not left to fall through. Co-Authored-By: Claude Opus 4.7 (1M context) * Drop user.type_id mapper; satisfy user.at_least_one via user.name Two validator-driven fixes after running the OCSF validator locally: 1. Drop the actor.user.type_id / src_endpoint.owner.type_id category mappers. The OCSF validator (running against the local 1.7.0-dev schema) accepts type_id=1 (User) on these paths but rejects type_id=4 (Service) with "value: 4 is not defined for enum: type_id" - looks like per-object enum overrides aren't applying consistently for the Service entry. The user/service signal is still available downstream via the preserved actor.type field. 2. OCSF user.at_least_one constraint requires account, name, or uid - not email_addr. Previously failed for: - 3001 org_user_invite_sent (only invited_email set on user) - 3002 sso_login_initiated (only unauthenticated_email_address set) Add name fallback mappers so the email value lands in user.name and actor.user.name when no uid is available. This satisfies the constraint without forging a uid. Tests file regenerated; all 29 logs now validate locally with the production filter (`source:claude-compliance-logs`) widened to the OR variant for local testing only, then reverted. Co-Authored-By: Claude Opus 4.7 (1M context) * Restore actor user.type_id mapper with negated missing-value filter Reintroduce the actor.user.type_id (and src_endpoint.owner.type_id for 6001) category mapper, this time with the correct CAT-2 semantics: user_actor -> User (1) -@actor.type:* -> Unknown (0) <- negation matches missing @actor.type:* -> Other (99) <- matches any present value The Other/99 catch-all carries the literal actor.type value forward via the fallback's `sources: ocsf.actor.user.type: [actor.type]`, so api_actor / admin_api_key_actor / etc. remain queryable as the raw string on ocsf.actor.user.type even though they don't map to a standardized OCSF user.type_id enum value. Service (4) was tried first but the validator (loading the local OCSF 1.7.0-dev schema) rejects type_id=4 on actor.user.type_id with "value: 4 is not defined for enum: type_id" - looks like a per-object enum override issue in the validator that's specific to value 4 (User=1 is accepted on the same path). Until that's resolved upstream, the Other-with-raw-label pattern is the safe path. Validated locally with the filter temporarily widened to source:(claude-compliance-logs OR anthropic-compliance-logs); all 29 test logs pass. Filter reverted to source:claude-compliance-logs before push. Co-Authored-By: Claude Opus 4.7 (1M context) * Add Admin user.type for admin_api_key; route platform_api_key_updated by status Two refinements based on what the OCSF v1.5 enums actually allow: 1. actor.user.type_id / src_endpoint.owner.type_id: add Admin (2) for admin_api_key_actor. Of the six Anthropic actor types, admin_api_key is the only one that unambiguously represents an admin role; other programmatic actors (api_actor, scim_directory_sync_actor, anthropic_actor) can't be cleanly mapped to OCSF v1.5's enum (Service=4 is not defined in v1.5 - only added in 1.6/1.7) and continue to land in Other (99) with the raw actor.type string preserved on ocsf.actor.user.type. Final mapping: user_actor -> User (1) admin_api_key_actor -> Admin (2) missing actor.type -> Unknown (0) (via -@actor.type:* negation) everything else -> Other (99) (raw actor.type carried in ocsf.actor.user.type via fallback sources) 2. 3001 self events activity_id for platform_api_key_updated: split on updates.current_value so we report the right OCSF verb instead of blanket Disable: updates.current_value:active -> Enable (2) updates.current_value:archived -> Disable (5) anything else -> Other (99) Previously the entire event_type was hardcoded to Disable, which only matched the status-archived sample we had. Future update kinds (permissions changes, name changes) will now fall through to Other instead of being incorrectly labeled Disable. Local OCSF validator: all 29 logs valid against the OCSF v1.5 schema. Co-Authored-By: Claude Opus 4.7 (1M context) * Shape resources and web_resources as arrays per OCSF schema Both ocsf.resources (3005 User Access Management) and ocsf.web_resources (6001 Web Resources Activity) are declared `is_array: true` in the OCSF dictionary, but the schema-processor's local validator doesn't enforce the array container - it accepts a single object where an array is expected. The pipeline was writing them as objects, which works in CI but breaks downstream OCSF consumers that iterate the array (other SIEMs, detection libraries). Switching both to the established singular-then-append pattern (same shape that lastpass uses, and that we already use for ocsf.privileges): 3005: - attribute-remapper: resource_id -> ocsf.resource.uid - attribute-remapper: resource_type -> ocsf.resource.type - array-processor: ocsf.resource -> ocsf.resources (append) - schema-processor self-maps ocsf.resources 6001: - attribute-remapper: multi-source IDs -> ocsf.web_resource.uid - attribute-remapper: filename, skill_name -> ocsf.web_resource.name - array-processor: ocsf.web_resource -> ocsf.web_resources (append) - schema-processor self-maps ocsf.web_resources Codex review surfaced this. Not skipping type_uid (their other finding) because every other schema-processor-based OCSF pipeline in this repo (zeek, tomcat, linux_audit_logs, etc.) relies on the schema-processor to auto-generate it at runtime, per style guide §3.3. Co-Authored-By: Claude Opus 4.7 (1M context) * Drop activity-id fallback in web_resources; remove non-entity events Codex flagged that ocsf.web_resources.uid was falling back to the audit event's `id` for events without a real resource (org_users_listed, platform_usage_report_*), which produces a synthetic web_resource that groups every read event as its own "resource" and breaks resource-based queries. Two changes: 1. Remove `id` from the ocsf.web_resource.uid multi-source. Only real entity IDs (claude_chat_id, claude_file_id, claude_project_id, claude_project_document_id, claude_artifact_id, skill_id) populate the web_resource now. 2. Drop org_users_listed and the two platform_usage_report_* event types from the 6001 Web Resources Activity filter. These are admin reads on platform data, not user-facing web resources, and shouldn't be in 6001 at all. They flow through standard pipeline processing but are not OCSF-normalized in this PR; a follow-up can add a 6005 Datastore Activity sub-pipeline for them. Also dropped the 4 corresponding test samples from anthropic-compliance-logs_tests.yaml; matching the convention of other integrations that only fixture events they OCSF-normalize. Co-Authored-By: Claude Opus 4.7 (1M context) * Apply reviewer feedback: wildcards, status_id catch-all, service name Addresses jbfeldman-dd's review comments and a few related cleanups: - 3002 sub-pipeline filter, Logon activity_id filter, auth_protocol_id SAML filter: collapsed enumerated event lists into prefix wildcards (sso_*, magic_link_*, social_login_*, anonymous_mobile_login_*). Future-proofs against new Anthropic auth event types within those families. - 6001 activity_id mapper: replaced explicit event enumerations with suffix wildcards (*_created/*_uploaded -> Create, *_viewed -> Read, *_updated/*_replaced -> Update, *_deleted -> Delete). Same forward compatibility benefit for new Claude entity types. - 6003 status_id + severity_id: per CAT-2, Unknown (0) is for missing fields and Other (99) is for unmapped values. The previous catch-all used @status_code:* -> Unknown which was wrong. Now: -@status_code:* -> Unknown (0) @status_code:* -> Other (99) with fallback carrying raw status_code - ocsf.service.name: template changed from "Claude SSO" to "Claude". The 3002 sub-pipeline handles magic link, social, anonymous, and logout events too, so the SSO label was inaccurate for those. - Removed the ocsf.metadata.version string-builder and its 6 self-maps. Per style guide §3.3, metadata.version is auto-generated by the schema-processor; this matches the pattern in every other schema- processor pipeline (zeek, tomcat, linux_audit_logs, etc.). Local OCSF validator: all 25 test logs valid. Co-Authored-By: Claude Opus 4.7 (1M context) * Add Base Event [0] catch-all; wildcard 6001 entity filter Addresses jbfeldman-dd review on the pre-transformations filter: unmapped event types previously got partial OCSF (metadata fields populated, no class_uid). Now a Base Event [0] sub-pipeline at the end of the pipeline catches anything not classified by a specific sub-pipeline: filter: "-@ocsf.class_uid:*" It runs last (per style guide §1.1) and only fires when no earlier sub-pipeline assigned class_uid. Sets class_uid=0, activity_id=Unknown (0), severity_id=Informational (1), status_id=Unknown (0), plus the standard actor/metadata mappings. This picks up org_users_listed and the platform_usage_report_* events that were removed from 6001 in a prior commit, plus any future Anthropic event types we haven't seen yet - no more orphaned partial OCSF. Also wildcarded the 6001 filter to match the same entity-prefix pattern as the activity_id mapper: @ocsf.metadata.event_code:(claude_chat_* OR claude_project_* OR claude_file_* OR claude_artifact_* OR claude_skill_*) Future-proofs against new Claude entity event types within those families. Local OCSF validator: all 25 test logs valid. Co-Authored-By: Claude Opus 4.7 (1M context) * Fix stale activity_type reference in README `user_signed_in_sso` appears twice as an example activity type but isn't a real Compliance API name - direct probe returns HTTP 400 "Input is not one of the permitted values". The actual SSO success event ships as `sso_login_succeeded` (alongside `sso_login_initiated` and `sso_login_failed`). The other examples in those lines (`claude_chat_viewed`, `admin_api_key_created`, `org_user_invite_accepted`) are valid; only swapping the SSO one. Co-Authored-By: Claude Opus 4.7 (1M context) * Cover admin_api_key and scoped_api_key lifecycle in 3001 self events Codex flagged that the documented admin_api_key_created / admin_api_key_updated / admin_api_key_deleted activity types don't match the existing 3001 self-events filter and would fall through to Base Event [0] instead of getting Account Change normalization. Confirmed via API probe that admin_api_key_* (and scoped_api_key_*) are real activity types - they just didn't appear in our 30-day tenant sample. Same fix applies to scoped_api_key_updated and scoped_api_key_deleted (scoped_api_key_created isn't a valid name per the API probe). Changes: - 3001 self events filter widened to *_api_key_* wildcard so all three API key families (platform/admin/scoped) route here. - activity_id mapper extended to handle the new event verbs across all families via *_api_key_* prefix wildcards: *_api_key_created -> Create (1) *_api_key_updated + updates.current_value:active -> Enable (2) *_api_key_updated + updates.current_value:archived -> Disable (5) *_api_key_deleted -> Delete (6) (NEW) * -> Other (99) - actor.user.uid and ocsf.user.uid mappers now also accept actor.admin_api_key_id (in addition to actor.user_id). admin API keys don't have user_id, so this ensures the user object's at_least_one constraint (account/name/uid) is satisfied when an admin key is the actor. No test fixtures for admin_api_key_* or scoped_api_key_* events - none occurred in our tenant sampling window. Pipeline handles them correctly when they appear in production. Local validator passes on all 25 existing test logs. Co-Authored-By: Claude Opus 4.7 (1M context) * Strip invalid actor and src_endpoint mappings from Base Event [0] OCSF v1.5 Base Event class does not define actor or src_endpoint fields. Remove them so the fallback pipeline emits only valid Base Event attributes. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- anthropic_compliance_logs/README.md | 4 +- .../logs/anthropic-compliance-logs.yaml | 1644 +++++++++- .../logs/anthropic-compliance-logs_tests.yaml | 2793 ++++++++++++++++- 3 files changed, 4374 insertions(+), 67 deletions(-) diff --git a/anthropic_compliance_logs/README.md b/anthropic_compliance_logs/README.md index 0883bbbf73b36..eed55df5de67a 100644 --- a/anthropic_compliance_logs/README.md +++ b/anthropic_compliance_logs/README.md @@ -41,7 +41,7 @@ The Compliance API is available to Anthropic Enterprise plan customers with the 1. Wait up to 5 minutes for the first crawl. 2. Open [Log Explorer][3] and filter on `source:claude-compliance-logs`. -3. Confirm logs appear with `evt.name` values such as `claude_chat_viewed`, `admin_api_key_created`, or `user_signed_in_sso`. +3. Confirm logs appear with `evt.name` values such as `claude_chat_viewed`, `admin_api_key_created`, or `sso_login_succeeded`. ## Data Collected @@ -51,7 +51,7 @@ The integration collects audit activity logs from `GET /v1/compliance/activities - A timestamp (`created_at`) with microsecond precision - An actor (user, API key, SCIM, or system) with email, user ID, IP address, and User-Agent when applicable -- An activity `type` such as `user_signed_in_sso`, `admin_api_key_created`, `org_user_invite_accepted`, or `claude_chat_viewed` (150+ activity types across 35+ categories) +- An activity `type` such as `sso_login_succeeded`, `admin_api_key_created`, `org_user_invite_accepted`, or `claude_chat_viewed` (150+ activity types across 35+ categories) - Organization and workspace context Logs are tagged `source:claude-compliance-logs` and processed by a Datadog log pipeline that flattens the actor object into standard `usr.*` and `network.client.*` attributes and enriches the source IP with GeoIP and the User-Agent string. diff --git a/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs.yaml b/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs.yaml index 975da722e2bbf..d8065b01a1f84 100644 --- a/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs.yaml +++ b/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs.yaml @@ -88,6 +88,104 @@ facets: name: User ID path: usr.id source: log + - groups: + - OCSF + name: Activity ID + path: ocsf.activity_id + source: log + type: integer + - groups: + - OCSF + name: Activity Name + path: ocsf.activity_name + source: log + - groups: + - OCSF + name: Category + path: ocsf.category_name + source: log + - groups: + - OCSF + name: Category ID + path: ocsf.category_uid + source: log + type: integer + - groups: + - OCSF + name: Class + path: ocsf.class_name + source: log + - groups: + - OCSF + name: Class ID + path: ocsf.class_uid + source: log + type: integer + - groups: + - OCSF + name: Type ID + path: ocsf.type_uid + source: log + type: integer + - groups: + - OCSF + name: Severity ID + path: ocsf.severity_id + source: log + type: integer + - groups: + - OCSF + name: Status + path: ocsf.status + source: log + - groups: + - OCSF + name: Status ID + path: ocsf.status_id + source: log + type: integer + - groups: + - OCSF + name: Event Code + path: ocsf.metadata.event_code + source: log + - groups: + - OCSF + name: Product Name + path: ocsf.metadata.product.name + source: log + - groups: + - OCSF + name: Vendor Name + path: ocsf.metadata.product.vendor_name + source: log + - groups: + - OCSF + name: Email Address + path: ocsf.actor.user.email_addr + source: log + - groups: + - OCSF + name: Unique ID + path: ocsf.actor.user.uid + source: log + - groups: + - OCSF + name: Source IP Address + path: ocsf.src_endpoint.ip + source: log + - groups: + - OCSF + name: Auth Protocol ID + path: ocsf.auth_protocol_id + source: log + type: integer + - groups: + - OCSF + name: Multi Factor Authentication + path: ocsf.is_mfa + source: log + type: boolean pipeline: type: pipeline name: Claude Compliance Logs @@ -108,7 +206,7 @@ pipeline: sourceType: attribute target: evt.name targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: true - type: attribute-remapper name: Map `actor.email_address` to `usr.email` @@ -118,7 +216,7 @@ pipeline: sourceType: attribute target: usr.email targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: true - type: attribute-remapper name: Map `actor.user_id` to `usr.id` @@ -128,7 +226,7 @@ pipeline: sourceType: attribute target: usr.id targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: true - type: attribute-remapper name: Map `actor.ip_address` to `network.client.ip` @@ -138,7 +236,7 @@ pipeline: sourceType: attribute target: network.client.ip targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: true - type: attribute-remapper name: Map `actor.user_agent` to `http.useragent` @@ -148,7 +246,7 @@ pipeline: sourceType: attribute target: http.useragent targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: true - type: geo-ip-parser name: GeoIP parser on `network.client.ip` @@ -163,3 +261,1539 @@ pipeline: - http.useragent target: http.useragent_details encoded: false + - type: pipeline + name: OCSF pre transformations + enabled: true + ocsf: + isOcsf: true + filter: + query: "*" + processors: + - type: string-builder-processor + name: Add product name + enabled: true + template: "Claude" + target: ocsf.metadata.product.name + replaceMissing: false + - type: string-builder-processor + name: Add product vendor + enabled: true + template: "Anthropic" + target: ocsf.metadata.product.vendor_name + replaceMissing: false + - type: attribute-remapper + name: Map `type` to `ocsf.metadata.event_code` + enabled: true + sources: + - type + sourceType: attribute + target: ocsf.metadata.event_code + targetType: attribute + preserveSource: true + overrideOnConflict: true + - type: attribute-remapper + name: Map `created_at` to `ocsf.metadata.original_time` + enabled: true + sources: + - created_at + sourceType: attribute + target: ocsf.metadata.original_time + targetType: attribute + preserveSource: true + overrideOnConflict: true + - type: grok-parser + name: Parse `created_at` to `ocsf.time` + enabled: true + source: created_at + samples: + - "2026-05-22T15:21:54.358426Z" + grok: + supportRules: "" + matchRules: | + parsing_time %{date("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"):ocsf.time} + parsing_time_ms %{date("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"):ocsf.time} + parsing_time_s %{date("yyyy-MM-dd'T'HH:mm:ss'Z'"):ocsf.time} + - type: pipeline + name: OCSF sub pipeline for class Account Change [3001] - target events + enabled: true + ocsf: + isOcsf: true + filter: + query: "@ocsf.metadata.event_code:(org_user_deleted OR org_user_invite_sent)" + processors: + - type: schema-processor + name: Apply OCSF schema for 3001 + enabled: true + mappers: + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` + sources: + - ocsf.metadata.event_code + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id` to `ocsf.metadata.uid` + sources: + - id + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.original_time` to `ocsf.metadata.original_time` + sources: + - ocsf.metadata.original_time + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address` to `ocsf.actor.user.email_addr` + sources: + - actor.email_address + target: ocsf.actor.user.email_addr + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_id`, `actor.admin_api_key_id` to `ocsf.actor.user.uid` + sources: + - actor.user_id + - actor.admin_api_key_id + target: ocsf.actor.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.ip_address` to `ocsf.src_endpoint.ip` + sources: + - actor.ip_address + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_agent` to `ocsf.http_request.user_agent` + sources: + - actor.user_agent + target: ocsf.http_request.user_agent + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `organization_id`, `organization_uuid` to `ocsf.actor.user.org.uid` + sources: + - organization_id + - organization_uuid + target: ocsf.actor.user.org.uid + preserveSource: true + overrideOnConflict: false + - type: schema-remapper + name: Map `deleted_user_id` to `ocsf.user.uid` + sources: + - deleted_user_id + target: ocsf.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `deleted_user_email`, `invited_email` to `ocsf.user.email_addr` + sources: + - deleted_user_email + - invited_email + target: ocsf.user.email_addr + preserveSource: true + overrideOnConflict: false + - type: schema-remapper + name: Map `deleted_user_email`, `invited_email` to `ocsf.user.name` + sources: + - deleted_user_email + - invited_email + target: ocsf.user.name + preserveSource: true + overrideOnConflict: false + - type: schema-category-mapper + name: ocsf.actor.user.type_id + categories: + - filter: + query: "@actor.type:user_actor" + name: User + id: 1 + - filter: + query: "@actor.type:admin*" + name: Admin + id: 2 + - filter: + query: "-@actor.type:*" + name: Unknown + id: 0 + - filter: + query: "@actor.type:*" + name: Other + id: 99 + targets: + name: ocsf.actor.user.type + id: ocsf.actor.user.type_id + fallback: + values: + ocsf.actor.user.type: Other + ocsf.actor.user.type_id: "99" + sources: + ocsf.actor.user.type: + - actor.type + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@ocsf.metadata.event_code:org_user_invite_sent" + name: Create + id: 1 + - filter: + query: "@ocsf.metadata.event_code:org_user_deleted" + name: Delete + id: 6 + - filter: + query: "@ocsf.metadata.event_code:*" + name: Other + id: 99 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - type + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "*" + name: Success + id: 1 + targets: + name: ocsf.status + id: ocsf.status_id + schema: + schemaType: ocsf + version: 1.5.0 + className: Account Change + classUid: 3001 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Account Change [3001] - self events + enabled: true + ocsf: + isOcsf: true + filter: + query: "@ocsf.metadata.event_code:(org_user_invite_accepted OR claude_user_settings_updated OR *_api_key_*)" + processors: + - type: schema-processor + name: Apply OCSF schema for 3001 + enabled: true + mappers: + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` + sources: + - ocsf.metadata.event_code + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id` to `ocsf.metadata.uid` + sources: + - id + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.original_time` to `ocsf.metadata.original_time` + sources: + - ocsf.metadata.original_time + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address` to `ocsf.actor.user.email_addr` + sources: + - actor.email_address + target: ocsf.actor.user.email_addr + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_id`, `actor.admin_api_key_id` to `ocsf.actor.user.uid` + sources: + - actor.user_id + - actor.admin_api_key_id + target: ocsf.actor.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.ip_address` to `ocsf.src_endpoint.ip` + sources: + - actor.ip_address + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_agent` to `ocsf.http_request.user_agent` + sources: + - actor.user_agent + target: ocsf.http_request.user_agent + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `organization_id`, `organization_uuid` to `ocsf.actor.user.org.uid` + sources: + - organization_id + - organization_uuid + target: ocsf.actor.user.org.uid + preserveSource: true + overrideOnConflict: false + - type: schema-remapper + name: Map `actor.user_id`, `actor.admin_api_key_id` to `ocsf.user.uid` + sources: + - actor.user_id + - actor.admin_api_key_id + target: ocsf.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address` to `ocsf.user.email_addr` + sources: + - actor.email_address + target: ocsf.user.email_addr + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.actor.user.type_id + categories: + - filter: + query: "@actor.type:user_actor" + name: User + id: 1 + - filter: + query: "@actor.type:admin*" + name: Admin + id: 2 + - filter: + query: "-@actor.type:*" + name: Unknown + id: 0 + - filter: + query: "@actor.type:*" + name: Other + id: 99 + targets: + name: ocsf.actor.user.type + id: ocsf.actor.user.type_id + fallback: + values: + ocsf.actor.user.type: Other + ocsf.actor.user.type_id: "99" + sources: + ocsf.actor.user.type: + - actor.type + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@ocsf.metadata.event_code:(*_api_key_created OR org_user_invite_accepted)" + name: Create + id: 1 + - filter: + query: "@ocsf.metadata.event_code:*_api_key_updated AND @updates.current_value:active" + name: Enable + id: 2 + - filter: + query: "@ocsf.metadata.event_code:*_api_key_updated AND @updates.current_value:archived" + name: Disable + id: 5 + - filter: + query: "@ocsf.metadata.event_code:*_api_key_deleted" + name: Delete + id: 6 + - filter: + query: "@ocsf.metadata.event_code:*" + name: Other + id: 99 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - type + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "*" + name: Success + id: 1 + targets: + name: ocsf.status + id: ocsf.status_id + schema: + schemaType: ocsf + version: 1.5.0 + className: Account Change + classUid: 3001 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Authentication [3002] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@ocsf.metadata.event_code:(sso_* OR magic_link_* OR social_login_* OR anonymous_mobile_login_* OR user_logged_out)" + processors: + - type: string-builder-processor + name: Add service name + enabled: true + template: "Claude" + target: ocsf.service.name + replaceMissing: false + - type: schema-processor + name: Apply OCSF schema for 3002 + enabled: true + mappers: + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` + sources: + - ocsf.metadata.event_code + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id` to `ocsf.metadata.uid` + sources: + - id + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.original_time` to `ocsf.metadata.original_time` + sources: + - ocsf.metadata.original_time + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address`, `actor.unauthenticated_email_address` to `ocsf.actor.user.email_addr` + sources: + - actor.email_address + - actor.unauthenticated_email_address + target: ocsf.actor.user.email_addr + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_id` to `ocsf.actor.user.uid` + sources: + - actor.user_id + target: ocsf.actor.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address`, `actor.unauthenticated_email_address` to `ocsf.actor.user.name` + sources: + - actor.email_address + - actor.unauthenticated_email_address + target: ocsf.actor.user.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address`, `actor.unauthenticated_email_address` to `ocsf.user.email_addr` + sources: + - actor.email_address + - actor.unauthenticated_email_address + target: ocsf.user.email_addr + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_id` to `ocsf.user.uid` + sources: + - actor.user_id + target: ocsf.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address`, `actor.unauthenticated_email_address` to `ocsf.user.name` + sources: + - actor.email_address + - actor.unauthenticated_email_address + target: ocsf.user.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.ip_address` to `ocsf.src_endpoint.ip` + sources: + - actor.ip_address + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_agent` to `ocsf.http_request.user_agent` + sources: + - actor.user_agent + target: ocsf.http_request.user_agent + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `organization_id`, `organization_uuid` to `ocsf.actor.user.org.uid` + sources: + - organization_id + - organization_uuid + target: ocsf.actor.user.org.uid + preserveSource: true + overrideOnConflict: false + - type: schema-remapper + name: Map `ocsf.service.name` to `ocsf.service.name` + sources: + - ocsf.service.name + target: ocsf.service.name + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.actor.user.type_id + categories: + - filter: + query: "@actor.type:user_actor" + name: User + id: 1 + - filter: + query: "@actor.type:admin*" + name: Admin + id: 2 + - filter: + query: "-@actor.type:*" + name: Unknown + id: 0 + - filter: + query: "@actor.type:*" + name: Other + id: 99 + targets: + name: ocsf.actor.user.type + id: ocsf.actor.user.type_id + fallback: + values: + ocsf.actor.user.type: Other + ocsf.actor.user.type_id: "99" + sources: + ocsf.actor.user.type: + - actor.type + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@ocsf.metadata.event_code:(sso_* OR magic_link_* OR social_login_* OR anonymous_mobile_login_*)" + name: Logon + id: 1 + - filter: + query: "@ocsf.metadata.event_code:user_logged_out" + name: Logoff + id: 2 + - filter: + query: "@ocsf.metadata.event_code:*" + name: Other + id: 99 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - type + - type: schema-remapper + name: Map `provider` to `ocsf.actor.idp.name` + sources: + - provider + target: ocsf.actor.idp.name + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.auth_protocol_id + categories: + - filter: + query: "@auth_method:sso OR @ocsf.metadata.event_code:sso_*" + name: SAML + id: 5 + - filter: + query: "@auth_method:social OR @ocsf.metadata.event_code:social_login_succeeded" + name: OpenID + id: 4 + - filter: + query: "@ocsf.metadata.event_code:*" + name: Other + id: 99 + targets: + name: ocsf.auth_protocol + id: ocsf.auth_protocol_id + fallback: + values: + ocsf.auth_protocol: Other + ocsf.auth_protocol_id: "99" + sources: + ocsf.auth_protocol: + - auth_method + - type + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "@ocsf.metadata.event_code:(sso_login_succeeded OR magic_link_login_succeeded OR social_login_succeeded OR sso_second_factor_magic_link OR user_logged_out)" + name: Success + id: 1 + - filter: + query: "@ocsf.metadata.event_code:(sso_login_failed OR magic_link_login_failed)" + name: Failure + id: 2 + - filter: + query: "@ocsf.metadata.event_code:(sso_login_initiated OR magic_link_login_initiated OR anonymous_mobile_login_attempted)" + name: Unknown + id: 0 + - filter: + query: "@ocsf.metadata.event_code:*" + name: Other + id: 99 + targets: + name: ocsf.status + id: ocsf.status_id + fallback: + values: + ocsf.status: Other + ocsf.status_id: "99" + sources: + ocsf.status: + - type + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + schema: + schemaType: ocsf + version: 1.5.0 + className: Authentication + classUid: 3002 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class User Access Management [3005] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@ocsf.metadata.event_code:(role_assignment_granted OR role_assignment_revoked)" + processors: + - type: grok-parser + name: Parse `created_at` to `ocsf.time` + enabled: true + source: created_at + samples: + - "2026-05-22T15:21:54.358426Z" + grok: + supportRules: "" + matchRules: | + parsing_time %{date("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"):ocsf.time} + parsing_time_ms %{date("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"):ocsf.time} + parsing_time_s %{date("yyyy-MM-dd'T'HH:mm:ss'Z'"):ocsf.time} + - type: attribute-remapper + name: Map `role` to `ocsf.privilege` + enabled: true + sources: + - role + sourceType: attribute + target: ocsf.privilege + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: array-processor + name: Move privilege into privileges array + enabled: true + operation: + source: ocsf.privilege + target: ocsf.privileges + preserveSource: false + type: append + - type: attribute-remapper + name: Map `resource_id` to `ocsf.resource.uid` + enabled: true + sources: + - resource_id + sourceType: attribute + target: ocsf.resource.uid + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: Map `resource_type` to `ocsf.resource.type` + enabled: true + sources: + - resource_type + sourceType: attribute + target: ocsf.resource.type + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: array-processor + name: Move resource into resources array + enabled: true + operation: + source: ocsf.resource + target: ocsf.resources + preserveSource: false + type: append + - type: schema-processor + name: Apply OCSF schema for 3005 + enabled: true + mappers: + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` + sources: + - ocsf.metadata.event_code + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id` to `ocsf.metadata.uid` + sources: + - id + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.original_time` to `ocsf.metadata.original_time` + sources: + - ocsf.metadata.original_time + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address` to `ocsf.actor.user.email_addr` + sources: + - actor.email_address + target: ocsf.actor.user.email_addr + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_id` to `ocsf.actor.user.uid` + sources: + - actor.user_id + target: ocsf.actor.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.ip_address` to `ocsf.src_endpoint.ip` + sources: + - actor.ip_address + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_agent` to `ocsf.http_request.user_agent` + sources: + - actor.user_agent + target: ocsf.http_request.user_agent + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `organization_id`, `organization_uuid` to `ocsf.user.org.uid` + sources: + - organization_id + - organization_uuid + target: ocsf.user.org.uid + preserveSource: true + overrideOnConflict: false + - type: schema-remapper + name: Map `target_id` to `ocsf.user.uid` + sources: + - target_id + target: ocsf.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.privileges` to `ocsf.privileges` + sources: + - ocsf.privileges + target: ocsf.privileges + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.resources` to `ocsf.resources` + sources: + - ocsf.resources + target: ocsf.resources + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.actor.user.type_id + categories: + - filter: + query: "@actor.type:user_actor" + name: User + id: 1 + - filter: + query: "@actor.type:admin*" + name: Admin + id: 2 + - filter: + query: "-@actor.type:*" + name: Unknown + id: 0 + - filter: + query: "@actor.type:*" + name: Other + id: 99 + targets: + name: ocsf.actor.user.type + id: ocsf.actor.user.type_id + fallback: + values: + ocsf.actor.user.type: Other + ocsf.actor.user.type_id: "99" + sources: + ocsf.actor.user.type: + - actor.type + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@ocsf.metadata.event_code:role_assignment_granted" + name: Assign Privileges + id: 1 + - filter: + query: "@ocsf.metadata.event_code:role_assignment_revoked" + name: Revoke Privileges + id: 2 + - filter: + query: "@ocsf.metadata.event_code:*" + name: Other + id: 99 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - type + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "*" + name: Success + id: 1 + targets: + name: ocsf.status + id: ocsf.status_id + schema: + schemaType: ocsf + version: 1.5.0 + className: User Access Management + classUid: 3005 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Web Resources Activity [6001] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@ocsf.metadata.event_code:(claude_chat_* OR claude_project_* OR claude_file_* OR claude_artifact_* OR claude_skill_*)" + processors: + - type: attribute-remapper + name: Map `claude_chat_id`, `claude_file_id`, `claude_project_document_id`, `claude_artifact_id`, `skill_id`, `claude_project_id` to `ocsf.web_resource.uid` + enabled: true + sources: + - claude_chat_id + - claude_file_id + - claude_project_document_id + - claude_artifact_id + - skill_id + - claude_project_id + sourceType: attribute + target: ocsf.web_resource.uid + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: Map `filename`, `skill_name` to `ocsf.web_resource.name` + enabled: true + sources: + - filename + - skill_name + sourceType: attribute + target: ocsf.web_resource.name + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: array-processor + name: Move web_resource into web_resources array + enabled: true + operation: + source: ocsf.web_resource + target: ocsf.web_resources + preserveSource: false + type: append + - type: schema-processor + name: Apply OCSF schema for 6001 + enabled: true + mappers: + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` + sources: + - ocsf.metadata.event_code + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id` to `ocsf.metadata.uid` + sources: + - id + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.original_time` to `ocsf.metadata.original_time` + sources: + - ocsf.metadata.original_time + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address` to `ocsf.src_endpoint.owner.email_addr` + sources: + - actor.email_address + target: ocsf.src_endpoint.owner.email_addr + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_id`, `actor.admin_api_key_id` to `ocsf.src_endpoint.owner.uid` + sources: + - actor.user_id + - actor.admin_api_key_id + target: ocsf.src_endpoint.owner.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.ip_address` to `ocsf.src_endpoint.ip` + sources: + - actor.ip_address + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_agent` to `ocsf.http_request.user_agent` + sources: + - actor.user_agent + target: ocsf.http_request.user_agent + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `organization_id`, `organization_uuid` to `ocsf.src_endpoint.owner.org.uid` + sources: + - organization_id + - organization_uuid + target: ocsf.src_endpoint.owner.org.uid + preserveSource: true + overrideOnConflict: false + - type: schema-remapper + name: Map `ocsf.web_resources` to `ocsf.web_resources` + sources: + - ocsf.web_resources + target: ocsf.web_resources + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.src_endpoint.owner.type_id + categories: + - filter: + query: "@actor.type:user_actor" + name: User + id: 1 + - filter: + query: "@actor.type:admin*" + name: Admin + id: 2 + - filter: + query: "-@actor.type:*" + name: Unknown + id: 0 + - filter: + query: "@actor.type:*" + name: Other + id: 99 + targets: + name: ocsf.src_endpoint.owner.type + id: ocsf.src_endpoint.owner.type_id + fallback: + values: + ocsf.src_endpoint.owner.type: Other + ocsf.src_endpoint.owner.type_id: "99" + sources: + ocsf.src_endpoint.owner.type: + - actor.type + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@ocsf.metadata.event_code:(*_created OR *_uploaded)" + name: Create + id: 1 + - filter: + query: "@ocsf.metadata.event_code:*_viewed" + name: Read + id: 2 + - filter: + query: "@ocsf.metadata.event_code:(*_updated OR *_replaced)" + name: Update + id: 3 + - filter: + query: "@ocsf.metadata.event_code:*_deleted" + name: Delete + id: 4 + - filter: + query: "@ocsf.metadata.event_code:*" + name: Other + id: 99 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - type + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "*" + name: Success + id: 1 + targets: + name: ocsf.status + id: ocsf.status_id + schema: + schemaType: ocsf + version: 1.5.0 + className: Web Resources Activity + classUid: 6001 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class API Activity [6003] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@ocsf.metadata.event_code:compliance_api_accessed" + processors: + - type: schema-processor + name: Apply OCSF schema for 6003 + enabled: true + mappers: + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` + sources: + - ocsf.metadata.event_code + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id` to `ocsf.metadata.uid` + sources: + - id + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.original_time` to `ocsf.metadata.original_time` + sources: + - ocsf.metadata.original_time + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.api_key_id` to `ocsf.actor.user.uid` + sources: + - actor.api_key_id + target: ocsf.actor.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.api_key_id` to `ocsf.actor.app_uid` + sources: + - actor.api_key_id + target: ocsf.actor.app_uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.ip_address` to `ocsf.src_endpoint.ip` + sources: + - actor.ip_address + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_agent` to `ocsf.http_request.user_agent` + sources: + - actor.user_agent + target: ocsf.http_request.user_agent + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `request_method` to `ocsf.http_request.http_method` + sources: + - request_method + target: ocsf.http_request.http_method + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `url` to `ocsf.http_request.url.url_string` + sources: + - url + target: ocsf.http_request.url.url_string + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `request_id` to `ocsf.http_request.uid` + sources: + - request_id + target: ocsf.http_request.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `status_code` to `ocsf.http_response.code` + sources: + - status_code + target: ocsf.http_response.code + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `request_method` to `ocsf.api.operation` + sources: + - request_method + target: ocsf.api.operation + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `request_id` to `ocsf.api.request.uid` + sources: + - request_id + target: ocsf.api.request.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `status_code` to `ocsf.api.response.code` + sources: + - status_code + target: ocsf.api.response.code + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-category-mapper + name: ocsf.actor.user.type_id + categories: + - filter: + query: "@actor.type:user_actor" + name: User + id: 1 + - filter: + query: "@actor.type:admin*" + name: Admin + id: 2 + - filter: + query: "-@actor.type:*" + name: Unknown + id: 0 + - filter: + query: "@actor.type:*" + name: Other + id: 99 + targets: + name: ocsf.actor.user.type + id: ocsf.actor.user.type_id + fallback: + values: + ocsf.actor.user.type: Other + ocsf.actor.user.type_id: "99" + sources: + ocsf.actor.user.type: + - actor.type + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@request_method:POST" + name: Create + id: 1 + - filter: + query: "@request_method:GET" + name: Read + id: 2 + - filter: + query: "@request_method:(PUT OR PATCH)" + name: Update + id: 3 + - filter: + query: "@request_method:DELETE" + name: Delete + id: 4 + - filter: + query: "@request_method:*" + name: Other + id: 99 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - request_method + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "@status_code:[200 TO 299]" + name: Success + id: 1 + - filter: + query: "@status_code:[400 TO 599]" + name: Failure + id: 2 + - filter: + query: "-@status_code:*" + name: Unknown + id: 0 + - filter: + query: "@status_code:*" + name: Other + id: 99 + targets: + name: ocsf.status + id: ocsf.status_id + fallback: + values: + ocsf.status: Other + ocsf.status_id: "99" + sources: + ocsf.status: + - status_code + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "@status_code:[200 TO 399]" + name: Informational + id: 1 + - filter: + query: "@status_code:[400 TO 499]" + name: Medium + id: 3 + - filter: + query: "@status_code:[500 TO 599]" + name: High + id: 4 + - filter: + query: "-@status_code:*" + name: Unknown + id: 0 + - filter: + query: "@status_code:*" + name: Other + id: 99 + targets: + name: ocsf.severity + id: ocsf.severity_id + fallback: + values: + ocsf.severity: Other + ocsf.severity_id: "99" + sources: + ocsf.severity: + - status_code + schema: + schemaType: ocsf + version: 1.5.0 + className: API Activity + classUid: 6003 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Base Event [0] + enabled: true + ocsf: + isOcsf: true + filter: + query: "-@ocsf.class_uid:*" + processors: + - type: schema-processor + name: Apply OCSF schema for 0 + enabled: true + mappers: + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` + sources: + - ocsf.metadata.event_code + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id` to `ocsf.metadata.uid` + sources: + - id + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.original_time` to `ocsf.metadata.original_time` + sources: + - ocsf.metadata.original_time + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.status + id: ocsf.status_id + schema: + schemaType: ocsf + version: 1.5.0 + className: Base Event + classUid: 0 + extensions: [] + profiles: [] diff --git a/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs_tests.yaml b/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs_tests.yaml index 17d3d7c1f3948..a3fb1fd394044 100644 --- a/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs_tests.yaml +++ b/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs_tests.yaml @@ -1,72 +1,2745 @@ id: "anthropic-compliance-logs" tests: - - sample: |- + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36" + }, + "claude_artifact_id" : "claude_artifact_01EXAMPLEARTIFACT0", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:25:13.701734Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_artifact_viewed" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_artifact_id: "claude_artifact_01EXAMPLEARTIFACT0" + created_at: "2026-05-22T15:25:13.701734Z" + evt: + name: "claude_artifact_viewed" + http: + useragent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36" + useragent_details: + browser: + family: "Chrome" + major: "148" + minor: "0" + patch: "0" + patch_minor: "0" + device: + brand: "Apple" + category: "Desktop" + family: "Mac" + model: "Mac" + os: + family: "Mac OS X" + major: "10" + minor: "15" + patch: "7" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 2 + activity_name: "Read" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36" + metadata: + event_code: "claude_artifact_viewed" + original_time: "2026-05-22T15:25:13.701734Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463513701 + web_resources: + - uid: "claude_artifact_01EXAMPLEARTIFACT0" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_artifact_viewed" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- { "actor" : { "email_address" : "user@example.com", - "user_id" : "user_01FBY4qyk7SdPxJCAd4EfPbT", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36" + }, + "claude_artifact_id" : "claude_artifact_01EXAMPLEARTIFACT0", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:25:13.701734Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_artifact_viewed" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463513701 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.3883.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:54.358426Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_chat_created", + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0 Claude/1.3883.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_chat_id: "claude_chat_01EXAMPLECHATID000000" + created_at: "2026-05-22T15:21:54.358426Z" + evt: + name: "claude_chat_created" + http: + useragent: "Mozilla/5.0 Claude/1.3883.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 1 + activity_name: "Create" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0 Claude/1.3883.0" + metadata: + event_code: "claude_chat_created" + original_time: "2026-05-22T15:21:54.358426Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463314358 + web_resources: + - uid: "claude_chat_01EXAMPLECHATID000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_chat_created" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", "ip_address" : "192.0.2.1", "type" : "user_actor", - "user_agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" + "user_agent" : "Mozilla/5.0 Claude/1.3883.0" }, - "organization_id" : "org_01GuSHHxdWNCcTtk6Wr5arBM", - "organization_uuid" : "80cb55fa-462c-4bc0-82d6-07ebb1a6f004", - "created_at" : "2026-05-05T16:04:57.150724Z", - "id" : "activity_01R1sBnxj7yvtdZnt8DsfpRL", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:54.358426Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_chat_created", + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463314358 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.5354.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:03.415347Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_chat_deleted", + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0 Claude/1.5354.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_chat_id: "claude_chat_01EXAMPLECHATID000000" + created_at: "2026-05-22T15:21:03.415347Z" + evt: + name: "claude_chat_deleted" + http: + useragent: "Mozilla/5.0 Claude/1.5354.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 4 + activity_name: "Delete" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0 Claude/1.5354.0" + metadata: + event_code: "claude_chat_deleted" + original_time: "2026-05-22T15:21:03.415347Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463263415 + web_resources: + - uid: "claude_chat_01EXAMPLECHATID000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_chat_deleted" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.5354.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:03.415347Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_chat_deleted", + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463263415 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:44.621308Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_chat_updated", + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_chat_id: "claude_chat_01EXAMPLECHATID000000" + claude_project_id: "claude_proj_01EXAMPLEPROJECT00000" + created_at: "2026-05-22T15:21:44.621308Z" + evt: + name: "claude_chat_updated" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 3 + activity_name: "Update" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_chat_updated" + original_time: "2026-05-22T15:21:44.621308Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463304621 + web_resources: + - uid: "claude_chat_01EXAMPLECHATID000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_chat_updated" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:44.621308Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_chat_updated", + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463304621 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:53.556370Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_chat_viewed", + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_chat_id: "claude_chat_01EXAMPLECHATID000000" + created_at: "2026-05-22T15:21:53.556370Z" + evt: + name: "claude_chat_viewed" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 2 + activity_name: "Read" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_chat_viewed" + original_time: "2026-05-22T15:21:53.556370Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463313556 + web_resources: + - uid: "claude_chat_01EXAMPLECHATID000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_chat_viewed" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:53.556370Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", "type" : "claude_chat_viewed", - "claude_chat_id" : "claude_chat_01AxWT9aH4swoDJ8u6dShxMV" + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463313556 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "filename" : "example-image.png", + "claude_file_id" : "claude_file_01EXAMPLEFILEID000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:32.702968Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_file_uploaded" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_file_id: "claude_file_01EXAMPLEFILEID000000" + created_at: "2026-05-22T15:21:32.702968Z" + evt: + name: "claude_file_uploaded" + filename: "example-image.png" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 1 + activity_name: "Create" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_file_uploaded" + original_time: "2026-05-22T15:21:32.702968Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463292702 + web_resources: + - name: "example-image.png" + uid: "claude_file_01EXAMPLEFILEID000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_file_uploaded" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "filename" : "example-image.png", + "claude_file_id" : "claude_file_01EXAMPLEFILEID000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:32.702968Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_file_uploaded" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463292702 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.3883.0" + }, + "filename" : "example-screenshot.png", + "claude_file_id" : "claude_file_01EXAMPLEFILEID000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:50.616332Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_file_viewed" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0 Claude/1.3883.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_file_id: "claude_file_01EXAMPLEFILEID000000" + created_at: "2026-05-22T15:21:50.616332Z" + evt: + name: "claude_file_viewed" + filename: "example-screenshot.png" + http: + useragent: "Mozilla/5.0 Claude/1.3883.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 2 + activity_name: "Read" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0 Claude/1.3883.0" + metadata: + event_code: "claude_file_viewed" + original_time: "2026-05-22T15:21:50.616332Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463310616 + web_resources: + - name: "example-screenshot.png" + uid: "claude_file_01EXAMPLEFILEID000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_file_viewed" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.3883.0" + }, + "filename" : "example-screenshot.png", + "claude_file_id" : "claude_file_01EXAMPLEFILEID000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:50.616332Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_file_viewed" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463310616 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:10.594873Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_created" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_project_id: "claude_proj_01EXAMPLEPROJECT00000" + created_at: "2026-05-22T15:21:10.594873Z" + evt: + name: "claude_project_created" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 1 + activity_name: "Create" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_project_created" + original_time: "2026-05-22T15:21:10.594873Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463270594 + web_resources: + - uid: "claude_proj_01EXAMPLEPROJECT00000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_project_created" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:10.594873Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_created" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463270594 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "filename" : "example-document.pdf", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:23:05.845058Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_document_uploaded", + "claude_project_document_id" : "claude_proj_doc_01EXAMPLEDOC0000000000" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_project_document_id: "claude_proj_doc_01EXAMPLEDOC0000000000" + claude_project_id: "claude_proj_01EXAMPLEPROJECT00000" + created_at: "2026-05-22T15:23:05.845058Z" + evt: + name: "claude_project_document_uploaded" + filename: "example-document.pdf" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 1 + activity_name: "Create" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_project_document_uploaded" + original_time: "2026-05-22T15:23:05.845058Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463385845 + web_resources: + - name: "example-document.pdf" + uid: "claude_proj_doc_01EXAMPLEDOC0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_project_document_uploaded" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "filename" : "example-document.pdf", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:23:05.845058Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_document_uploaded", + "claude_project_document_id" : "claude_proj_doc_01EXAMPLEDOC0000000000" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463385845 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "filename" : "example-file.txt", + "claude_file_id" : "claude_file_01EXAMPLEFILEID000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:20:23.462825Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_file_uploaded" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_file_id: "claude_file_01EXAMPLEFILEID000000" + claude_project_id: "claude_proj_01EXAMPLEPROJECT00000" + created_at: "2026-05-22T15:20:23.462825Z" + evt: + name: "claude_project_file_uploaded" + filename: "example-file.txt" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 1 + activity_name: "Create" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_project_file_uploaded" + original_time: "2026-05-22T15:20:23.462825Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463223462 + web_resources: + - name: "example-file.txt" + uid: "claude_file_01EXAMPLEFILEID000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_project_file_uploaded" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "filename" : "example-file.txt", + "claude_file_id" : "claude_file_01EXAMPLEFILEID000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:20:23.462825Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_file_uploaded" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463223462 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "preview_only" : false, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:48.506301Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_viewed" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_project_id: "claude_proj_01EXAMPLEPROJECT00000" + created_at: "2026-05-22T15:21:48.506301Z" + evt: + name: "claude_project_viewed" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 2 + activity_name: "Read" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_project_viewed" + original_time: "2026-05-22T15:21:48.506301Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463308506 + web_resources: + - uid: "claude_proj_01EXAMPLEPROJECT00000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + preview_only: false + type: "claude_project_viewed" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "preview_only" : false, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:48.506301Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_viewed" } - result: - custom: + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463308506 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.3883.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "skill_name" : "example-skill-name", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:44.267747Z", + "skill_id" : "skill_01EXAMPLESKILLID00000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_skill_created" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0 Claude/1.3883.0" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-05-22T15:21:44.267747Z" + evt: + name: "claude_skill_created" + http: + useragent: "Mozilla/5.0 Claude/1.3883.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 1 + activity_name: "Create" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0 Claude/1.3883.0" + metadata: + event_code: "claude_skill_created" + original_time: "2026-05-22T15:21:44.267747Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463304267 + web_resources: + - name: "example-skill-name" + uid: "skill_01EXAMPLESKILLID00000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + skill_id: "skill_01EXAMPLESKILLID00000000" + skill_name: "example-skill-name" + type: "claude_skill_created" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.3883.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "skill_name" : "example-skill-name", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:44.267747Z", + "skill_id" : "skill_01EXAMPLESKILLID00000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_skill_created" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463304267 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.7196.1" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "skill_name" : "example-skill-name", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:17.287525Z", + "skill_id" : "skill_01EXAMPLESKILLID00000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_skill_replaced" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0 Claude/1.7196.1" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-05-22T15:21:17.287525Z" + evt: + name: "claude_skill_replaced" + http: + useragent: "Mozilla/5.0 Claude/1.7196.1" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 3 + activity_name: "Update" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0 Claude/1.7196.1" + metadata: + event_code: "claude_skill_replaced" + original_time: "2026-05-22T15:21:17.287525Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463277287 + web_resources: + - name: "example-skill-name" + uid: "skill_01EXAMPLESKILLID00000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + skill_id: "skill_01EXAMPLESKILLID00000000" + skill_name: "example-skill-name" + type: "claude_skill_replaced" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.7196.1" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "skill_name" : "example-skill-name", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:17.287525Z", + "skill_id" : "skill_01EXAMPLESKILLID00000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_skill_replaced" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463277287 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "created_at" : "2026-05-22T15:21:50.371418Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_user_settings_updated", + "updates" : [ { + "previous_value" : { + "example_tool" : false + }, + "type" : "mcp_tools_enabled", + "current_value" : { + "example_tool" : true + } + } ] + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-05-22T15:21:50.371418Z" + evt: + name: "claude_user_settings_updated" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 99 + activity_name: "claude_user_settings_updated" actor: - type: "user_actor" - claude_chat_id: "claude_chat_01AxWT9aH4swoDJ8u6dShxMV" - created_at: "2026-05-05T16:04:57.150724Z" - evt: - name: "claude_chat_viewed" - http: - useragent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" - useragent_details: - browser: - family: "Safari" - major: "17" - minor: "0" - device: - brand: "Apple" - category: "Desktop" - family: "Mac" - model: "Mac" - os: - family: "Mac OS X" - major: "10" - minor: "15" - patch: "7" - id: "activity_01R1sBnxj7yvtdZnt8DsfpRL" - network: - client: - geoip: {} - ip: "192.0.2.1" - organization_id: "org_01GuSHHxdWNCcTtk6Wr5arBM" - organization_uuid: "80cb55fa-462c-4bc0-82d6-07ebb1a6f004" - usr: - email: "user@example.com" - id: "user_01FBY4qyk7SdPxJCAd4EfPbT" - message: |- - { - "actor" : { - "email_address" : "user@example.com", - "user_id" : "user_01FBY4qyk7SdPxJCAd4EfPbT", - "ip_address" : "192.0.2.1", - "type" : "user_actor", - "user_agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" + user: + email_addr: "user@example.com" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Account Change" + class_uid: 3001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_user_settings_updated" + original_time: "2026-05-22T15:21:50.371418Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Success" + status_id: 1 + time: 1779463310371 + user: + email_addr: "user@example.com" + uid: "user_01EXAMPLEUSERID0000000000" + type: "claude_user_settings_updated" + updates: + - current_value: + example_tool: true + previous_value: + example_tool: false + type: "mcp_tools_enabled" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "created_at" : "2026-05-22T15:21:50.371418Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_user_settings_updated", + "updates" : [ { + "previous_value" : { + "example_tool" : false }, - "organization_id" : "org_01GuSHHxdWNCcTtk6Wr5arBM", - "organization_uuid" : "80cb55fa-462c-4bc0-82d6-07ebb1a6f004", - "created_at" : "2026-05-05T16:04:57.150724Z", - "id" : "activity_01R1sBnxj7yvtdZnt8DsfpRL", - "type" : "claude_chat_viewed", - "claude_chat_id" : "claude_chat_01AxWT9aH4swoDJ8u6dShxMV" - } - tags: - - "source:LOGS_SOURCE" - timestamp: 1777997097150 + "type" : "mcp_tools_enabled", + "current_value" : { + "example_tool" : true + } + } ] + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463310371 + - + sample: |- + { + "actor" : { + "ip_address" : "192.0.2.1", + "type" : "api_actor", + "api_key_id" : "apikey_01EXAMPLEAPIKEY000000000", + "user_agent" : "example-client/1.0" + }, + "status_code" : 200, + "created_at" : "2026-05-22T15:21:38.920308Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "request_method" : "GET", + "type" : "compliance_api_accessed", + "request_id" : "req_011CbEXAMPLEREQUEST00000", + "url" : "https://api.anthropic.com/v1/compliance/activities?" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + api_key_id: "apikey_01EXAMPLEAPIKEY000000000" + ip_address: "192.0.2.1" + type: "api_actor" + user_agent: "example-client/1.0" + created_at: "2026-05-22T15:21:38.920308Z" + evt: + name: "compliance_api_accessed" + http: + useragent: "example-client/1.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 2 + activity_name: "Read" + actor: + app_uid: "apikey_01EXAMPLEAPIKEY000000000" + user: + type: "api_actor" + type_id: 99 + uid: "apikey_01EXAMPLEAPIKEY000000000" + api: + operation: "GET" + request: + uid: "req_011CbEXAMPLEREQUEST00000" + response: + code: 200 + category_name: "Application Activity" + category_uid: 6 + class_name: "API Activity" + class_uid: 6003 + http_request: + http_method: "GET" + uid: "req_011CbEXAMPLEREQUEST00000" + url: + url_string: "https://api.anthropic.com/v1/compliance/activities?" + user_agent: "example-client/1.0" + http_response: + code: 200 + metadata: + event_code: "compliance_api_accessed" + original_time: "2026-05-22T15:21:38.920308Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Success" + status_id: 1 + time: 1779463298920 + request_id: "req_011CbEXAMPLEREQUEST00000" + request_method: "GET" + status_code: 200 + type: "compliance_api_accessed" + url: "https://api.anthropic.com/v1/compliance/activities?" + message: |- + { + "actor" : { + "ip_address" : "192.0.2.1", + "type" : "api_actor", + "api_key_id" : "apikey_01EXAMPLEAPIKEY000000000", + "user_agent" : "example-client/1.0" + }, + "status_code" : 200, + "created_at" : "2026-05-22T15:21:38.920308Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "request_method" : "GET", + "type" : "compliance_api_accessed", + "request_id" : "req_011CbEXAMPLEREQUEST00000", + "url" : "https://api.anthropic.com/v1/compliance/activities?" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463298920 + - + sample: |- + { + "actor" : { + "admin_api_key_id" : "admin_api_key_01EXAMPLEADMIN0000", + "ip_address" : "192.0.2.1", + "type" : "admin_api_key_actor", + "user_agent" : "python-requests/2.32.5" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:07:03.419782Z", + "deleted_user_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "org_user_deleted" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + admin_api_key_id: "admin_api_key_01EXAMPLEADMIN0000" + ip_address: "192.0.2.1" + type: "admin_api_key_actor" + user_agent: "python-requests/2.32.5" + created_at: "2026-05-22T15:07:03.419782Z" + deleted_user_id: "user_01EXAMPLEUSERID0000000000" + evt: + name: "org_user_deleted" + http: + useragent: "python-requests/2.32.5" + useragent_details: + browser: + family: "Python Requests" + major: "2" + minor: "32" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 6 + activity_name: "Delete" + actor: + user: + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "Admin" + type_id: 2 + uid: "admin_api_key_01EXAMPLEADMIN0000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Account Change" + class_uid: 3001 + http_request: + user_agent: "python-requests/2.32.5" + metadata: + event_code: "org_user_deleted" + original_time: "2026-05-22T15:07:03.419782Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Success" + status_id: 1 + time: 1779462423419 + user: + uid: "user_01EXAMPLEUSERID0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "org_user_deleted" + message: |- + { + "actor" : { + "admin_api_key_id" : "admin_api_key_01EXAMPLEADMIN0000", + "ip_address" : "192.0.2.1", + "type" : "admin_api_key_actor", + "user_agent" : "python-requests/2.32.5" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:07:03.419782Z", + "deleted_user_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "org_user_deleted" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779462423419 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "deleted_user_email" : "user@example.com", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-04-22T12:38:58.658822Z", + "deleted_user_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "org_user_deleted" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-04-22T12:38:58.658822Z" + deleted_user_email: "user@example.com" + deleted_user_id: "user_01EXAMPLEUSERID0000000000" + evt: + name: "org_user_deleted" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 6 + activity_name: "Delete" + actor: + user: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Account Change" + class_uid: 3001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "org_user_deleted" + original_time: "2026-04-22T12:38:58.658822Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Success" + status_id: 1 + time: 1776861538658 + user: + email_addr: "user@example.com" + name: "user@example.com" + uid: "user_01EXAMPLEUSERID0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "org_user_deleted" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "deleted_user_email" : "user@example.com", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-04-22T12:38:58.658822Z", + "deleted_user_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "org_user_deleted" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1776861538658 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "invite_id" : "invite_01EXAMPLEINVITE0000000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-15T15:22:37.490878Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "org_user_invite_accepted" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-05-15T15:22:37.490878Z" + evt: + name: "org_user_invite_accepted" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + invite_id: "invite_01EXAMPLEINVITE0000000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 1 + activity_name: "Create" + actor: + user: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Account Change" + class_uid: 3001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "org_user_invite_accepted" + original_time: "2026-05-15T15:22:37.490878Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Success" + status_id: 1 + time: 1778858557490 + user: + email_addr: "user@example.com" + uid: "user_01EXAMPLEUSERID0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "org_user_invite_accepted" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "invite_id" : "invite_01EXAMPLEINVITE0000000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-15T15:22:37.490878Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "org_user_invite_accepted" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1778858557490 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "invited_role" : "owner", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-15T15:22:11.738584Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "invited_email" : "invitee@example.com", + "type" : "org_user_invite_sent" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-05-15T15:22:11.738584Z" + evt: + name: "org_user_invite_sent" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + invited_email: "invitee@example.com" + invited_role: "owner" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 1 + activity_name: "Create" + actor: + user: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Account Change" + class_uid: 3001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "org_user_invite_sent" + original_time: "2026-05-15T15:22:11.738584Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + status: "Success" + status_id: 1 + time: 1778858531738 + user: + email_addr: "invitee@example.com" + name: "invitee@example.com" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "org_user_invite_sent" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "invited_role" : "owner", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-15T15:22:11.738584Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "invited_email" : "invitee@example.com", + "type" : "org_user_invite_sent" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1778858531738 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:23.115384Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "platform_api_key_created", + "api_key_id" : "apikey_01EXAMPLEAPIKEY000000000" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + api_key_id: "apikey_01EXAMPLEAPIKEY000000000" + created_at: "2026-05-22T15:21:23.115384Z" + evt: + name: "platform_api_key_created" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 1 + activity_name: "Create" + actor: + user: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Account Change" + class_uid: 3001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "platform_api_key_created" + original_time: "2026-05-22T15:21:23.115384Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + status: "Success" + status_id: 1 + time: 1779463283115 + user: + email_addr: "user@example.com" + uid: "user_01EXAMPLEUSERID0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "platform_api_key_created" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:23.115384Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "platform_api_key_created", + "api_key_id" : "apikey_01EXAMPLEAPIKEY000000000" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463283115 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:23:11.707169Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "platform_api_key_updated", + "updates" : [ { + "previous_value" : "active", + "type" : "status", + "current_value" : "archived" + } ], + "api_key_id" : "apikey_01EXAMPLEAPIKEY000000000" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + api_key_id: "apikey_01EXAMPLEAPIKEY000000000" + created_at: "2026-05-22T15:23:11.707169Z" + evt: + name: "platform_api_key_updated" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 5 + activity_name: "Disable" + actor: + user: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Account Change" + class_uid: 3001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "platform_api_key_updated" + original_time: "2026-05-22T15:23:11.707169Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + status: "Success" + status_id: 1 + time: 1779463391707 + user: + email_addr: "user@example.com" + uid: "user_01EXAMPLEUSERID0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "platform_api_key_updated" + updates: + - current_value: "archived" + previous_value: "active" + type: "status" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:23:11.707169Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "platform_api_key_updated", + "updates" : [ { + "previous_value" : "active", + "type" : "status", + "current_value" : "archived" + } ], + "api_key_id" : "apikey_01EXAMPLEAPIKEY000000000" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463391707 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "role" : "chat_project:owner", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "resource_type" : "chat_project", + "target_type" : "organization_member", + "created_at" : "2026-05-22T15:21:10.596119Z", + "resource_id" : "claude_proj_01EXAMPLEPROJECT00000", + "target_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "role_assignment_granted" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-05-22T15:21:10.596119Z" + evt: + name: "role_assignment_granted" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 1 + activity_name: "Assign Privileges" + actor: + user: + email_addr: "user@example.com" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "User Access Management" + class_uid: 3005 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "role_assignment_granted" + original_time: "2026-05-22T15:21:10.596119Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + privileges: + - "chat_project:owner" + resources: + - type: "chat_project" + uid: "claude_proj_01EXAMPLEPROJECT00000" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + status: "Success" + status_id: 1 + time: 1779463270596 + user: + org: + uid: "org_01EXAMPLEORGID00000000000" + uid: "user_01EXAMPLEUSERID0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + resource_id: "claude_proj_01EXAMPLEPROJECT00000" + resource_type: "chat_project" + role: "chat_project:owner" + target_id: "user_01EXAMPLEUSERID0000000000" + target_type: "organization_member" + type: "role_assignment_granted" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "role" : "chat_project:owner", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "resource_type" : "chat_project", + "target_type" : "organization_member", + "created_at" : "2026-05-22T15:21:10.596119Z", + "resource_id" : "claude_proj_01EXAMPLEPROJECT00000", + "target_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "role_assignment_granted" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463270596 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.6259.0" + }, + "role" : "skill:viewer", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "resource_type" : "skill", + "target_type" : "organization_member", + "created_at" : "2026-05-21T20:47:11.194821Z", + "resource_id" : "skill_01EXAMPLESKILLID00000000", + "target_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "role_assignment_revoked" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0 Claude/1.6259.0" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-05-21T20:47:11.194821Z" + evt: + name: "role_assignment_revoked" + http: + useragent: "Mozilla/5.0 Claude/1.6259.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 2 + activity_name: "Revoke Privileges" + actor: + user: + email_addr: "user@example.com" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "User Access Management" + class_uid: 3005 + http_request: + user_agent: "Mozilla/5.0 Claude/1.6259.0" + metadata: + event_code: "role_assignment_revoked" + original_time: "2026-05-21T20:47:11.194821Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + privileges: + - "skill:viewer" + resources: + - type: "skill" + uid: "skill_01EXAMPLESKILLID00000000" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Success" + status_id: 1 + time: 1779396431194 + user: + org: + uid: "org_01EXAMPLEORGID00000000000" + uid: "user_01EXAMPLEUSERID0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + resource_id: "skill_01EXAMPLESKILLID00000000" + resource_type: "skill" + role: "skill:viewer" + target_id: "user_01EXAMPLEUSERID0000000000" + target_type: "organization_member" + type: "role_assignment_revoked" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.6259.0" + }, + "role" : "skill:viewer", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "resource_type" : "skill", + "target_type" : "organization_member", + "created_at" : "2026-05-21T20:47:11.194821Z", + "resource_id" : "skill_01EXAMPLESKILLID00000000", + "target_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "role_assignment_revoked" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779396431194 + - + sample: |- + { + "actor" : { + "unauthenticated_email_address" : "user@example.com", + "ip_address" : "192.0.2.1", + "type" : "unauthenticated_user_actor", + "user_agent" : "Mozilla/5.0" + }, + "created_at" : "2026-05-22T15:19:09.946100Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "sso_login_initiated" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + ip_address: "192.0.2.1" + type: "unauthenticated_user_actor" + unauthenticated_email_address: "user@example.com" + user_agent: "Mozilla/5.0" + created_at: "2026-05-22T15:19:09.946100Z" + evt: + name: "sso_login_initiated" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 1 + activity_name: "Logon" + actor: + user: + email_addr: "user@example.com" + name: "user@example.com" + type: "unauthenticated_user_actor" + type_id: 99 + auth_protocol: "SAML" + auth_protocol_id: 5 + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Authentication" + class_uid: 3002 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "sso_login_initiated" + original_time: "2026-05-22T15:19:09.946100Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + service: + name: "Claude" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Unknown" + status_id: 0 + time: 1779463149946 + user: + email_addr: "user@example.com" + name: "user@example.com" + type: "sso_login_initiated" + message: |- + { + "actor" : { + "unauthenticated_email_address" : "user@example.com", + "ip_address" : "192.0.2.1", + "type" : "unauthenticated_user_actor", + "user_agent" : "Mozilla/5.0" + }, + "created_at" : "2026-05-22T15:19:09.946100Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "sso_login_initiated" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463149946 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "auth_method" : "sso", + "created_at" : "2026-05-22T15:19:14.445010Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "sso_login_succeeded" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + auth_method: "sso" + created_at: "2026-05-22T15:19:14.445010Z" + evt: + name: "sso_login_succeeded" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 1 + activity_name: "Logon" + actor: + user: + email_addr: "user@example.com" + name: "user@example.com" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + auth_protocol: "SAML" + auth_protocol_id: 5 + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Authentication" + class_uid: 3002 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "sso_login_succeeded" + original_time: "2026-05-22T15:19:14.445010Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + service: + name: "Claude" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Success" + status_id: 1 + time: 1779463154445 + user: + email_addr: "user@example.com" + name: "user@example.com" + uid: "user_01EXAMPLEUSERID0000000000" + type: "sso_login_succeeded" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "auth_method" : "sso", + "created_at" : "2026-05-22T15:19:14.445010Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "sso_login_succeeded" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463154445 From 5b6408438a7a226a97314d225ad48593bad13a7b Mon Sep 17 00:00:00 2001 From: Arthur Garreau <51090691+agarov@users.noreply.github.com> Date: Thu, 28 May 2026 16:31:28 +0200 Subject: [PATCH 18/44] [Traefik Mesh] Fix Traefik Mesh dashboard service tag (#23613) * Fix Traefik Mesh dashboard service tag * Traefik mesh changelog * Remove fixed changelog entry for Traefik Mesh * Apply suggestion from @buraizu Remove extra blank Co-authored-by: Bryce Eadie --------- Co-authored-by: Bryce Eadie --- .../dashboards/traefik_mesh_overview.json | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/traefik_mesh/assets/dashboards/traefik_mesh_overview.json b/traefik_mesh/assets/dashboards/traefik_mesh_overview.json index f621603991fc8..55708c1c9afe1 100644 --- a/traefik_mesh/assets/dashboards/traefik_mesh_overview.json +++ b/traefik_mesh/assets/dashboards/traefik_mesh_overview.json @@ -584,17 +584,17 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:traefik_mesh.router.requests.count{$host,$service,$router,code:2*} by {code,router,service}.as_count()" + "query": "sum:traefik_mesh.router.requests.count{$host,$traefik_service,$router,code:2*} by {code,router,traefik_service}.as_count()" }, { "data_source": "metrics", "name": "query2", - "query": "sum:traefik_mesh.router.requests.count{$host AND $service AND $router AND code:4* OR code:5*} by {code,router,service}.as_count()" + "query": "sum:traefik_mesh.router.requests.count{$host AND $traefik_service AND $router AND code:4* OR code:5*} by {code,router,traefik_service}.as_count()" }, { "data_source": "metrics", "name": "query3", - "query": "sum:traefik_mesh.router.requests.count{$host,$service,$routercode:3*} by {code,router,service}.as_count()" + "query": "sum:traefik_mesh.router.requests.count{$host,$traefik_service,$router,code:3*} by {code,router,traefik_service}.as_count()" } ], "response_format": "timeseries", @@ -633,7 +633,7 @@ { "name": "query1", "data_source": "metrics", - "query": "sum:traefik_mesh.router.requests.count{$router, $endpoint} by {protocol,service}", + "query": "sum:traefik_mesh.router.requests.count{$router, $endpoint} by {protocol,traefik_service}", "aggregator": "sum" } ], @@ -694,7 +694,7 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:traefik_mesh.router.responses.bytes.count{$service, $router, $host} by {service,router,host,protocol}" + "query": "sum:traefik_mesh.router.responses.bytes.count{$traefik_service, $router, $host} by {traefik_service,router,host,protocol}" } ], "response_format": "timeseries", @@ -900,17 +900,17 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:traefik_mesh.service.requests.count{$host,$service,$endpoint,code:2*} by {protocol,code,method}.as_count()" + "query": "sum:traefik_mesh.service.requests.count{$host,$traefik_service,$endpoint,code:2*} by {protocol,code,method}.as_count()" }, { "data_source": "metrics", "name": "query2", - "query": "sum:traefik_mesh.service.requests.count{$host AND $service AND $endpoint AND code:4* OR code:5*} by {protocol,code,method}.as_count()" + "query": "sum:traefik_mesh.service.requests.count{$host AND $traefik_service AND $endpoint AND code:4* OR code:5*} by {protocol,code,method}.as_count()" }, { "data_source": "metrics", "name": "query3", - "query": "sum:traefik_mesh.service.requests.count{$host,$service ,$endpoint,code:3*} by {protocol,code,method}.as_count()" + "query": "sum:traefik_mesh.service.requests.count{$host,$traefik_service,$endpoint,code:3*} by {protocol,code,method}.as_count()" } ], "response_format": "timeseries", @@ -948,7 +948,7 @@ { "name": "query1", "data_source": "metrics", - "query": "avg:traefik_mesh.service.requests.count{$host} by {protocol,service,router}", + "query": "avg:traefik_mesh.service.requests.count{$host} by {protocol,traefik_service,router}", "aggregator": "sum" } ], @@ -1009,7 +1009,7 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:traefik_mesh.service.responses.bytes.count{$host, $service, $endpoint} by {host,service,endpoint}" + "query": "sum:traefik_mesh.service.responses.bytes.count{$host, $traefik_service, $endpoint} by {host,traefik_service,endpoint}" } ], "response_format": "timeseries", @@ -1058,7 +1058,7 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:traefik_mesh.service.server.up{$host, $service, $endpoint} by {service}" + "query": "sum:traefik_mesh.service.server.up{$host, $traefik_service, $endpoint} by {traefik_service}" } ], "response_format": "timeseries", @@ -1098,7 +1098,7 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:traefik_mesh.service.request.duration.seconds.sum{$host, $service, $endpoint} by {service,endpoint}" + "query": "sum:traefik_mesh.service.request.duration.seconds.sum{$host, $traefik_service, $endpoint} by {traefik_service,endpoint}" } ], "response_format": "timeseries", @@ -1452,8 +1452,8 @@ "default": "*" }, { - "name": "service", - "prefix": "service", + "name": "traefik_service", + "prefix": "traefik_service", "available_values": [], "default": "*" }, @@ -1485,4 +1485,4 @@ "layout_type": "ordered", "notify_list": [], "reflow_type": "fixed" -} \ No newline at end of file +} From 853007939309535d3a4c37354bc89d40d2e6f933 Mon Sep 17 00:00:00 2001 From: Eric Weaver Date: Thu, 28 May 2026 12:56:26 -0400 Subject: [PATCH 19/44] [SDBM-2637] Restore agent hostname fallback resolution for SQL Server named instance configs (#23862) * Reintroduce old agent hostname fallback * Update test * Add changelog --- sqlserver/changelog.d/23862.fixed | 1 + .../datadog_checks/sqlserver/sqlserver.py | 3 + sqlserver/tests/test_unit.py | 66 +++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 sqlserver/changelog.d/23862.fixed diff --git a/sqlserver/changelog.d/23862.fixed b/sqlserver/changelog.d/23862.fixed new file mode 100644 index 0000000000000..816b836c62be4 --- /dev/null +++ b/sqlserver/changelog.d/23862.fixed @@ -0,0 +1 @@ +Restore agent hostname instrumentation for SQL Server named instance host configurations. \ No newline at end of file diff --git a/sqlserver/datadog_checks/sqlserver/sqlserver.py b/sqlserver/datadog_checks/sqlserver/sqlserver.py index 3953b8f1bd3fb..8d15b54580c5d 100644 --- a/sqlserver/datadog_checks/sqlserver/sqlserver.py +++ b/sqlserver/datadog_checks/sqlserver/sqlserver.py @@ -328,6 +328,9 @@ def port(self): return self.host_and_port[1] def resolve_db_host(self): + if "\\" in self.host: + # SQL Server instance names are not resolvable, this preserves original fallback behavior prior to v7.79.0 + return datadog_agent.get_hostname() return agent_host_resolver(self.host) @property diff --git a/sqlserver/tests/test_unit.py b/sqlserver/tests/test_unit.py index 21e43ecd686aa..99b2d7790826a 100644 --- a/sqlserver/tests/test_unit.py +++ b/sqlserver/tests/test_unit.py @@ -11,6 +11,7 @@ import mock import pytest +from datadog_checks.base.stubs.datadog_agent import datadog_agent from datadog_checks.dev import EnvVars from datadog_checks.sqlserver import SQLServer from datadog_checks.sqlserver.connection import split_sqlserver_host_port @@ -908,6 +909,71 @@ def test_split_sqlserver_host(instance_host, split_host, split_port): assert (s_host, s_port) == (split_host, split_port) +AGENT_HOSTNAME = 'sql-agent-host.example.com' + + +@pytest.fixture +def agent_hostname_for_resolve_db_host(): + datadog_agent.set_hostname(AGENT_HOSTNAME) + yield + datadog_agent.reset_hostname() + + +@pytest.mark.parametrize( + 'instance_host,host_part', + [ + (r'SQL-HOST01\INSTANCE01,1601', r'SQL-HOST01\INSTANCE01'), + (r'MY-SERVER\SQLEXPRESS,1433', r'MY-SERVER\SQLEXPRESS'), + (r'MY-SERVER\SQLEXPRESS', r'MY-SERVER\SQLEXPRESS'), + ], +) +def test_resolve_db_host_named_instance_returns_agent_hostname( + agent_hostname_for_resolve_db_host, instance_host, host_part +): + instance = { + 'host': instance_host, + 'username': 'datadog', + 'password': 'secret', + } + check = SQLServer(CHECK_NAME, {}, [instance]) + assert check.host == host_part + + # Agent 7.79+ base resolver returns the literal host string for unresolvable names. + with mock.patch( + 'datadog_checks.sqlserver.sqlserver.agent_host_resolver', + return_value=host_part, + ): + assert check.resolve_db_host() == AGENT_HOSTNAME + assert check.resolved_hostname == AGENT_HOSTNAME + assert check.database_hostname == AGENT_HOSTNAME + + +@pytest.mark.parametrize( + 'instance_host,host_part,base_resolver_return', + [ + ('db.example.com,1433', 'db.example.com', 'resolved-db.example.com'), + ('192.0.2.10,1433', '192.0.2.10', '192.0.2.10'), + ], +) +def test_resolve_db_host_plain_host_delegates_to_base_resolver( + agent_hostname_for_resolve_db_host, instance_host, host_part, base_resolver_return +): + instance = { + 'host': instance_host, + 'username': 'datadog', + 'password': 'secret', + } + check = SQLServer(CHECK_NAME, {}, [instance]) + assert check.host == host_part + + with mock.patch( + 'datadog_checks.sqlserver.sqlserver.agent_host_resolver', + return_value=base_resolver_return, + ) as mock_resolver: + assert check.resolve_db_host() == base_resolver_return + mock_resolver.assert_called_once_with(host_part) + + @pytest.mark.parametrize( "query,expected_comments,is_proc,expected_name", [ From c7ef3522d7c1084ff8a4d94f10775478c6509c74 Mon Sep 17 00:00:00 2001 From: jbfeldman-dd Date: Thu, 28 May 2026 15:29:34 -0400 Subject: [PATCH 20/44] [azure_active_directory] Add grok parsers to extract port from IP:port (#23870) * [azure_active_directory] Add grok parsers to extract port from IP:port in ocsf.src_endpoint.ip and network.client.ip Co-Authored-By: Claude Opus 4.6 * [azure_active_directory] Add grok parsers to extract port from IP:port in ocsf.src_endpoint.ip and network.client.ip Co-Authored-By: Claude Opus 4.6 * add facet * Fix test expectations: port values are strings from grok parser Co-Authored-By: Claude Opus 4.6 * Add integer cast remappers for port fields after grok parsers Co-Authored-By: Claude Opus 4.6 * Move ocsf.src_endpoint.port integer cast to post transformations pipeline Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- .../assets/logs/azure.activedirectory.yaml | 92 +++++++++++++++++++ .../logs/azure.activedirectory_tests.yaml | 14 +-- 2 files changed, 100 insertions(+), 6 deletions(-) diff --git a/azure_active_directory/assets/logs/azure.activedirectory.yaml b/azure_active_directory/assets/logs/azure.activedirectory.yaml index e8050796da873..c39db0e784b51 100644 --- a/azure_active_directory/assets/logs/azure.activedirectory.yaml +++ b/azure_active_directory/assets/logs/azure.activedirectory.yaml @@ -83,6 +83,11 @@ facets: name: Client IP path: network.client.ip source: log + - groups: + - Web Access + name: Client Port + path: network.client.port + source: log - groups: - User name: User Email @@ -200,6 +205,13 @@ facets: name: Source IP Address path: ocsf.src_endpoint.ip source: log + - facetType: range + groups: + - OCSF + name: Src Endpoint Port + path: ocsf.src_endpoint.port + source: log + type: integer - groups: - OCSF name: Event Code @@ -281,6 +293,29 @@ pipeline: overrideOnConflict: false sourceType: attribute targetType: attribute + - type: grok-parser + name: Parse `network.client.ip` to `network.client.ip`, `network.client.port` + enabled: true + source: network.client.ip + grok: + supportRules: | + matchRules: | + ipv4_rule %{ipv4:network.client.ip}(:%{port:network.client.port})? + ipv6_rule \[?%{ipv6:network.client.ip}\]?(:%{port:network.client.port})? + samples: + - 15.113.255.209 + - 15.113.255.209:21341 + - type: attribute-remapper + name: Map `network.client.port` to `network.client.port` + enabled: true + sources: + - network.client.port + sourceType: attribute + target: network.client.port + targetType: attribute + targetFormat: integer + preserveSource: false + overrideOnConflict: false - type: arithmetic-processor name: Compute duration in nanoseconds from durationMs in miliseconds enabled: true @@ -548,6 +583,18 @@ pipeline: targetType: attribute preserveSource: true overrideOnConflict: false + - type: grok-parser + name: Parse `ocsf.src_endpoint.ip` to `ocsf.src_endpoint.ip`, `ocsf.src_endpoint.port` + enabled: true + source: ocsf.src_endpoint.ip + grok: + supportRules: | + matchRules: | + ipv4_rule %{ipv4:ocsf.src_endpoint.ip}(:%{port:ocsf.src_endpoint.port})? + ipv6_rule \[?%{ipv6:ocsf.src_endpoint.ip}\]?(:%{port:ocsf.src_endpoint.port})? + samples: + - 15.113.255.209 + - 15.113.255.209:21341 - type: attribute-remapper name: Map `properties.resultReason` to `ocsf.status_code` enabled: true @@ -1019,6 +1066,18 @@ pipeline: targetType: attribute preserveSource: true overrideOnConflict: false + - type: grok-parser + name: Parse `ocsf.src_endpoint.ip` to `ocsf.src_endpoint.ip`, `ocsf.src_endpoint.port` + enabled: true + source: ocsf.src_endpoint.ip + grok: + supportRules: | + matchRules: | + ipv4_rule %{ipv4:ocsf.src_endpoint.ip}(:%{port:ocsf.src_endpoint.port})? + ipv6_rule \[?%{ipv6:ocsf.src_endpoint.ip}\]?(:%{port:ocsf.src_endpoint.port})? + samples: + - 15.113.255.209 + - 15.113.255.209:21341 - type: attribute-remapper name: Map `properties.deviceDetail.operatingSystem` to `ocsf.src_endpoint.os.name` enabled: true @@ -1877,6 +1936,18 @@ pipeline: targetType: attribute preserveSource: true overrideOnConflict: false + - type: grok-parser + name: Parse `ocsf.src_endpoint.ip` to `ocsf.src_endpoint.ip`, `ocsf.src_endpoint.port` + enabled: true + source: ocsf.src_endpoint.ip + grok: + supportRules: | + matchRules: | + ipv4_rule %{ipv4:ocsf.src_endpoint.ip}(:%{port:ocsf.src_endpoint.port})? + ipv6_rule \[?%{ipv6:ocsf.src_endpoint.ip}\]?(:%{port:ocsf.src_endpoint.port})? + samples: + - 15.113.255.209 + - 15.113.255.209:21341 - type: string-builder-processor name: Add dst_endpoint.hostname enabled: true @@ -2194,6 +2265,16 @@ pipeline: targetType: attribute preserveSource: false overrideOnConflict: false + - type: attribute-remapper + name: Map `ocsf.src_endpoint.port` to `network.client.port` + enabled: true + sources: + - ocsf.src_endpoint.port + sourceType: attribute + target: callerIpAddress + targetType: attribute + preserveSource: false + overrideOnConflict: false - type: pipeline name: OCSF post transformations enabled: true @@ -2296,3 +2377,14 @@ pipeline: targetFormat: integer preserveSource: false overrideOnConflict: false + - type: attribute-remapper + name: Map `ocsf.src_endpoint.port` to `ocsf.src_endpoint.port` + enabled: true + sources: + - ocsf.src_endpoint.port + sourceType: attribute + target: ocsf.src_endpoint.port + targetType: attribute + targetFormat: integer + preserveSource: false + overrideOnConflict: false diff --git a/azure_active_directory/assets/logs/azure.activedirectory_tests.yaml b/azure_active_directory/assets/logs/azure.activedirectory_tests.yaml index 64cebba79dbcd..5d8e13a038638 100644 --- a/azure_active_directory/assets/logs/azure.activedirectory_tests.yaml +++ b/azure_active_directory/assets/logs/azure.activedirectory_tests.yaml @@ -328,7 +328,7 @@ tests: "tenantId": "4d3bac44-0230-4732-9e70-cc00736f0a97", "resultSignature": "None", "durationMs": 0, - "callerIpAddress": "192.182.149.21", + "callerIpAddress": "192.182.149.21:43210", "correlationId": "a13bd0fa-70d0-4e60-ae23-b687377b4695", "Level": 4, "properties": { @@ -353,7 +353,7 @@ tests: "id": "018af091-5465-4aed-9d6f-8c40981b2375", "displayName": null, "userPrincipalName": "test.test@datadoghq.com", - "ipAddress": "192.182.149.21", + "ipAddress": "192.182.149.21:43210", "roles": [] } }, @@ -384,7 +384,7 @@ tests: result: custom: Level: 4 - callerIpAddress: "192.182.149.21" + callerIpAddress: "192.182.149.21:43210" category: "AuditLogs" correlationId: "a13bd0fa-70d0-4e60-ae23-b687377b4695" duration: 0.0 @@ -397,6 +397,7 @@ tests: client: geoip: {} ip: "192.182.149.21" + port: 43210 ocsf: activity_id: 6 activity_name: "Delete" @@ -427,6 +428,7 @@ tests: severity_id: 1 src_endpoint: ip: "192.182.149.21" + port: 43210 status: "Success" status_code: "" status_id: 1 @@ -453,7 +455,7 @@ tests: initiatedBy: user: id: "018af091-5465-4aed-9d6f-8c40981b2375" - ipAddress: "192.182.149.21" + ipAddress: "192.182.149.21:43210" userPrincipalName: "test.test@datadoghq.com" loggedByService: "Core Directory" operationName: "Delete user" @@ -481,7 +483,7 @@ tests: name: "test.test@datadoghq.com" message: |- { - "callerIpAddress" : "192.182.149.21", + "callerIpAddress" : "192.182.149.21:43210", "resourceId" : "/tenants/4d3bac44-0230-4732-9e70-cc00736f0a97/providers/Microsoft.aadiam", "operationVersion" : "1.0", "tenantId" : "4d3bac44-0230-4732-9e70-cc00736f0a97", @@ -522,7 +524,7 @@ tests: "resultType" : "", "initiatedBy" : { "user" : { - "ipAddress" : "192.182.149.21", + "ipAddress" : "192.182.149.21:43210", "id" : "018af091-5465-4aed-9d6f-8c40981b2375", "userPrincipalName" : "test.test@datadoghq.com" } From 3c44c778343c3ea48be2b1c493de4dc7137adffe Mon Sep 17 00:00:00 2001 From: Juanpe Araque Date: Fri, 29 May 2026 08:48:03 +0100 Subject: [PATCH 21/44] Add file-based YAML metrics loading for OpenMetrics V2 (#22750) * Add file-based YAML metrics loading for OpenMetrics V2 Add a new module `metrics_file.py` with: - MetricsPredicate protocol for conditional file loading - ConfigOptionTruthy, ConfigOptionEquals predicates - AllOf, AnyOf composable predicates - MetricsFile dataclass for declaring YAML metric files - MetricsConfig type alias Modify OpenMetricsBaseCheckV2 to: - Add METRICS_FILES class variable for declaring metric files - Load metrics from YAML files in get_config_with_defaults - Support convention-based discovery of metrics.yml - Add _load_file_based_metrics and _load_metrics_file helpers Include 39 unit tests covering all predicates, file loading, convention-based discovery, and config integration. * Add changelog entry for file-based metrics loading * Address PR review feedback - Add error handling in _load_metrics_file: catch yaml.YAMLError and validate loaded data is a dict, raising RuntimeError with descriptive messages for both cases - Document AllOf/AnyOf behavior with zero predicates (follows Python's all()/any() semantics) in docstrings - Refactor tests: convert from class-based to plain functions, use pytest fixtures for common setup, parametrize predicate tests - Add error scenario tests: malformed YAML, empty files, non-dict content * Address review feedback for file-based metrics loading Renames: - MetricsFile -> MetricsMapping (better describes the concept) - MetricsConfig -> RawMetricsConfig (clearer purpose) - metrics_file.py -> metrics_mapping.py - METRICS_FILES -> METRICS_MAP Improvements: - Add should_load() to MetricsMapping dataclass to encapsulate predicate evaluation - Extract _apply_file_metrics() helper so subclasses can build their own defaults dict without re-implementing file loading - Move yaml import inside _load_metrics_file to avoid loading pyyaml for integrations that don't use file-based metrics - Move typing imports behind TYPE_CHECKING - Use self.METRICS_MAP directly instead of local variable rename - Trim docstrings: shorter, newline after opening quotes, concise examples - Consolidate tests: plain functions, parametrize composites, extract _write_yaml helper, remove duplicate test patterns (370 -> 230 lines) * Fix file metrics mutation, add caching, and support metrics.yaml extension - Remove _apply_file_metrics in favour of inlining the logic in get_config_with_defaults, making the ownership of defaults explicit - Build a fresh list with list() + file_metrics instead of extend() so that any list returned by get_default_config is never mutated in place - Add instance-level _file_metrics cache on _load_file_based_metrics so YAML files are read only once per check instance regardless of how many times create_scraper is called (e.g. via refresh_scrapers overrides) - Support both metrics.yaml and metrics.yml for convention discovery, with metrics.yaml taking precedence when both are present - Document the contract on get_default_config: the returned dict can be mutated by the framework so subclasses must return a new dict each call * Apply ruff formatting * Improve type checking and tests * Update dependency resolution * Address review: ConfigurationError, public exports, fail-once cache - Export MetricsMapping, MetricsPredicate, ConfigOptionTruthy/Equals, AllOf, AnyOf from v2 package via lazy_loader stub. - Raise ConfigurationError (instead of RuntimeError) for malformed/unreadable YAML; catch OSError so FS errors surface as configuration errors. - Seal _file_metrics = [] before the load attempt so a malformed file fails once rather than re-raising on every scrape. - Document that overrides of get_config_with_defaults must call super() to keep file-based loading active. - Tests: vacuous AllOf()/AnyOf(), cache suppresses predicate re-evaluation on config change, permanent-failure-fails-once. * Harden METRICS_MAP defaults and metrics-file diagnostics - Use an immutable tuple as the METRICS_MAP class default to prevent subclasses from corrupting the base via .append; document yaml-then-yml convention discovery precedence. - Shallow-copy get_default_config() in get_config_with_defaults so a subclass that returns a cached top-level dict cannot accumulate file metrics across instances. - Reference the absolute file_path (not the relative path argument) in ConfigurationError messages so misconfigured metrics files name the directory being searched. - Patch _get_package_dir in test_load_file_based_metrics_no_files; add coverage for the cached-defaults pathway. * Tighten metrics_mapping internals and surface - Hoist yaml import to module top; yaml is a hard dependency of datadog_checks_base, not optional. - Chain ConfigurationError to the originating exception so YAML parse/IO context (line, column, errno) is preserved in stack traces. - Rename RawMetricsConfig to _RawMetricsConfig: it's the internal loader return type, not part of the integration-author surface. - Document the seal-on-any-error contract in _load_file_based_metrics: a single bad file in METRICS_MAP discards earlier successes; cache lands as []. - Document that ConfigOptionEquals treats a missing key as equal to None. - Add multi-file partial-success test and a lazy_loader stub smoke test for the v2 public re-exports. * Resolve metrics file paths in the caller and widen loader return type - Resolve METRICS_MAP file paths once in _load_file_based_metrics; _load_metrics_file now takes an absolute path. Removes the duplicate _get_package_dir lookup and makes the helper self-contained, so tests can pass a real path instead of patching the lookup. - Widen _RawMetricsConfig inner value to dict[str, Any] so the alias matches the transformer shapes the scraper consumes downstream. - Use tuple literals in the MetricsMapping docstring example and the inline test subclasses, matching the tuple[MetricsMapping, ...] annotation hardened in the previous commit. --------- Co-authored-by: dd-agent-integrations-bot[bot] --- datadog_checks_base/changelog.d/22750.added | 1 + .../base/checks/openmetrics/v2/__init__.py | 3 + .../base/checks/openmetrics/v2/__init__.pyi | 22 + .../base/checks/openmetrics/v2/base.py | 91 +++- .../checks/openmetrics/v2/metrics_mapping.py | 118 +++++ .../test_v2/test_metrics_mapping.py | 422 ++++++++++++++++++ 6 files changed, 655 insertions(+), 2 deletions(-) create mode 100644 datadog_checks_base/changelog.d/22750.added create mode 100644 datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/__init__.pyi create mode 100644 datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/metrics_mapping.py create mode 100644 datadog_checks_base/tests/base/checks/openmetrics/test_v2/test_metrics_mapping.py diff --git a/datadog_checks_base/changelog.d/22750.added b/datadog_checks_base/changelog.d/22750.added new file mode 100644 index 0000000000000..e8a49db170139 --- /dev/null +++ b/datadog_checks_base/changelog.d/22750.added @@ -0,0 +1 @@ +Add file-based YAML metrics loading for OpenMetrics V2 checks with composable predicates \ No newline at end of file diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/__init__.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/__init__.py index 46dd167dcde48..cc1fa89df93d3 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/__init__.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/__init__.py @@ -1,3 +1,6 @@ # (C) Datadog, Inc. 2020-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import lazy_loader + +__getattr__, __dir__, __all__ = lazy_loader.attach_stub(__name__, __file__) diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/__init__.pyi b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/__init__.pyi new file mode 100644 index 0000000000000..a2c49c408d85c --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/__init__.pyi @@ -0,0 +1,22 @@ +# (C) Datadog, Inc. 2025-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from .base import OpenMetricsBaseCheckV2 +from .metrics_mapping import ( + AllOf, + AnyOf, + ConfigOptionEquals, + ConfigOptionTruthy, + MetricsMapping, + MetricsPredicate, +) + +__all__ = [ + 'AllOf', + 'AnyOf', + 'ConfigOptionEquals', + 'ConfigOptionTruthy', + 'MetricsMapping', + 'MetricsPredicate', + 'OpenMetricsBaseCheckV2', +] diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py index 1a476138157de..2fb5062e146bf 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py @@ -1,9 +1,14 @@ # (C) Datadog, Inc. 2020-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + from collections import ChainMap from contextlib import contextmanager +from pathlib import Path +from typing import TYPE_CHECKING +import yaml from requests.exceptions import RequestException from datadog_checks.base.checks import AgentCheck @@ -12,6 +17,11 @@ from .scraper import OpenMetricsScraper +if TYPE_CHECKING: + from collections.abc import Mapping + + from .metrics_mapping import MetricsMapping, _RawMetricsConfig + class OpenMetricsBaseCheckV2(AgentCheck): """ @@ -32,6 +42,14 @@ class OpenMetricsBaseCheckV2(AgentCheck): DEFAULT_METRIC_LIMIT = 2000 + METRICS_MAP: tuple[MetricsMapping, ...] = () + """YAML files with metric name mappings to load automatically. + + When empty (default), looks for ``metrics.yaml`` next to the check module, + falling back to ``metrics.yml`` if the former is absent. When set, only the + declared files are loaded (with predicates controlling conditional loading). + """ + # Allow tracing for openmetrics integrations def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) @@ -52,6 +70,9 @@ def __init__(self, name, init_config, instances): # All configured scrapers keyed by the endpoint self.scrapers = {} + # Cache for file-based metrics loaded from METRICS_MAP; None means not yet loaded + self._file_metrics: list[_RawMetricsConfig] | None = None + self.check_initializations.append(self.configure_scrapers) def check(self, _): @@ -105,14 +126,80 @@ def set_dynamic_tags(self, *tags): scraper.set_dynamic_tags(*tags) def get_config_with_defaults(self, config): - return ChainMap(config, self.get_default_config()) + """Combine instance config with class defaults and file-based metric mappings. + + Subclasses that override this method must call ``super().get_config_with_defaults(config)``; + otherwise the YAML mappings declared via ``METRICS_MAP`` (or discovered by convention) are + silently skipped. + """ + defaults = dict(self.get_default_config()) + if file_metrics := self._load_file_based_metrics(config): + defaults['metrics'] = list(defaults.get('metrics', [])) + file_metrics + return ChainMap(config, defaults) - def get_default_config(self): + def get_default_config(self) -> dict: + """Return instance-level default scraper configuration values. + + The returned dict can be mutated by the framework before being wrapped + in a ``ChainMap``. Avoid returning a shared or instance-level object to avoid + state leakage between check executions. + """ return {} def refresh_scrapers(self): pass + def _load_file_based_metrics(self, config: Mapping) -> list[_RawMetricsConfig]: + """Load metric mappings from YAML files declared in ``METRICS_MAP``. + + Results are cached for the lifetime of the check instance. Predicates + are evaluated once against the first ``config`` supplied; ``METRICS_MAP`` + is a class-level declaration and the instance config does not change + between runs, so subsequent calls always receive the same effective + configuration. + + Falls back to convention-based discovery of ``metrics.yaml`` or + ``metrics.yml`` (in that order) when ``METRICS_MAP`` is empty. + + Permanent load failures (malformed YAML, unreadable files) are raised + once on the first call; the cache is sealed beforehand so subsequent + scrapes do not retry and re-raise the same error. A failure on any + single file in a multi-file ``METRICS_MAP`` discards results from + files loaded earlier in the same call: the cache lands as ``[]``, not + as a partial mapping. + """ + if self._file_metrics is not None: + return self._file_metrics + + self._file_metrics = [] + package_dir = self._get_package_dir() + if not self.METRICS_MAP: + for candidate in (Path("metrics.yaml"), Path("metrics.yml")): + resolved = package_dir / candidate + if resolved.is_file(): + self._file_metrics = [self._load_metrics_file(resolved)] + break + else: + self._file_metrics = [ + self._load_metrics_file(package_dir / source.path) + for source in self.METRICS_MAP + if source.should_load(config) + ] + + return self._file_metrics + + def _load_metrics_file(self, file_path: Path) -> _RawMetricsConfig: + try: + with open(file_path) as f: + data = yaml.safe_load(f) + except yaml.YAMLError as e: + raise ConfigurationError(f"Failed to parse metrics file {file_path}: {e}") from e + except OSError as e: + raise ConfigurationError(f"Failed to read metrics file {file_path}: {e}") from e + if not isinstance(data, dict): + raise ConfigurationError(f"Metrics file {file_path} must contain a YAML mapping, got {type(data).__name__}") + return data + @contextmanager def adopt_namespace(self, namespace): old_namespace = self.__NAMESPACE__ diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/metrics_mapping.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/metrics_mapping.py new file mode 100644 index 0000000000000..3a4582b849dab --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/metrics_mapping.py @@ -0,0 +1,118 @@ +# (C) Datadog, Inc. 2025-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, Protocol + +from datadog_checks.base.config import is_affirmative + +if TYPE_CHECKING: + from datadog_checks.base.types import InstanceType + +_RawMetricsConfig = dict[str, str | dict[str, Any]] +"""Metric name mapping loaded from a YAML file. + +Keys are raw Prometheus metric names; values are either Datadog metric names +(simple renaming) or dicts describing transformer configuration. The inner +dict carries the full ``OpenMetricsScraper`` transformer shape (type, label +maps, nested options), so it is intentionally widened to ``dict[str, Any]``. + +Internal: the type the loader returns; integration authors should not need to +reference this directly. +""" + + +class MetricsPredicate(Protocol): + """ + Protocol for predicates that control whether a metrics mapping should be loaded. + + Implement ``should_load`` to create custom loading conditions. + """ + + def should_load(self, config: InstanceType) -> bool: ... + + +class ConfigOptionTruthy: + """ + Load metrics only if a configuration option is truthy. + + Uses ``is_affirmative`` to evaluate the value. Defaults to ``True`` + (include metrics unless explicitly disabled). + """ + + def __init__(self, option: str, default: bool = True) -> None: + self.option = option + self.default = default + + def should_load(self, config: InstanceType) -> bool: + return is_affirmative(config.get(self.option, self.default)) + + +class ConfigOptionEquals: + """ + Load metrics only if a configuration option equals a specific value. + + A missing key compares equal to ``None``: ``ConfigOptionEquals("flag", None)`` + matches both ``{"flag": None}`` and instances that omit the key entirely. + """ + + def __init__(self, option: str, value: Any) -> None: + self.option = option + self.value = value + + def should_load(self, config: InstanceType) -> bool: + return config.get(self.option) == self.value + + +class AllOf: + """ + Compose predicates: all must pass for the metrics to be loaded. + + Follows Python's ``all()`` semantics: returns ``True`` when empty. + """ + + def __init__(self, *predicates: MetricsPredicate) -> None: + self.predicates = predicates + + def should_load(self, config: InstanceType) -> bool: + return all(p.should_load(config) for p in self.predicates) + + +class AnyOf: + """ + Compose predicates: any passing is sufficient to load the metrics. + + Follows Python's ``any()`` semantics: returns ``False`` when empty. + """ + + def __init__(self, *predicates: MetricsPredicate) -> None: + self.predicates = predicates + + def should_load(self, config: InstanceType) -> bool: + return any(p.should_load(config) for p in self.predicates) + + +@dataclass(frozen=True) +class MetricsMapping: + """ + Declares a YAML file with metric name mappings to load automatically. + + Use in the ``METRICS_MAP`` class variable of ``OpenMetricsBaseCheckV2`` + subclasses. The YAML file should contain a flat mapping of Prometheus + metric names to Datadog metric names:: + + METRICS_MAP = ( + MetricsMapping(Path("metrics/default.yaml")), + MetricsMapping(Path("metrics/go.yaml"), predicate=ConfigOptionTruthy("go_metrics")), + ) + """ + + path: Path + predicate: MetricsPredicate | None = None + + def should_load(self, config: InstanceType) -> bool: + """Return whether this mapping should be loaded for the given config.""" + return self.predicate is None or self.predicate.should_load(config) diff --git a/datadog_checks_base/tests/base/checks/openmetrics/test_v2/test_metrics_mapping.py b/datadog_checks_base/tests/base/checks/openmetrics/test_v2/test_metrics_mapping.py new file mode 100644 index 0000000000000..12bbe22440b91 --- /dev/null +++ b/datadog_checks_base/tests/base/checks/openmetrics/test_v2/test_metrics_mapping.py @@ -0,0 +1,422 @@ +# (C) Datadog, Inc. 2025-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from pathlib import Path +from typing import Protocol +from unittest.mock import patch + +import pytest +import yaml + +from datadog_checks.base import OpenMetricsBaseCheckV2 +from datadog_checks.base.checks.openmetrics.v2.metrics_mapping import ( + AllOf, + AnyOf, + ConfigOptionEquals, + ConfigOptionTruthy, + MetricsMapping, +) +from datadog_checks.base.errors import ConfigurationError + +_DEFAULT_INSTANCE: dict[str, object] = {'openmetrics_endpoint': 'http://test:9090/metrics'} + + +class CheckFactory(Protocol): + def __call__( + self, + cls: type[OpenMetricsBaseCheckV2] | None = None, + instance: dict[str, object] | None = None, + ) -> OpenMetricsBaseCheckV2: ... + + +@pytest.fixture +def make_check() -> CheckFactory: + def factory(cls=None, instance=None): + cls = cls or OpenMetricsBaseCheckV2 + inst = _DEFAULT_INSTANCE | (instance or {}) + c = cls('test', {}, [inst]) + c.__NAMESPACE__ = 'test' + return c + + return factory + + +def write_yaml(tmp_path, filename, data): + path = tmp_path / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.dump(data)) + return path + + +# --------------------------------------------------------------------------- +# ConfigOptionTruthy +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "config, default, expected", + [ + ({"opt": True}, True, True), + ({"opt": False}, True, False), + ({}, True, True), + ({}, False, False), + ({"opt": "yes"}, True, True), + ({"opt": "no"}, True, False), + ], + ids=["true", "false", "missing_default_true", "missing_default_false", "string_yes", "string_no"], +) +def test_config_option_truthy(config: dict[str, object], default: bool, expected: bool): + pred = ConfigOptionTruthy("opt", default=default) + assert pred.should_load(config) is expected + + +# --------------------------------------------------------------------------- +# ConfigOptionEquals +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "config, value, expected", + [ + ({"mode": "advanced"}, "advanced", True), + ({"mode": "basic"}, "advanced", False), + ({}, "advanced", False), + ({}, None, True), + ], + ids=["equal", "not_equal", "missing", "none_matches_missing"], +) +def test_config_option_equals(config: dict[str, str], value: str | None, expected: bool): + pred = ConfigOptionEquals("mode", value) + assert pred.should_load(config) is expected + + +# --------------------------------------------------------------------------- +# AllOf / AnyOf +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "cls, config, expected", + [ + (AllOf, {"a": True, "b": True}, True), + (AllOf, {"a": True, "b": False}, False), + (AllOf, {"a": False, "b": False}, False), + (AnyOf, {"a": True, "b": True}, True), + (AnyOf, {"a": True, "b": False}, True), + (AnyOf, {"a": False, "b": False}, False), + ], + ids=["all_both", "all_one", "all_none", "any_both", "any_one", "any_none"], +) +def test_composite_predicates(cls: type[AllOf] | type[AnyOf], config: dict[str, bool], expected: bool): + pred = cls(ConfigOptionTruthy("a"), ConfigOptionTruthy("b")) + assert pred.should_load(config) is expected + + +@pytest.mark.parametrize( + "cls, expected", + [ + (AllOf, True), + (AnyOf, False), + ], + ids=["all_of_vacuous_truth", "any_of_vacuous_falsity"], +) +def test_composite_predicates_empty(cls: type[AllOf] | type[AnyOf], expected: bool): + assert cls().should_load({}) is expected + + +# --------------------------------------------------------------------------- +# MetricsMapping +# --------------------------------------------------------------------------- + + +def test_metrics_mapping_is_frozen(): + mm = MetricsMapping(Path("m.yaml")) + with pytest.raises(AttributeError): + mm.path = Path("other.yaml") + + +def test_metrics_mapping_should_load_no_predicate(): + mm = MetricsMapping(Path("m.yaml")) + assert mm.should_load({}) is True + + +def test_metrics_mapping_should_load_with_predicate(): + mm = MetricsMapping(Path("m.yaml"), predicate=ConfigOptionTruthy("opt", default=False)) + assert mm.should_load({}) is False + assert mm.should_load({"opt": True}) is True + + +# --------------------------------------------------------------------------- +# _load_metrics_file +# --------------------------------------------------------------------------- + + +def test_load_metrics_file(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "go.yaml", {"go_goroutines": "go.goroutines"}) + check = make_check() + assert check._load_metrics_file(tmp_path / "go.yaml") == {"go_goroutines": "go.goroutines"} + + +@pytest.mark.parametrize( + "filename, content, match", + [ + ("broken.yaml", "foo: [bar", "Failed to parse"), + ("empty.yaml", "", "must contain a YAML mapping, got NoneType"), + ("list.yaml", "[1, 2, 3]", "must contain a YAML mapping, got list"), + ], + ids=["malformed", "empty", "non_dict"], +) +def test_load_metrics_file_errors(make_check: CheckFactory, tmp_path: Path, filename: str, content: str, match: str): + (tmp_path / filename).write_text(content) + check = make_check() + with pytest.raises(ConfigurationError, match=match): + check._load_metrics_file(tmp_path / filename) + + +def test_load_metrics_file_missing(make_check: CheckFactory, tmp_path: Path): + check = make_check() + with pytest.raises(ConfigurationError, match="Failed to read metrics file"): + check._load_metrics_file(tmp_path / "nonexistent.yaml") + + +# --------------------------------------------------------------------------- +# _load_file_based_metrics +# --------------------------------------------------------------------------- + + +def test_load_file_based_metrics_no_files(make_check: CheckFactory, tmp_path: Path): + check = make_check() + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + assert check._load_file_based_metrics({}) == [] + + +@pytest.mark.parametrize("filename", ["metrics.yaml", "metrics.yml"]) +def test_load_file_based_metrics_convention_discovery(make_check: CheckFactory, tmp_path: Path, filename: str): + write_yaml(tmp_path, filename, {"raw": "dd.raw"}) + check = make_check() + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + result = check._load_file_based_metrics({}) + assert result == [{"raw": "dd.raw"}] + + +def test_load_file_based_metrics_convention_yaml_takes_precedence(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "metrics.yaml", {"from_yaml": "dd.yaml"}) + write_yaml(tmp_path, "metrics.yml", {"from_yml": "dd.yml"}) + check = make_check() + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + result = check._load_file_based_metrics({}) + assert result == [{"from_yaml": "dd.yaml"}] + + +def test_load_file_based_metrics_explicit(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "metrics/a.yaml", {"m1": "d1"}) + write_yaml(tmp_path, "metrics/b.yaml", {"m2": "d2"}) + + class Check(OpenMetricsBaseCheckV2): + METRICS_MAP = (MetricsMapping(Path("metrics/a.yaml")), MetricsMapping(Path("metrics/b.yaml"))) + + check = make_check(cls=Check) + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + result = check._load_file_based_metrics({}) + assert result == [{"m1": "d1"}, {"m2": "d2"}] + + +def test_load_file_based_metrics_predicate_filters(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "metrics/always.yaml", {"m1": "d1"}) + write_yaml(tmp_path, "metrics/conditional.yaml", {"m2": "d2"}) + + class Check(OpenMetricsBaseCheckV2): + METRICS_MAP = ( + MetricsMapping(Path("metrics/always.yaml")), + MetricsMapping(Path("metrics/conditional.yaml"), predicate=ConfigOptionTruthy("extra", default=False)), + ) + + check_base = make_check(cls=Check) + check_extra = make_check(cls=Check, instance={'extra': True}) + with patch.object(Check, '_get_package_dir', return_value=tmp_path): + assert len(check_base._load_file_based_metrics(check_base.instance)) == 1 + assert len(check_extra._load_file_based_metrics(check_extra.instance)) == 2 + + +def test_load_file_based_metrics_explicit_skips_convention(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "metrics.yml", {"convention": "metric"}) + write_yaml(tmp_path, "explicit.yaml", {"explicit": "metric"}) + + class Check(OpenMetricsBaseCheckV2): + METRICS_MAP = (MetricsMapping(Path("explicit.yaml")),) + + check = make_check(cls=Check) + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + result = check._load_file_based_metrics({}) + assert result == [{"explicit": "metric"}] + + +# --------------------------------------------------------------------------- +# _load_file_based_metrics caching +# --------------------------------------------------------------------------- + + +def test_load_file_based_metrics_cached_across_calls(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "metrics.yml", {"raw": "dd.raw"}) + check = make_check() + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + first = check._load_file_based_metrics({}) + second = check._load_file_based_metrics({}) + assert first is second + + +def test_load_file_based_metrics_cache_ignores_config_changes(make_check: CheckFactory, tmp_path: Path): + """Predicate re-evaluation is suppressed once the cache is populated: a second call with a + different config returns the first-call result without consulting the predicates again.""" + write_yaml(tmp_path, "metrics/always.yaml", {"m1": "d1"}) + write_yaml(tmp_path, "metrics/extra.yaml", {"m2": "d2"}) + + class Check(OpenMetricsBaseCheckV2): + METRICS_MAP = ( + MetricsMapping(Path("metrics/always.yaml")), + MetricsMapping(Path("metrics/extra.yaml"), predicate=ConfigOptionTruthy("extra", default=False)), + ) + + check = make_check(cls=Check) + with patch.object(Check, '_get_package_dir', return_value=tmp_path): + first = check._load_file_based_metrics({'extra': False}) + second = check._load_file_based_metrics({'extra': True}) + assert first is second + assert len(first) == 1 + + +def test_load_file_based_metrics_permanent_failure_fails_once(make_check: CheckFactory, tmp_path: Path): + """A malformed YAML file raises on the first call; subsequent calls return the empty cache.""" + (tmp_path / "metrics.yml").write_text("foo: [bar") + check = make_check() + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + with pytest.raises(ConfigurationError, match="Failed to parse"): + check._load_file_based_metrics({}) + assert check._load_file_based_metrics({}) == [] + + +def test_load_file_based_metrics_multi_file_failure_seals_empty(make_check: CheckFactory, tmp_path: Path): + """A mid-comprehension load failure discards earlier successes; the cache lands as [].""" + write_yaml(tmp_path, "metrics/a.yaml", {"a": "dd.a"}) + (tmp_path / "metrics" / "b.yaml").write_text("foo: [bar") + write_yaml(tmp_path, "metrics/c.yaml", {"c": "dd.c"}) + + class Check(OpenMetricsBaseCheckV2): + METRICS_MAP = ( + MetricsMapping(Path("metrics/a.yaml")), + MetricsMapping(Path("metrics/b.yaml")), + MetricsMapping(Path("metrics/c.yaml")), + ) + + check = make_check(cls=Check) + with patch.object(Check, '_get_package_dir', return_value=tmp_path): + with pytest.raises(ConfigurationError, match="Failed to parse"): + check._load_file_based_metrics({}) + assert check._load_file_based_metrics({}) == [] + + +def test_load_file_based_metrics_does_not_accumulate_on_repeated_scraper_creation( + make_check: CheckFactory, tmp_path: Path +): + """Repeated create_scraper calls (e.g. from refresh_scrapers) must not grow the metrics list.""" + write_yaml(tmp_path, "metrics.yml", {"raw": "dd.raw"}) + check = make_check() + instance = {'openmetrics_endpoint': 'http://test:9090/metrics'} + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + config_first = check.get_config_with_defaults(instance) + config_second = check.get_config_with_defaults(instance) + assert config_first['metrics'] == config_second['metrics'] + + +def test_load_file_based_metrics_does_not_mutate_get_default_config(make_check: CheckFactory, tmp_path: Path): + """File metrics must not mutate the list returned by get_default_config.""" + write_yaml(tmp_path, "metrics.yml", {"raw": "dd.raw"}) + SHARED_METRICS = [{"existing": "metric"}] + + class Check(OpenMetricsBaseCheckV2): + def get_default_config(self): + return {"metrics": SHARED_METRICS} + + check = make_check(cls=Check) + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + check.get_config_with_defaults({'openmetrics_endpoint': 'http://test:9090/metrics'}) + assert SHARED_METRICS == [{"existing": "metric"}] + + +def test_load_file_based_metrics_does_not_mutate_cached_default_dict(make_check: CheckFactory, tmp_path: Path): + """A subclass that caches its defaults dict at module level must not see file metrics accumulate.""" + write_yaml(tmp_path, "metrics.yml", {"raw": "dd.raw"}) + CACHED_DEFAULTS = {"metrics": [{"existing": "metric"}]} + + class Check(OpenMetricsBaseCheckV2): + def get_default_config(self): + return CACHED_DEFAULTS + + instance = {'openmetrics_endpoint': 'http://test:9090/metrics'} + first = make_check(cls=Check) + second = make_check(cls=Check) + with patch.object(Check, '_get_package_dir', return_value=tmp_path): + config_first = first.get_config_with_defaults(instance) + config_second = second.get_config_with_defaults(instance) + assert CACHED_DEFAULTS == {"metrics": [{"existing": "metric"}]} + assert config_first['metrics'] == config_second['metrics'] + assert len(config_first['metrics']) == 2 + + +# --------------------------------------------------------------------------- +# get_config_with_defaults +# --------------------------------------------------------------------------- + + +def test_get_config_with_defaults_merges_file_metrics(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "metrics.yml", {"raw": "dd_name"}) + check = make_check() + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + config = check.get_config_with_defaults({"openmetrics_endpoint": "http://test:9090/metrics"}) + assert {"raw": "dd_name"} in config["metrics"] + + +def test_get_config_with_defaults_combines_with_existing(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "metrics.yml", {"file_metric": "dd.file_metric"}) + + class Check(OpenMetricsBaseCheckV2): + def get_default_config(self): + return {"metrics": [{"existing": "metric"}], "extra_option": True} + + check = make_check(cls=Check) + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + config = check.get_config_with_defaults({"openmetrics_endpoint": "http://test:9090/metrics"}) + assert {"existing": "metric"} in config["metrics"] + assert {"file_metric": "dd.file_metric"} in config["metrics"] + assert config["extra_option"] is True + + +# --------------------------------------------------------------------------- +# Public re-exports (lazy_loader stub) +# --------------------------------------------------------------------------- + + +def test_public_reexports_resolve(): + """The lazy_loader stub at v2/__init__.pyi must expose the documented public surface.""" + from datadog_checks.base.checks.openmetrics.v2 import ( + AllOf, + AnyOf, + ConfigOptionEquals, + ConfigOptionTruthy, + MetricsMapping, + MetricsPredicate, + OpenMetricsBaseCheckV2, + ) + + assert all( + symbol is not None + for symbol in ( + AllOf, + AnyOf, + ConfigOptionEquals, + ConfigOptionTruthy, + MetricsMapping, + MetricsPredicate, + OpenMetricsBaseCheckV2, + ) + ) From ff7c2db9d77ddd0a21f1346d20f1ed0216f4c6b3 Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Thu, 19 Mar 2026 17:19:55 +0100 Subject: [PATCH 22/44] Creation of Tools framework (#22884) * started the tools implementation: base, cmd, types. also added a read_file tool * Changed tools input_schema and added some tools * Added ToolRegistry * slight corrections in cmd and truncation * Fix blocking subprocess in async run_command and harden BaseTool error handling Co-Authored-By: Claude Sonnet 4.6 * made BaseTool generic on TInput and moved allowed_tool_callers logic to registry * separated ToolProtocol from base and changed __init__ * changed truncation API and allowed specific timeout for each tool * solved some minor bugs * changed protocol and get_input_type and added tests for BaseTool * changed folder structure, nothing new but imports * test registry and truncation * test CmdTool and tools * get_input_type clarification * improved test_base * improved tests: registry and truncation * improved shell tests * few little bugs fixed * fixed grep test * init with nothing inside * delete some trivial tests * Move ToolInputs to pydantic models to easily handle schema generation * Improve tests, fixed grep when exits 1, fixed stderr bug in shell/base.py and fixed ReadFileInput * Shortened test_truncation --------- Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Juanpe Araque --- ddev/src/ddev/ai/__init__.py | 3 + ddev/src/ddev/ai/tools/__init__.py | 3 + ddev/src/ddev/ai/tools/core/__init__.py | 3 + ddev/src/ddev/ai/tools/core/base.py | 109 ++++++++++ ddev/src/ddev/ai/tools/core/protocol.py | 19 ++ ddev/src/ddev/ai/tools/core/registry.py | 32 +++ ddev/src/ddev/ai/tools/core/truncation.py | 93 ++++++++ ddev/src/ddev/ai/tools/core/types.py | 17 ++ ddev/src/ddev/ai/tools/shell/__init__.py | 3 + ddev/src/ddev/ai/tools/shell/base.py | 67 ++++++ ddev/src/ddev/ai/tools/shell/grep.py | 43 ++++ ddev/src/ddev/ai/tools/shell/list_files.py | 32 +++ ddev/src/ddev/ai/tools/shell/mkdir.py | 28 +++ ddev/src/ddev/ai/tools/shell/read_file.py | 41 ++++ ddev/tests/ai/__init__.py | 3 + ddev/tests/ai/tools/__init__.py | 3 + ddev/tests/ai/tools/core/__init__.py | 3 + ddev/tests/ai/tools/core/test_base.py | 227 ++++++++++++++++++++ ddev/tests/ai/tools/core/test_registry.py | 130 +++++++++++ ddev/tests/ai/tools/core/test_truncation.py | 176 +++++++++++++++ ddev/tests/ai/tools/shell/__init__.py | 3 + ddev/tests/ai/tools/shell/test_base.py | 209 ++++++++++++++++++ ddev/tests/ai/tools/shell/test_tools.py | 155 +++++++++++++ 23 files changed, 1402 insertions(+) create mode 100644 ddev/src/ddev/ai/__init__.py create mode 100644 ddev/src/ddev/ai/tools/__init__.py create mode 100644 ddev/src/ddev/ai/tools/core/__init__.py create mode 100644 ddev/src/ddev/ai/tools/core/base.py create mode 100644 ddev/src/ddev/ai/tools/core/protocol.py create mode 100644 ddev/src/ddev/ai/tools/core/registry.py create mode 100644 ddev/src/ddev/ai/tools/core/truncation.py create mode 100644 ddev/src/ddev/ai/tools/core/types.py create mode 100644 ddev/src/ddev/ai/tools/shell/__init__.py create mode 100644 ddev/src/ddev/ai/tools/shell/base.py create mode 100644 ddev/src/ddev/ai/tools/shell/grep.py create mode 100644 ddev/src/ddev/ai/tools/shell/list_files.py create mode 100644 ddev/src/ddev/ai/tools/shell/mkdir.py create mode 100644 ddev/src/ddev/ai/tools/shell/read_file.py create mode 100644 ddev/tests/ai/__init__.py create mode 100644 ddev/tests/ai/tools/__init__.py create mode 100644 ddev/tests/ai/tools/core/__init__.py create mode 100644 ddev/tests/ai/tools/core/test_base.py create mode 100644 ddev/tests/ai/tools/core/test_registry.py create mode 100644 ddev/tests/ai/tools/core/test_truncation.py create mode 100644 ddev/tests/ai/tools/shell/__init__.py create mode 100644 ddev/tests/ai/tools/shell/test_base.py create mode 100644 ddev/tests/ai/tools/shell/test_tools.py diff --git a/ddev/src/ddev/ai/__init__.py b/ddev/src/ddev/ai/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/tools/__init__.py b/ddev/src/ddev/ai/tools/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/tools/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/tools/core/__init__.py b/ddev/src/ddev/ai/tools/core/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/tools/core/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/tools/core/base.py b/ddev/src/ddev/ai/tools/core/base.py new file mode 100644 index 0000000000000..0d53f98e95e49 --- /dev/null +++ b/ddev/src/ddev/ai/tools/core/base.py @@ -0,0 +1,109 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import inspect +import typing +from abc import ABC, abstractmethod +from types import get_original_bases +from typing import Any + +from anthropic.types import ToolParam +from pydantic import BaseModel, ConfigDict + +from .types import ToolResult + + +class BaseToolInput(BaseModel): + model_config = ConfigDict(extra='forbid') + + @classmethod + def to_input_schema(cls) -> dict[str, object]: + schema = cls.model_json_schema() + schema.pop('title', None) + for prop in schema.get('properties', {}).values(): + prop.pop('title', None) + prop.pop('default', None) + if 'anyOf' in prop: + non_null = [t for t in prop['anyOf'] if t != {'type': 'null'}] + if len(non_null) == 1: + prop.update(non_null[0]) + del prop['anyOf'] + return schema + + +def _get_input_type(cls: type) -> type[BaseToolInput]: + """Extract the TInput type from a BaseTool subclass""" + if resolved := _resolve_base_tool_arg(cls, {}): + return resolved + raise TypeError(f"{cls.__name__} must be parameterized with an input type: class MyTool(BaseTool[MyInput])") + + +def _resolve_base_tool_arg(cls: type, type_map: dict[Any, Any]) -> type[BaseToolInput] | None: + """Resolve the TInput type from a BaseTool subclass, resolving through intermediate generics.""" + for base in get_original_bases(cls): + origin = typing.get_origin(base) or base + args = typing.get_args(base) + + if origin is BaseTool and args: + resolved = type_map.get(args[0], args[0]) + if isinstance(resolved, type): + return resolved + elif isinstance(origin, type) and issubclass(origin, BaseTool) and origin is not BaseTool: + # Call recursively until we find the generic type of the first BaseTool ancestor. + # Example: + # class EchoTool(BaseTool[EchoInput]): + # pass + # class ChildTool[T](EchoTool): + # pass + # class ConcreteChildTool(ChildTool[int]): + # pass + # _get_input_type(ConcreteChildTool) will resolve to EchoInput. + type_params = origin.__type_params__ + new_map = {param: type_map.get(arg, arg) for param, arg in zip(type_params, args, strict=False)} + if resolved := _resolve_base_tool_arg(origin, new_map): + return resolved + + return None + + +class BaseTool[TInput: BaseToolInput](ABC): + @property + @abstractmethod + def name(self) -> str: + """Unique tool name used in API calls.""" + ... + + @property + def description(self) -> str: + return inspect.cleandoc(self.__class__.__doc__) if self.__class__.__doc__ else "" + + @property + def input_schema(self) -> dict[str, object]: + return _get_input_type(type(self)).to_input_schema() + + @property + def definition(self) -> ToolParam: + """Build the Anthropic SDK ToolParam from this tool's metadata.""" + return { + "name": self.name, + "description": self.description, + "input_schema": self.input_schema, + } + + async def run(self, raw: dict[str, object]) -> ToolResult: + """Coerce raw dict to the typed Input class and delegate to __call__.""" + try: + input_cls = _get_input_type(type(self)) + validated: TInput = input_cls.model_validate(raw) # type: ignore[assignment] + except (TypeError, ValueError) as e: + return ToolResult(success=False, error=str(e)) + try: + return await self(validated) + except Exception as e: + return ToolResult(success=False, error=f"{type(e).__name__}: {str(e)}") + + @abstractmethod + async def __call__(self, tool_input: TInput) -> ToolResult: + """Call the tool with a typed input instance.""" + ... diff --git a/ddev/src/ddev/ai/tools/core/protocol.py b/ddev/src/ddev/ai/tools/core/protocol.py new file mode 100644 index 0000000000000..0ef3bc426ee3c --- /dev/null +++ b/ddev/src/ddev/ai/tools/core/protocol.py @@ -0,0 +1,19 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from typing import Protocol + +from anthropic.types import ToolParam + +from .types import ToolResult + + +class ToolProtocol(Protocol): + @property + def name(self) -> str: ... + @property + def description(self) -> str: ... + @property + def definition(self) -> ToolParam: ... + async def run(self, raw: dict[str, object]) -> ToolResult: ... diff --git a/ddev/src/ddev/ai/tools/core/registry.py b/ddev/src/ddev/ai/tools/core/registry.py new file mode 100644 index 0000000000000..29c6f92fb8801 --- /dev/null +++ b/ddev/src/ddev/ai/tools/core/registry.py @@ -0,0 +1,32 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from typing import Final + +from anthropic.types import ToolParam + +from .protocol import ToolProtocol +from .types import ToolResult + +ALLOWED_TOOL_CALLERS: Final = ["code_execution_20260120"] + + +class ToolRegistry: + """Registry holding all available tools.""" + + def __init__(self, tools: list[ToolProtocol]) -> None: + self._tools: dict[str, ToolProtocol] = {tool.name: tool for tool in tools} + + @property + def definitions(self) -> list[ToolParam]: + """Return Anthropic SDK tool definitions for all registered tools. + Each tool definition dict is not mutated, but a new dict is returned with the allowed_callers key added.""" + return [{**tool.definition, "allowed_callers": ALLOWED_TOOL_CALLERS} for tool in self._tools.values()] + + async def run(self, name: str, raw: dict[str, object]) -> ToolResult: + """Execute a tool by name, returning an error result if not found.""" + tool = self._tools.get(name) + if tool is None: + return ToolResult(success=False, error=f"Unknown tool: {name!r}") + return await tool.run(raw) diff --git a/ddev/src/ddev/ai/tools/core/truncation.py b/ddev/src/ddev/ai/tools/core/truncation.py new file mode 100644 index 0000000000000..91b309484a996 --- /dev/null +++ b/ddev/src/ddev/ai/tools/core/truncation.py @@ -0,0 +1,93 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import re +from dataclasses import dataclass +from typing import Final + +MAX_CHARS: Final = 50_000 +HEAD_RATIO: Final = 0.6 + +ERROR_PATTERN = re.compile(r"ERROR|FAILED|Exception|Traceback|fatal|panic", re.IGNORECASE) + + +@dataclass +class TruncationMeta: + total_size: int + shown_size: int + truncated_size: int + hint: str + + +@dataclass +class TruncateResult: + output: str + truncated: bool + meta: TruncationMeta | None + + +def extract_error_lines(lines: list[str]) -> list[tuple[int, str]]: + """Return (index, line) pairs from lines matching error patterns.""" + return [(i, line) for i, line in enumerate(lines) if ERROR_PATTERN.search(line)] + + +def truncate( + content: str, + max_chars: int = MAX_CHARS, + head_ratio: float = HEAD_RATIO, +) -> TruncateResult: + """Truncate content using error-aware head+tail strategy.""" + if len(content) <= max_chars: + return TruncateResult(output=content, truncated=False, meta=None) + + total = len(content) + gap_marker_approx = 50 + content_budget = max(0, max_chars - gap_marker_approx) + head_chars = int(content_budget * head_ratio) + tail_chars = content_budget - head_chars + + head = content[:head_chars] + tail = content[-tail_chars:] if tail_chars > 0 else "" + middle = content[head_chars : total - tail_chars] + + error_lines = extract_error_lines(middle.splitlines()) + + errors_dropped = False + if error_lines: + error_snippet = "\n".join(line for _, line in error_lines) + available = max_chars - len(error_snippet) - gap_marker_approx + if available > 0: + head_share = int(available * head_ratio) + tail_share = available - head_share + head = content[:head_share] + tail = content[-tail_share:] if tail_share > 0 else "" + removed = total - len(head) - len(error_snippet) - len(tail) + gap = f"\n\n[... {removed} characters removed (errors preserved above) ...]\n\n" + result = head + gap + error_snippet + "\n" + tail + else: + errors_dropped = True + removed = total - head_chars - tail_chars + gap = f"\n\n[... {removed} characters removed ...]\n\n" + result = head + gap + tail + else: + removed = total - head_chars - tail_chars + gap = f"\n\n[... {removed} characters removed ...]\n\n" + result = head + gap + tail + + shown = len(result) + if errors_dropped: + hint = ( + f"Output truncated: showing {shown} of {total} characters. " + f"Error lines were detected in the truncated region but could not be preserved " + f"(error snippet exceeded the remaining budget)." + ) + else: + hint = f"Output truncated: showing {shown} of {total} characters." + meta = TruncationMeta( + total_size=total, + shown_size=shown, + truncated_size=total - shown, + hint=hint, + ) + return TruncateResult(output=result, truncated=True, meta=meta) diff --git a/ddev/src/ddev/ai/tools/core/types.py b/ddev/src/ddev/ai/tools/core/types.py new file mode 100644 index 0000000000000..1e5c89c9929c7 --- /dev/null +++ b/ddev/src/ddev/ai/tools/core/types.py @@ -0,0 +1,17 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from pydantic import BaseModel + + +class ToolResult(BaseModel): + """Validated result of a tool call.""" + + success: bool + data: str | None = None + error: str | None = None + truncated: bool = False + total_size: int | None = None + shown_size: int | None = None + hint: str | None = None diff --git a/ddev/src/ddev/ai/tools/shell/__init__.py b/ddev/src/ddev/ai/tools/shell/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/tools/shell/base.py b/ddev/src/ddev/ai/tools/shell/base.py new file mode 100644 index 0000000000000..fc2e269e21f5d --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/base.py @@ -0,0 +1,67 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio +from abc import abstractmethod +from typing import ClassVar + +from ddev.ai.tools.core.base import BaseTool, BaseToolInput +from ddev.ai.tools.core.truncation import TruncateResult, truncate +from ddev.ai.tools.core.types import ToolResult + + +class CmdTool[TInput: BaseToolInput](BaseTool[TInput]): + """Base for tools that execute shell commands.""" + + timeout: ClassVar[int] = 10 + + @abstractmethod + def cmd(self, tool_input: TInput) -> list[str]: + """Builds the shell command from validated tool input.""" + ... + + async def __call__(self, tool_input: TInput) -> ToolResult: + return await run_command(self.cmd(tool_input), timeout=self.timeout) + + +async def run_command(cmd: list[str], timeout: int = 10) -> ToolResult: + try: + proc = await asyncio.create_subprocess_exec( + *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + stdout_bytes, stderr_bytes = await asyncio.wait_for(proc.communicate(), timeout=timeout) + except FileNotFoundError: + return ToolResult(success=False, error=f"Command not found: {cmd[0]!r}") + except asyncio.TimeoutError: + proc.kill() + await proc.communicate() + return ToolResult(success=False, error=f"Command timed out after {timeout}s: {cmd}") + except Exception as e: + return ToolResult(success=False, error=f"{type(e).__name__}: {e}") + + # errors="replace" to keep output readable in case of non-UTF-8 characters + stdout = stdout_bytes.decode("utf-8", errors="replace") + stderr = stderr_bytes.decode("utf-8", errors="replace") + + output = stdout + if proc.returncode != 0 and stderr: + output = (output + "\n" + stderr) if output else stderr + elif not output and stderr: + output = stderr + + if not output.strip(): + return ToolResult(success=proc.returncode == 0, data="(no output)") + + result: TruncateResult = truncate(output) + + if result.truncated and result.meta is not None: + return ToolResult( + success=proc.returncode == 0, + data=result.output, + truncated=True, + total_size=result.meta.total_size, + shown_size=result.meta.shown_size, + hint=result.meta.hint, + ) + + return ToolResult(success=proc.returncode == 0, data=result.output) diff --git a/ddev/src/ddev/ai/tools/shell/grep.py b/ddev/src/ddev/ai/tools/shell/grep.py new file mode 100644 index 0000000000000..15cabe87594ee --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/grep.py @@ -0,0 +1,43 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.core.types import ToolResult + +from .base import CmdTool, run_command + + +class GrepInput(BaseToolInput): + pattern: Annotated[str, Field(description="Regex pattern to search for")] + path: Annotated[str, Field(description="File or directory to search in")] + recursive: Annotated[bool, Field(description="Search recursively in directories (default: true)")] = True + + +class GrepTool(CmdTool[GrepInput]): + """Searches for a regex pattern in files. Returns matching lines with file path and line + numbers. Use to find specific config values, ports, hostnames across files. Supports extended + regex syntax. Output might be truncated for large results.""" + + timeout = 30 + + @property + def name(self) -> str: + return "grep" + + async def __call__(self, tool_input: GrepInput) -> ToolResult: + result = await run_command(self.cmd(tool_input), timeout=self.timeout) + # grep exits 1 when no lines match — not a failure + if not result.success and result.error is None: + return result.model_copy(update={"success": True}) + return result + + def cmd(self, tool_input: GrepInput) -> list[str]: + cmd = ["grep", "-n", "-E"] + if tool_input.recursive: + cmd.append("-r") + cmd += ["--", tool_input.pattern, tool_input.path] + return cmd diff --git a/ddev/src/ddev/ai/tools/shell/list_files.py b/ddev/src/ddev/ai/tools/shell/list_files.py new file mode 100644 index 0000000000000..a54ed6baceb44 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/list_files.py @@ -0,0 +1,32 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput + +from .base import CmdTool + + +class ListFilesInput(BaseToolInput): + path: Annotated[str, Field(description="Path to list files from")] + recursive: Annotated[bool, Field(description="Whether to list recursively (default: false)")] = False + + +class ListFilesTool(CmdTool[ListFilesInput]): + """Lists files and directories at the given path. Use to explore directory structure and find + config files. Non-recursive by default - set recursive=true for a deep listing.""" + + timeout = 30 + + @property + def name(self) -> str: + return "list_files" + + def cmd(self, tool_input: ListFilesInput) -> list[str]: + cmd = ["find", tool_input.path, "-mindepth", "1"] + if not tool_input.recursive: + cmd += ["-maxdepth", "1"] + return cmd diff --git a/ddev/src/ddev/ai/tools/shell/mkdir.py b/ddev/src/ddev/ai/tools/shell/mkdir.py new file mode 100644 index 0000000000000..7bd25733a245d --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/mkdir.py @@ -0,0 +1,28 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput + +from .base import CmdTool + + +class MkdirInput(BaseToolInput): + path: Annotated[str, Field(description="Path of the directory to create")] + + +class MkdirTool(CmdTool[MkdirInput]): + """Creates a directory at the given path, including any missing parent directories. + Use to create directories for config files, logs, source code.""" + + timeout = 5 + + @property + def name(self) -> str: + return "mkdir" + + def cmd(self, tool_input: MkdirInput) -> list[str]: + return ["mkdir", "-p", tool_input.path] diff --git a/ddev/src/ddev/ai/tools/shell/read_file.py b/ddev/src/ddev/ai/tools/shell/read_file.py new file mode 100644 index 0000000000000..22aa19f7c8844 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/read_file.py @@ -0,0 +1,41 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput + +from .base import CmdTool + + +class ReadFileInput(BaseToolInput): + path: Annotated[str, Field(description="Absolute or relative path to the file to read")] + offset: Annotated[ + int, Field(description="Line number to start reading from (0-indexed, default: 0). Must be >= 0.", ge=0) + ] = 0 + limit: Annotated[ + int | None, Field(description="Number of lines to read (default: all remaining lines). Must be >= 1.", ge=1) + ] = None + + +class ReadFileTool(CmdTool[ReadFileInput]): + """Reads contents of a text file from the host filesystem. + Use to inspect config files, logs, source code. Do not use for binary files. + Supports offset/limit for paging through large files.""" + + @property + def name(self) -> str: + return "read_file" + + def cmd(self, tool_input: ReadFileInput) -> list[str]: + path = tool_input.path + offset = tool_input.offset + limit = tool_input.limit + if offset == 0 and limit is None: + return ["cat", path] + start = offset + 1 + if limit is not None: + return ["awk", f"NR>={start} && NR<={start + limit - 1}", path] + return ["awk", f"NR>={start}", path] diff --git a/ddev/tests/ai/__init__.py b/ddev/tests/ai/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/tools/__init__.py b/ddev/tests/ai/tools/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/tools/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/tools/core/__init__.py b/ddev/tests/ai/tools/core/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/tools/core/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/tools/core/test_base.py b/ddev/tests/ai/tools/core/test_base.py new file mode 100644 index 0000000000000..96cd0f8b07d0c --- /dev/null +++ b/ddev/tests/ai/tools/core/test_base.py @@ -0,0 +1,227 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio +from typing import Annotated + +import pytest +from pydantic import Field + +from ddev.ai.tools.core.base import BaseTool, BaseToolInput, _get_input_type +from ddev.ai.tools.core.types import ToolResult + +# --------------------------------------------------------------------------- +# Minimal concrete tools used across tests +# --------------------------------------------------------------------------- + + +class SimpleInput(BaseToolInput): + message: Annotated[str, Field(description="A message to echo")] + + +class FullInput(BaseToolInput): + required_str: Annotated[str, Field(description="A required string")] + optional_int: Annotated[int | None, Field(description="An optional integer")] = None + flag: Annotated[bool, Field(description="A boolean flag")] = False + + +class EchoTool(BaseTool[SimpleInput]): + """Echo the message back.""" + + @property + def name(self) -> str: + return "echo" + + async def __call__(self, tool_input: SimpleInput) -> ToolResult: + return ToolResult(success=True, data=tool_input.message) + + +class FailingTool(BaseTool[SimpleInput]): + """A tool that always raises.""" + + @property + def name(self) -> str: + return "failing" + + async def __call__(self, tool_input: SimpleInput) -> ToolResult: + raise RuntimeError("something went wrong") + + +class FullInputTool(BaseTool[FullInput]): + """Tool using FullInput.""" + + @property + def name(self) -> str: + return "full" + + async def __call__(self, tool_input: FullInput) -> ToolResult: + return ToolResult(success=True) + + +# --------------------------------------------------------------------------- +# BaseToolInput.to_input_schema() +# --------------------------------------------------------------------------- + + +def test_to_input_schema_type_and_required(): + schema = SimpleInput.to_input_schema() + assert schema["type"] == "object" + assert schema["required"] == ["message"] + + +def test_to_input_schema_field_description(): + schema = SimpleInput.to_input_schema() + assert schema["properties"]["message"]["description"] == "A message to echo" + assert schema["properties"]["message"]["type"] == "string" + + +def test_to_input_schema_no_title_keys(): + schema = FullInput.to_input_schema() + assert "title" not in schema + for prop in schema["properties"].values(): + assert "title" not in prop + + +def test_to_input_schema_additional_properties_false(): + assert SimpleInput.to_input_schema().get("additionalProperties") is False + + +def test_to_input_schema_optional_fields_not_required(): + schema = FullInput.to_input_schema() + assert "required_str" in schema["required"] + assert "optional_int" not in schema["required"] + assert "flag" not in schema["required"] + + +def test_to_input_schema_anyof_flattened_for_optional_int(): + schema = FullInput.to_input_schema() + prop = schema["properties"]["optional_int"] + assert "anyOf" not in prop + assert prop["type"] == "integer" + + +def test_to_input_schema_all_optional_no_required_key(): + class AllOptional(BaseToolInput): + x: Annotated[str, Field(description="x")] = "default" + y: Annotated[int, Field(description="y")] = 0 + + schema = AllOptional.to_input_schema() + assert "required" not in schema + + +# --------------------------------------------------------------------------- +# _get_input_type +# --------------------------------------------------------------------------- + + +def test_get_input_type_returns_correct_input_type(): + class ChildTool(EchoTool): + pass + + assert _get_input_type(EchoTool) is SimpleInput + assert _get_input_type(FullInputTool) is FullInput + assert _get_input_type(ChildTool) is SimpleInput + + +def test_get_input_type_unparameterized_subclass(): + # A class that extends BaseTool without a type argument cannot be resolved + class BareSubclass(BaseTool): # type: ignore[type-arg] + @property + def name(self) -> str: + return "bare" + + async def __call__(self, tool_input) -> ToolResult: # type: ignore[override] + return ToolResult(success=True) + + with pytest.raises(TypeError, match="BareSubclass"): + _get_input_type(BareSubclass) + + +def test_resolves_through_intermediate_generic(): + # Simulates the CmdTool[TInput] -> BaseTool[TInput] pattern + class IntermediateTool[T](BaseTool[T]): + @property + def name(self) -> str: + return "intermediate" + + async def __call__(self, tool_input: T) -> ToolResult: # type: ignore[override] + return ToolResult(success=True) + + class ConcreteTool(IntermediateTool[SimpleInput]): + pass + + assert _get_input_type(ConcreteTool) is SimpleInput + + +# --------------------------------------------------------------------------- +# BaseTool +# --------------------------------------------------------------------------- + + +@pytest.fixture +def echo_tool() -> EchoTool: + return EchoTool() + + +@pytest.fixture +def failing_tool() -> FailingTool: + return FailingTool() + + +def test_build_tool(echo_tool: EchoTool): + assert echo_tool.name == "echo" + assert echo_tool.description == "Echo the message back." + assert echo_tool.input_schema == SimpleInput.to_input_schema() + assert echo_tool.definition == { + "name": "echo", + "description": "Echo the message back.", + "input_schema": SimpleInput.to_input_schema(), + } + + +def test_build_tool_no_docstring(): + class NoDocstringTool(BaseTool[SimpleInput]): + @property + def name(self) -> str: + return "nodoc" + + async def __call__(self, tool_input: SimpleInput) -> ToolResult: + return ToolResult(success=True) + + assert NoDocstringTool().description == "" + + +# --- run(): happy path --- + + +def test_run_valid_input_returns_success(echo_tool: EchoTool): + result = asyncio.run(echo_tool.run({"message": "hello"})) + assert result.success is True + assert result.data == "hello" + + +# --- run(): input validation failures --- + + +@pytest.mark.parametrize( + "raw", + [ + {}, + {"message": "hi", "extra": "oops"}, + ], +) +def test_run_invalid_input_returns_failure(echo_tool: EchoTool, raw: dict): + result = asyncio.run(echo_tool.run(raw)) + assert result.success is False + assert result.error is not None + + +# --- run(): __call__ exception handling --- + + +def test_run_captures_exception_from_call(failing_tool: FailingTool): + result = asyncio.run(failing_tool.run({"message": "boom"})) + assert isinstance(result, ToolResult) + assert result.success is False + assert "RuntimeError" in result.error + assert "something went wrong" in result.error diff --git a/ddev/tests/ai/tools/core/test_registry.py b/ddev/tests/ai/tools/core/test_registry.py new file mode 100644 index 0000000000000..fdd42714b6ed4 --- /dev/null +++ b/ddev/tests/ai/tools/core/test_registry.py @@ -0,0 +1,130 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio + +import pytest + +from ddev.ai.tools.core.registry import ALLOWED_TOOL_CALLERS, ToolRegistry +from ddev.ai.tools.core.types import ToolResult + +# --------------------------------------------------------------------------- +# Fake tools — implement ToolProtocol without depending on BaseTool +# --------------------------------------------------------------------------- + + +class FakeTool: + """Minimal ToolProtocol implementation for registry tests.""" + + def __init__(self, name: str, result: ToolResult | None = None) -> None: + self._name = name + self._result = result or ToolResult(success=True, data=f"{name} ok") + self.last_raw: dict[str, object] | None = None + + @property + def name(self) -> str: + return self._name + + @property + def description(self) -> str: + return f"Fake tool {self._name}" + + @property + def definition(self) -> dict: + return {"name": self._name, "description": self.description, "input_schema": {}} + + async def run(self, raw: dict[str, object]) -> ToolResult: + self.last_raw = raw + return self._result + + +# --------------------------------------------------------------------------- +# ToolRegistry.__init__ +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "tools,expected_names", + [ + ([FakeTool("alpha")], {"alpha"}), + ([], set()), + ([FakeTool("a"), FakeTool("b"), FakeTool("c")], {"a", "b", "c"}), + ], +) +def test_registry_registers_tools(tools, expected_names): + registry = ToolRegistry(tools) + assert set(registry._tools.keys()) == expected_names + + +def test_duplicate_name_last_one_wins(): + # This design is intentional to allow for tool overrides. + first = FakeTool("dup") + second = FakeTool("dup") + registry = ToolRegistry([first, second]) + assert registry._tools["dup"] is second + + +# --------------------------------------------------------------------------- +# ToolRegistry.definitions +# --------------------------------------------------------------------------- + + +def test_empty_registry_returns_empty_list(): + assert ToolRegistry([]).definitions == [] + + +def test_tool_registry_definitions_returns_all_tool_definitions(): + registry = ToolRegistry([FakeTool("a"), FakeTool("b")]) + assert len(registry.definitions) == 2 + for defn in registry.definitions: + assert defn["allowed_callers"] == ALLOWED_TOOL_CALLERS + + +def test_definition_contains_tool_name(): + registry = ToolRegistry([FakeTool("mytool")]) + assert registry.definitions[0]["name"] == "mytool" + + +# --------------------------------------------------------------------------- +# ToolRegistry.run +# --------------------------------------------------------------------------- + + +def test_run_dispatches_to_correct_tool(): + tool_a = FakeTool("a", ToolResult(success=True, data="from a")) + tool_b = FakeTool("b", ToolResult(success=True, data="from b")) + registry = ToolRegistry([tool_a, tool_b]) + + result = asyncio.run(registry.run("b", {})) + assert result.success is True + assert result.data == "from b" + + +def test_passes_raw_dict_to_tool_unchanged(): + tool = FakeTool("t") + registry = ToolRegistry([tool]) + raw = {"key": "value", "num": 42} + + asyncio.run(registry.run("t", raw)) + assert tool.last_raw == raw + + +def test_returns_tool_result_on_tool_failure(): + registry = ToolRegistry([FakeTool("t", ToolResult(success=False, error="bad input"))]) + result = asyncio.run(registry.run("t", {})) + assert result.success is False + assert result.error == "bad input" + + +def test_unknown_tool_returns_failure(): + registry = ToolRegistry([FakeTool("known_tool")]) + result = asyncio.run(registry.run("unknown_tool", {})) + assert result.success is False + assert "Unknown tool: 'unknown_tool'" in result.error + + +def test_empty_registry_always_returns_unknown_error(): + registry = ToolRegistry([]) + result = asyncio.run(registry.run("anything", {})) + assert result.success is False + assert result.error is not None diff --git a/ddev/tests/ai/tools/core/test_truncation.py b/ddev/tests/ai/tools/core/test_truncation.py new file mode 100644 index 0000000000000..b4f1478dd486f --- /dev/null +++ b/ddev/tests/ai/tools/core/test_truncation.py @@ -0,0 +1,176 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ddev.ai.tools.core.truncation import extract_error_lines, truncate + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def make_content(n_chars: int, char: str = "x") -> str: + """Build a string of exactly n_chars characters.""" + return char * n_chars + + +def make_content_with_error(total: int, error_line: str = "ERROR: something failed") -> str: + """Build a string longer than MAX_CHARS with an error line in the middle.""" + half = total // 2 + padding = "x" * 80 + "\n" + before = padding * (half // len(padding) + 1) + after = padding * (half // len(padding) + 1) + return before[:half] + "\n" + error_line + "\n" + after[:half] + + +# --------------------------------------------------------------------------- +# extract_error_lines +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "keyword", + [ + "ERROR", + "FAILED", + "Exception", + "Traceback", + "fatal", + "panic", + "error", + "failed", + "exception", + "traceback", + "FATAL", + "PANIC", + ], +) +def test_detects_each_error_keyword_case_insensitive(keyword: str): + lines = ["normal line", f"this is a {keyword} here", "another normal"] + result = extract_error_lines(lines) + assert len(result) == 1 + + +def test_extract_error_lines_returns_correct_index(): + lines = ["ok", "ok", "ERROR: boom", "ok"] + result = extract_error_lines(lines) + assert result[0][0] == 2 + + +def test_extract_error_lines_returns_multiple_matching_lines(): + lines = ["ERROR: first", "normal", "Traceback: second"] + result = extract_error_lines(lines) + assert len(result) == 2 + + +def test_extract_error_lines_clean_content_returns_empty(): + lines = ["everything", "is", "fine"] + assert extract_error_lines(lines) == [] + + +def test_extract_error_lines_empty_input_returns_empty(): + assert extract_error_lines([]) == [] + + +# --------------------------------------------------------------------------- +# truncate — max_char limit works as expected +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "content,expected_truncated", + [ + ("hello world", False), + (make_content(50), False), + (make_content(50 + 1), True), + ], +) +def test_max_char_limit(content, expected_truncated): + result = truncate(content, max_chars=50) + assert result.truncated is expected_truncated + if not expected_truncated: + assert result.meta is None + + +# --------------------------------------------------------------------------- +# truncate — basic head+tail (no errors) +# --------------------------------------------------------------------------- + + +@pytest.fixture +def content() -> str: + return make_content(500) + + +@pytest.fixture +def max_chars() -> int: + return 200 + + +def test_truncate_basic_head_tail_no_errors(content: str, max_chars: int): + result = truncate(content, max_chars=max_chars) + assert len(result.output) <= max_chars + 150 # gap marker adds ~50 chars + assert result.truncated is True + + assert result.meta is not None + assert result.meta.total_size == len(content) + assert result.meta.shown_size == len(result.output) + assert result.meta.truncated_size == result.meta.total_size - result.meta.shown_size + + assert "[..." in result.output and "characters removed" in result.output + + assert str(result.meta.shown_size) in result.meta.hint + assert str(result.meta.total_size) in result.meta.hint + + +def test_truncate_starts_and_ends_with_content(content: str, max_chars: int): + content = "START" + content + "END" + result = truncate(content, max_chars=max_chars) + assert result.output.startswith("START") + assert result.output.endswith("END") + + +# --------------------------------------------------------------------------- +# truncate — error-aware preservation +# --------------------------------------------------------------------------- + + +@pytest.fixture +def content_with_error() -> str: + padding = "x" * 80 + "\n" + middle_error = "ERROR: critical failure detected\n" + return padding * 5 + middle_error + padding * 5 + + +def test_error_aware_preservation(content_with_error: str, max_chars: int): + result = truncate(content_with_error, max_chars=max_chars) + assert "ERROR: critical failure detected" in result.output + assert "errors preserved" in result.output + assert "could not be preserved" not in result.meta.hint + + +@pytest.mark.parametrize("keyword", ["FAILED", "Exception", "Traceback", "fatal", "panic"]) +def test_each_error_keyword_triggers_preservation(keyword: str): + padding = "y" * 80 + "\n" + content = padding * 5 + f"{keyword}: something bad\n" + padding * 5 + result = truncate(content, max_chars=200) + assert keyword in result.output + + +# --------------------------------------------------------------------------- +# truncate — errors too large to preserve (fallback to plain head+tail) +# --------------------------------------------------------------------------- + + +def test_falls_back_to_plain_truncation_when_error_snippet_exceeds_budget(): + max_chars = 200 + error_lines = "\n".join([f"ERROR: line {i}" for i in range(50)]) # ~700 chars of errors + padding = "x" * 80 + "\n" + content = padding * 5 + error_lines + padding * 5 + + result = truncate(content, max_chars=max_chars) + + assert result.truncated is True + assert "could not be preserved" in result.meta.hint + assert "errors preserved" not in result.output diff --git a/ddev/tests/ai/tools/shell/__init__.py b/ddev/tests/ai/tools/shell/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/tools/shell/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/tools/shell/test_base.py b/ddev/tests/ai/tools/shell/test_base.py new file mode 100644 index 0000000000000..5d7431239a5e7 --- /dev/null +++ b/ddev/tests/ai/tools/shell/test_base.py @@ -0,0 +1,209 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio +from typing import Annotated +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.core.truncation import MAX_CHARS +from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.shell.base import CmdTool, run_command + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def make_proc(returncode: int = 0, stdout: bytes = b"", stderr: bytes = b"") -> MagicMock: + proc = MagicMock() + proc.returncode = returncode + proc.communicate = AsyncMock(return_value=(stdout, stderr)) + proc.kill = MagicMock() + return proc + + +def patch_proc(proc: MagicMock): + return patch("asyncio.create_subprocess_exec", new=AsyncMock(return_value=proc)) + + +async def _raise_timeout(coro, *args, **kwargs): + coro.close() + raise asyncio.TimeoutError() + + +# --------------------------------------------------------------------------- +# Minimal CmdTool subclass for testing +# --------------------------------------------------------------------------- + + +class GreetInput(BaseToolInput): + name: Annotated[str, Field(description="Name to greet")] + + +class GreetTool(CmdTool[GreetInput]): + """Greet someone.""" + + @property + def name(self) -> str: + return "greet" + + def cmd(self, tool_input: GreetInput) -> list[str]: + return ["echo", f"hello {tool_input.name}"] + + +class SlowGreetTool(GreetTool): + timeout = 60 + + +@pytest.fixture +def proc() -> MagicMock: + return make_proc(returncode=0, stdout=b"hello\n") + + +@pytest.fixture +def greet_tool() -> GreetTool: + return GreetTool() + + +@pytest.fixture +def slow_greet_tool() -> SlowGreetTool: + return SlowGreetTool() + + +# --------------------------------------------------------------------------- +# run_command — output and exit code handling +# --------------------------------------------------------------------------- + + +def test_run_command_success(proc): + with patch_proc(proc): + result = asyncio.run(run_command(["echo", "hello"])) + assert result.success is True + assert result.data == "hello\n" + assert result.truncated is False + + +def test_run_command_failure_combines_stdout_and_stderr(): + proc = make_proc(returncode=1, stdout=b"partial\n", stderr=b"error\n") + with patch_proc(proc): + result = asyncio.run(run_command(["cmd"])) + assert result.success is False + assert "partial" in result.data + assert "error" in result.data + + +def test_run_command_failure_stderr_only_when_no_stdout(): + proc = make_proc(returncode=1, stdout=b"", stderr=b"fatal error\n") + with patch_proc(proc): + result = asyncio.run(run_command(["cmd"])) + assert result.success is False and result.data == "fatal error\n" + + +def test_run_command_ignores_stderr_on_zero_exit(): + proc = make_proc(returncode=0, stdout=b"out\n", stderr=b"warning\n") + with patch_proc(proc): + result = asyncio.run(run_command(["cmd"])) + assert result.success is True + assert "warning" not in result.data + + +def test_run_command_stderr_included_when_stdout_empty_on_success(): + proc = make_proc(returncode=0, stdout=b"", stderr=b"info: initialized\n") + with patch_proc(proc): + result = asyncio.run(run_command(["cmd"])) + assert result.success is True + assert result.data == "info: initialized\n" + + +@pytest.mark.parametrize( + "returncode,stdout,stderr", + [ + (0, b"", b""), + (0, b" \n ", b""), + (1, b"", b""), + ], +) +def test_run_command_empty_output(returncode, stdout, stderr): + proc = make_proc(returncode=returncode, stdout=stdout, stderr=stderr) + with patch_proc(proc): + result = asyncio.run(run_command(["cmd"])) + assert result.data == "(no output)" + + +# --------------------------------------------------------------------------- +# run_command — exceptions +# --------------------------------------------------------------------------- + + +def test_run_command_not_found(): + with patch("asyncio.create_subprocess_exec", side_effect=FileNotFoundError()): + result = asyncio.run(run_command(["nonexistent"])) + assert result.success is False + assert "Command not found" in result.error + assert "nonexistent" in result.error + + +def test_run_command_timeout(): + proc = make_proc() + with patch_proc(proc): + with patch("asyncio.wait_for", new=_raise_timeout): + result = asyncio.run(run_command(["sleep", "100"], timeout=5)) + assert result.success is False + assert "5s" in result.error + proc.kill.assert_called_once() + + +def test_run_command_unexpected_exception(): + with patch("asyncio.create_subprocess_exec", side_effect=OSError("permission denied")): + result = asyncio.run(run_command(["cmd"])) + assert result.success is False + assert "OSError" in result.error + assert "permission denied" in result.error + + +# --------------------------------------------------------------------------- +# run_command — truncation +# --------------------------------------------------------------------------- + + +def test_run_command_truncation(): + large = ("x" * 80 + "\n") * 700 + proc = make_proc(stdout=large.encode()) + with patch_proc(proc): + result = asyncio.run(run_command(["cmd"])) + assert result.truncated is True + assert result.total_size == len(large) + assert result.shown_size == len(result.data) + assert result.hint is not None + + +def test_run_command_no_truncation_at_limit(): + proc = make_proc(stdout=("x" * MAX_CHARS).encode()) + with patch_proc(proc): + result = asyncio.run(run_command(["cmd"])) + assert result.truncated is False + assert result.total_size is None + assert result.hint is None + + +# --------------------------------------------------------------------------- +# CmdTool +# --------------------------------------------------------------------------- + + +def test_cmd_tool_timeouts(greet_tool: GreetTool, slow_greet_tool: SlowGreetTool): + assert GreetTool.timeout == 10 # default timeout + assert SlowGreetTool.timeout == 60 # custom timeout + + +def test_cmd_tool_dispatches_with_correct_timeout(greet_tool: GreetTool, slow_greet_tool: SlowGreetTool): + for tool, expected_timeout in [(greet_tool, 10), (slow_greet_tool, 60)]: + with patch( + "ddev.ai.tools.shell.base.run_command", new=AsyncMock(return_value=ToolResult(success=True)) + ) as mock_run: + asyncio.run(tool.run({"name": "world"})) + mock_run.assert_called_once_with(["echo", "hello world"], timeout=expected_timeout) diff --git a/ddev/tests/ai/tools/shell/test_tools.py b/ddev/tests/ai/tools/shell/test_tools.py new file mode 100644 index 0000000000000..573946e34bae3 --- /dev/null +++ b/ddev/tests/ai/tools/shell/test_tools.py @@ -0,0 +1,155 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio +from unittest.mock import AsyncMock, patch + +import pytest + +from ddev.ai.tools.shell.grep import GrepInput, GrepTool +from ddev.ai.tools.shell.list_files import ListFilesInput, ListFilesTool +from ddev.ai.tools.shell.mkdir import MkdirInput, MkdirTool +from ddev.ai.tools.shell.read_file import ReadFileInput, ReadFileTool + +# --------------------------------------------------------------------------- +# Tool metadata +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "tool_cls,expected_name,expected_timeout", + [ + (GrepTool, "grep", 30), + (ListFilesTool, "list_files", 30), + (MkdirTool, "mkdir", 5), + (ReadFileTool, "read_file", 10), + ], +) +def test_tool_meta(tool_cls, expected_name, expected_timeout): + assert tool_cls().name == expected_name + assert tool_cls.timeout == expected_timeout + + +# --------------------------------------------------------------------------- +# GrepTool +# --------------------------------------------------------------------------- + + +@pytest.fixture +def grep_tool() -> GrepTool: + return GrepTool() + + +def test_grep_cmd_full_command(grep_tool: GrepTool): + assert grep_tool.cmd(GrepInput(pattern="ERROR", path="/var/log", recursive=True)) == [ + "grep", + "-n", + "-E", + "-r", + "--", + "ERROR", + "/var/log", + ] + assert grep_tool.cmd(GrepInput(pattern="ERROR", path="/var/log", recursive=False)) == [ + "grep", + "-n", + "-E", + "--", + "ERROR", + "/var/log", + ] + + +def test_grep_cmd_pattern_and_path_placement(grep_tool: GrepTool): + # pattern is always second-to-last, path is always last + pattern = r"^\d+\.\d+\.\d+" + cmd = grep_tool.cmd(GrepInput(pattern=pattern, path="/my dir/sub dir")) + assert cmd[-2] == pattern + assert cmd[-1] == "/my dir/sub dir" + + +def test_grep_no_matches_returns_success(grep_tool: GrepTool): + from ddev.ai.tools.core.types import ToolResult + + no_match_result = ToolResult(success=False, data="(no output)", error=None) + with patch("ddev.ai.tools.shell.grep.run_command", new=AsyncMock(return_value=no_match_result)): + result = asyncio.run(grep_tool(GrepInput(pattern="nomatch", path="/tmp"))) + assert result.success is True + assert result.data == "(no output)" + + +# --------------------------------------------------------------------------- +# ListFilesTool +# --------------------------------------------------------------------------- + + +@pytest.fixture +def list_files_tool() -> ListFilesTool: + return ListFilesTool() + + +def test_list_files_cmd_non_recursive(list_files_tool: ListFilesTool): + # non-recursive by default — maxdepth 1 present, mindepth before maxdepth + cmd_default = list_files_tool.cmd(ListFilesInput(path="/tmp")) + cmd_explicit = list_files_tool.cmd(ListFilesInput(path="/var", recursive=False)) + + assert cmd_default == ["find", "/tmp", "-mindepth", "1", "-maxdepth", "1"] + assert cmd_explicit == ["find", "/var", "-mindepth", "1", "-maxdepth", "1"] + + +def test_list_files_cmd_recursive(list_files_tool: ListFilesTool): + cmd = list_files_tool.cmd(ListFilesInput(path="/var", recursive=True)) + assert cmd == ["find", "/var", "-mindepth", "1"] + + +# --------------------------------------------------------------------------- +# MkdirTool +# --------------------------------------------------------------------------- + + +@pytest.fixture +def mkdir_tool() -> MkdirTool: + return MkdirTool() + + +def test_mkdir_cmd(mkdir_tool: MkdirTool): + assert mkdir_tool.cmd(MkdirInput(path="/a/b/c")) == ["mkdir", "-p", "/a/b/c"] + assert mkdir_tool.cmd(MkdirInput(path="/my dir/sub dir")) == ["mkdir", "-p", "/my dir/sub dir"] + + +# --------------------------------------------------------------------------- +# ReadFileTool +# --------------------------------------------------------------------------- + + +@pytest.fixture +def read_file_tool() -> ReadFileTool: + return ReadFileTool() + + +@pytest.fixture +def path() -> str: + return "/etc/config.conf" + + +@pytest.mark.parametrize( + "offset,limit,expected_cmd", + [ + (None, None, ["cat", "/etc/config.conf"]), + (0, None, ["cat", "/etc/config.conf"]), + (1, None, ["awk", "NR>=2", "/etc/config.conf"]), + (5, None, ["awk", "NR>=6", "/etc/config.conf"]), + (0, 10, ["awk", "NR>=1 && NR<=10", "/etc/config.conf"]), + (0, 1, ["awk", "NR>=1 && NR<=1", "/etc/config.conf"]), + (1, 10, ["awk", "NR>=2 && NR<=11", "/etc/config.conf"]), + (5, 1, ["awk", "NR>=6 && NR<=6", "/etc/config.conf"]), + (10, 5, ["awk", "NR>=11 && NR<=15", "/etc/config.conf"]), + (3, 5, ["awk", "NR>=4 && NR<=8", "/etc/config.conf"]), + ], +) +def test_read_file_cmd(read_file_tool: ReadFileTool, offset, limit, expected_cmd): + if offset is None and limit is None: + inp = ReadFileInput(path="/etc/config.conf") + else: + inp = ReadFileInput(path="/etc/config.conf", offset=offset or 0, limit=limit) + assert read_file_tool.cmd(inp) == expected_cmd From 6a75869e3e5b179357d63ec8dcb90c19034ebc1d Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Mon, 23 Mar 2026 17:25:59 +0100 Subject: [PATCH 23/44] Tool framework extension (#22995) * added create, edit and append tools * separated ToolProtocol from base and changed __init__ * modified new tools to by typed as base is * place new tools into fs/ and http/ and fix imports * new hash check system for file-editing tools * test file registry created * add locks and tools tests * change read_file to accept new BaseInput * change read_file to accept new BaseInput * Migrate fs/ tools to BaseInput new model. Improve tests as well * Improved fs/tests, simplified them * Add http_get tests * Add line numbers to read_file tool * Read_file starts from line 0 * Write file tools can only write files registered in file registry * Minor updates and bugs fixed * Create unique test files for each tool and create handler for truncation * Fix suggestions and nits * Add logger in core/base and few more fixes in file_registry and tests * Delete _refresh_if_known so agent can edit files that were not created by itself --- ddev/src/ddev/ai/tools/core/base.py | 7 +- ddev/src/ddev/ai/tools/core/truncation.py | 16 +++ ddev/src/ddev/ai/tools/fs/__init__.py | 3 + ddev/src/ddev/ai/tools/fs/append_file.py | 45 +++++++ ddev/src/ddev/ai/tools/fs/base.py | 31 +++++ ddev/src/ddev/ai/tools/fs/create_file.py | 44 +++++++ ddev/src/ddev/ai/tools/fs/edit_file.py | 67 ++++++++++ ddev/src/ddev/ai/tools/fs/file_registry.py | 36 +++++ ddev/src/ddev/ai/tools/fs/read_file.py | 53 ++++++++ ddev/src/ddev/ai/tools/http/__init__.py | 3 + ddev/src/ddev/ai/tools/http/http_get.py | 49 +++++++ ddev/src/ddev/ai/tools/shell/base.py | 17 +-- ddev/src/ddev/ai/tools/shell/read_file.py | 41 ------ ddev/tests/ai/tools/fs/__init__.py | 3 + ddev/tests/ai/tools/fs/conftest.py | 45 +++++++ ddev/tests/ai/tools/fs/test_append_file.py | 91 +++++++++++++ ddev/tests/ai/tools/fs/test_base.py | 121 +++++++++++++++++ ddev/tests/ai/tools/fs/test_create_file.py | 85 ++++++++++++ ddev/tests/ai/tools/fs/test_edit_file.py | 110 ++++++++++++++++ ddev/tests/ai/tools/fs/test_file_registry.py | 109 ++++++++++++++++ ddev/tests/ai/tools/fs/test_read_file.py | 105 +++++++++++++++ ddev/tests/ai/tools/fs/test_workflow.py | 63 +++++++++ ddev/tests/ai/tools/http/__init__.py | 3 + ddev/tests/ai/tools/http/test_http_get.py | 130 +++++++++++++++++++ ddev/tests/ai/tools/shell/test_tools.py | 40 ------ 25 files changed, 1221 insertions(+), 96 deletions(-) create mode 100644 ddev/src/ddev/ai/tools/fs/__init__.py create mode 100644 ddev/src/ddev/ai/tools/fs/append_file.py create mode 100644 ddev/src/ddev/ai/tools/fs/base.py create mode 100644 ddev/src/ddev/ai/tools/fs/create_file.py create mode 100644 ddev/src/ddev/ai/tools/fs/edit_file.py create mode 100644 ddev/src/ddev/ai/tools/fs/file_registry.py create mode 100644 ddev/src/ddev/ai/tools/fs/read_file.py create mode 100644 ddev/src/ddev/ai/tools/http/__init__.py create mode 100644 ddev/src/ddev/ai/tools/http/http_get.py delete mode 100644 ddev/src/ddev/ai/tools/shell/read_file.py create mode 100644 ddev/tests/ai/tools/fs/__init__.py create mode 100644 ddev/tests/ai/tools/fs/conftest.py create mode 100644 ddev/tests/ai/tools/fs/test_append_file.py create mode 100644 ddev/tests/ai/tools/fs/test_base.py create mode 100644 ddev/tests/ai/tools/fs/test_create_file.py create mode 100644 ddev/tests/ai/tools/fs/test_edit_file.py create mode 100644 ddev/tests/ai/tools/fs/test_file_registry.py create mode 100644 ddev/tests/ai/tools/fs/test_read_file.py create mode 100644 ddev/tests/ai/tools/fs/test_workflow.py create mode 100644 ddev/tests/ai/tools/http/__init__.py create mode 100644 ddev/tests/ai/tools/http/test_http_get.py diff --git a/ddev/src/ddev/ai/tools/core/base.py b/ddev/src/ddev/ai/tools/core/base.py index 0d53f98e95e49..3df60077712db 100644 --- a/ddev/src/ddev/ai/tools/core/base.py +++ b/ddev/src/ddev/ai/tools/core/base.py @@ -3,6 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import inspect +import logging import typing from abc import ABC, abstractmethod from types import get_original_bases @@ -13,6 +14,8 @@ from .types import ToolResult +logger = logging.getLogger(__name__) + class BaseToolInput(BaseModel): model_config = ConfigDict(extra='forbid') @@ -101,7 +104,9 @@ async def run(self, raw: dict[str, object]) -> ToolResult: try: return await self(validated) except Exception as e: - return ToolResult(success=False, error=f"{type(e).__name__}: {str(e)}") + msg = str(e) or repr(e) + logger.exception("Unhandled exception in tool %s: %s", type(self).__name__, msg) + return ToolResult(success=False, error=f"{type(e).__name__}: {msg}") @abstractmethod async def __call__(self, tool_input: TInput) -> ToolResult: diff --git a/ddev/src/ddev/ai/tools/core/truncation.py b/ddev/src/ddev/ai/tools/core/truncation.py index 91b309484a996..6888929a173fa 100644 --- a/ddev/src/ddev/ai/tools/core/truncation.py +++ b/ddev/src/ddev/ai/tools/core/truncation.py @@ -6,6 +6,8 @@ from dataclasses import dataclass from typing import Final +from ddev.ai.tools.core.types import ToolResult + MAX_CHARS: Final = 50_000 HEAD_RATIO: Final = 0.6 @@ -91,3 +93,17 @@ def truncate( hint=hint, ) return TruncateResult(output=result, truncated=True, meta=meta) + + +def make_tool_result(success: bool, data: str, result: TruncateResult) -> ToolResult: + """Build a ToolResult, forwarding truncation metadata when present.""" + if result.truncated and result.meta is not None: + return ToolResult( + success=success, + data=data, + truncated=True, + total_size=result.meta.total_size, + shown_size=result.meta.shown_size, + hint=result.meta.hint, + ) + return ToolResult(success=success, data=data) diff --git a/ddev/src/ddev/ai/tools/fs/__init__.py b/ddev/src/ddev/ai/tools/fs/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/tools/fs/append_file.py b/ddev/src/ddev/ai/tools/fs/append_file.py new file mode 100644 index 0000000000000..c3908059af46f --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/append_file.py @@ -0,0 +1,45 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from pathlib import Path +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.core.types import ToolResult + +from .base import FileRegistryTool + + +class AppendFileInput(BaseToolInput): + path: Annotated[str, Field(description="Path of the file to append to")] + content: Annotated[str, Field(description="Content to append to the file")] + + +class AppendFileTool(FileRegistryTool[AppendFileInput]): + """Appends content to the end of an existing file. + Fails if the file was modified since the last read.""" + + @property + def name(self) -> str: + return "append_file" + + async def __call__(self, tool_input: AppendFileInput) -> ToolResult: + path = Path(tool_input.path).resolve() + + async with self._registry.lock_for(str(path)): + current_content, fail = self._read_verified(str(path)) + if fail: + return fail + + content_to_append = tool_input.content.replace("\r\n", "\n") + separator = "" if not current_content or current_content.endswith("\n") else "\n" + new_content = current_content + separator + content_to_append + + try: + path.write_text(new_content, encoding="utf-8") + except OSError as e: + return ToolResult(success=False, error=str(e)) + self._register(str(path), new_content) + return ToolResult(success=True, data=f"Content appended to: {path}") diff --git a/ddev/src/ddev/ai/tools/fs/base.py b/ddev/src/ddev/ai/tools/fs/base.py new file mode 100644 index 0000000000000..b952b03363f63 --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/base.py @@ -0,0 +1,31 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from pathlib import Path + +from ddev.ai.tools.core.base import BaseTool, BaseToolInput +from ddev.ai.tools.core.types import ToolResult + +from .file_registry import FileRegistry + + +class FileRegistryTool[TInput: BaseToolInput](BaseTool[TInput]): + """Abstract base for file system tools with hash-based consistency checks.""" + + def __init__(self, file_registry: FileRegistry) -> None: + self._registry = file_registry + + def _register(self, path: str, content: str) -> None: + self._registry.record(path, content) + + def _read_verified(self, path: str) -> tuple[str, ToolResult | None]: + """Read file content and verify it matches the last recorded hash.""" + if not self._registry.is_known(path): + return "", ToolResult(success=False, error=f"Not authorized to modify '{path}'.") + try: + content = Path(path).read_text(encoding="utf-8") + except OSError as e: + return "", ToolResult(success=False, error=str(e)) + if not self._registry.verify(path, content): + return "", ToolResult(success=False, error=f"File '{path}' has changed since last read. Re-read and retry.") + return content, None diff --git a/ddev/src/ddev/ai/tools/fs/create_file.py b/ddev/src/ddev/ai/tools/fs/create_file.py new file mode 100644 index 0000000000000..aa3dff51428ea --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/create_file.py @@ -0,0 +1,44 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from pathlib import Path +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.core.types import ToolResult + +from .base import FileRegistryTool + + +class CreateFileInput(BaseToolInput): + path: Annotated[str, Field(description="Path of the file to create")] + content: Annotated[str, Field(description="Content of the file to create")] = "" + + +class CreateFileTool(FileRegistryTool[CreateFileInput]): + """Creates a new file and writes content into it (default: empty content). + Parent directories are created automatically if they do not exist (no need to call mkdir first). + Registers the file in the file registry. + Fails if the file already exists. + Use edit_file to modify existing files.""" + + @property + def name(self) -> str: + return "create_file" + + async def __call__(self, tool_input: CreateFileInput) -> ToolResult: + path = Path(tool_input.path).resolve() + + async with self._registry.lock_for(str(path)): + if path.exists(): + return ToolResult(success=False, error=f"File already exists: {path}") + + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(tool_input.content, encoding="utf-8") + except OSError as e: + return ToolResult(success=False, error=str(e)) + self._register(str(path), tool_input.content) + return ToolResult(success=True, data=f"File created: {path}") diff --git a/ddev/src/ddev/ai/tools/fs/edit_file.py b/ddev/src/ddev/ai/tools/fs/edit_file.py new file mode 100644 index 0000000000000..7e4eafde0ce62 --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/edit_file.py @@ -0,0 +1,67 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from pathlib import Path +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.core.types import ToolResult + +from .base import FileRegistryTool + + +class EditFileInput(BaseToolInput): + path: Annotated[str, Field(description="Path of the file to edit")] + old_string: Annotated[ + str, + Field( + description=( + "Exact non-empty text to replace. Must appear exactly once in the file " + "(hint: include surrounding context if needed)." + ), + min_length=1, + ), + ] + new_string: Annotated[str, Field(description="Text to replace old_string with")] + + +class EditFileTool(FileRegistryTool[EditFileInput]): + """Edits a file by replacing an exact string with a new one. + Fails if the file was modified since the last read. + old_string must appear exactly once in the file — if it appears multiple times, the call fails.""" + + @property + def name(self) -> str: + return "edit_file" + + async def __call__(self, tool_input: EditFileInput) -> ToolResult: + path = Path(tool_input.path).resolve() + + async with self._registry.lock_for(str(path)): + content, fail = self._read_verified(str(path)) + if fail: + return fail + + # Normalize line endings to avoid issues with different OSs + old_string = tool_input.old_string.replace("\r\n", "\n") + new_string = tool_input.new_string.replace("\r\n", "\n") + + count = content.count(old_string) + if count == 0: + return ToolResult(success=False, error="old_string not found in file") + if count > 1: + return ToolResult( + success=False, + error=f"old_string appears {count} times in the file", + hint="Include more surrounding context to make it unique", + ) + + new_content = content.replace(old_string, new_string, 1) + try: + path.write_text(new_content, encoding="utf-8") + except OSError as e: + return ToolResult(success=False, error=str(e)) + self._register(str(path), new_content) + return ToolResult(success=True, data=f"File edited: {path}") diff --git a/ddev/src/ddev/ai/tools/fs/file_registry.py b/ddev/src/ddev/ai/tools/fs/file_registry.py new file mode 100644 index 0000000000000..5bf64221100ed --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/file_registry.py @@ -0,0 +1,36 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio +import hashlib +from pathlib import Path + + +class FileRegistry: + """Tracks files created by the agent and their last-seen content hash.""" + + def __init__(self) -> None: + self._hashes: dict[str, str] = {} + self._locks: dict[str, asyncio.Lock] = {} + + def _normalize(self, path: str) -> str: + return Path(path).resolve().as_posix() + + def _hash(self, content: str) -> str: + return hashlib.sha256(content.encode()).hexdigest() + + def record(self, path: str, content: str) -> None: + self._hashes[self._normalize(path)] = self._hash(content) + + def is_known(self, path: str) -> bool: + return self._normalize(path) in self._hashes + + def lock_for(self, path: str) -> asyncio.Lock: + # Safe under single-threaded asyncio; asyncio.Lock is not thread-safe + return self._locks.setdefault(self._normalize(path), asyncio.Lock()) + + def verify(self, path: str, content: str) -> bool: + """Check whether content matches what was last recorded for path.""" + normalized = self._normalize(path) + stored = self._hashes.get(normalized) + return stored is not None and self._hash(content) == stored diff --git a/ddev/src/ddev/ai/tools/fs/read_file.py b/ddev/src/ddev/ai/tools/fs/read_file.py new file mode 100644 index 0000000000000..18367e1b0a6ac --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/read_file.py @@ -0,0 +1,53 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from pathlib import Path +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.core.truncation import make_tool_result, truncate +from ddev.ai.tools.core.types import ToolResult + +from .base import FileRegistryTool + + +class ReadFileInput(BaseToolInput): + path: Annotated[str, Field(description="Absolute or relative path to the file to read")] + offset: Annotated[ + int, Field(description="Line number to start reading from (0-indexed, default: 0). Must be >= 0.", ge=0) + ] = 0 + limit: Annotated[ + int | None, Field(description="Number of lines to read (default: all remaining lines). Must be >= 1.", ge=1) + ] = None + + +class ReadFileTool(FileRegistryTool[ReadFileInput]): + """Reads contents of a text file from the host filesystem. + Use to inspect config files, logs, source code. Do not use for binary files. + The output is a numbered list of lines starting from 0. + Supports offset/limit for paging through large files. + File does not need to be registered in the file registry. + Note: data="" is a valid result meaning no lines in range.""" + + @property + def name(self) -> str: + return "read_file" + + async def __call__(self, tool_input: ReadFileInput) -> ToolResult: + try: + content = Path(tool_input.path).resolve().read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as e: + return ToolResult(success=False, error=f"{tool_input.path}: {e}") + + self._register(tool_input.path, content) + + offset = tool_input.offset + limit = tool_input.limit + lines = content.splitlines(keepends=True) + slice_ = lines[offset : offset + limit] if limit is not None else lines[offset:] + width = len(str(offset + len(slice_))) + numbered = "".join(f"{offset + i:{width}}: {line}" for i, line in enumerate(slice_)) + truncate_result = truncate(numbered) + return make_tool_result(success=True, data=truncate_result.output, result=truncate_result) diff --git a/ddev/src/ddev/ai/tools/http/__init__.py b/ddev/src/ddev/ai/tools/http/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/tools/http/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/tools/http/http_get.py b/ddev/src/ddev/ai/tools/http/http_get.py new file mode 100644 index 0000000000000..a257763ad6c59 --- /dev/null +++ b/ddev/src/ddev/ai/tools/http/http_get.py @@ -0,0 +1,49 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +import httpx +from pydantic import Field + +from ddev.ai.tools.core.base import BaseTool, BaseToolInput +from ddev.ai.tools.core.truncation import make_tool_result, truncate +from ddev.ai.tools.core.types import ToolResult + + +class HttpGetInput(BaseToolInput): + url: Annotated[str, Field(description="Full URL to probe (must start with http:// or https://)")] + timeout: Annotated[float, Field(description="Request timeout in seconds (default: 10)", gt=0)] = 10.0 + + +class HttpGetTool(BaseTool[HttpGetInput]): + """Performs an HTTP GET request to check if an endpoint is reachable. + Use to validate that a metrics endpoint is accessible and inspect its response. + Returns the HTTP status code and response body (truncated if large).""" + + @property + def name(self) -> str: + return "http_get" + + async def __call__(self, tool_input: HttpGetInput) -> ToolResult: + url: str = tool_input.url + timeout: float = tool_input.timeout + + if not url.startswith(("http://", "https://")): + return ToolResult(success=False, error="URL must start with http:// or https://") + + try: + async with httpx.AsyncClient(timeout=timeout) as client: + response = await client.get(url) + except httpx.TimeoutException: + return ToolResult(success=False, error=f"Request timed out after {timeout}s") + except httpx.RequestError as e: + return ToolResult(success=False, error=f"Request failed for {url}: {e}") + + body = response.text + result = truncate(body) + + status_line = f"Status: {response.status_code}" + output = f"{status_line}\n\n{result.output}" + + return make_tool_result(success=True, data=output, result=result) diff --git a/ddev/src/ddev/ai/tools/shell/base.py b/ddev/src/ddev/ai/tools/shell/base.py index fc2e269e21f5d..099c4f10993eb 100644 --- a/ddev/src/ddev/ai/tools/shell/base.py +++ b/ddev/src/ddev/ai/tools/shell/base.py @@ -6,7 +6,7 @@ from typing import ClassVar from ddev.ai.tools.core.base import BaseTool, BaseToolInput -from ddev.ai.tools.core.truncation import TruncateResult, truncate +from ddev.ai.tools.core.truncation import make_tool_result, truncate from ddev.ai.tools.core.types import ToolResult @@ -52,16 +52,5 @@ async def run_command(cmd: list[str], timeout: int = 10) -> ToolResult: if not output.strip(): return ToolResult(success=proc.returncode == 0, data="(no output)") - result: TruncateResult = truncate(output) - - if result.truncated and result.meta is not None: - return ToolResult( - success=proc.returncode == 0, - data=result.output, - truncated=True, - total_size=result.meta.total_size, - shown_size=result.meta.shown_size, - hint=result.meta.hint, - ) - - return ToolResult(success=proc.returncode == 0, data=result.output) + result = truncate(output) + return make_tool_result(success=proc.returncode == 0, data=result.output, result=result) diff --git a/ddev/src/ddev/ai/tools/shell/read_file.py b/ddev/src/ddev/ai/tools/shell/read_file.py deleted file mode 100644 index 22aa19f7c8844..0000000000000 --- a/ddev/src/ddev/ai/tools/shell/read_file.py +++ /dev/null @@ -1,41 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) -from typing import Annotated - -from pydantic import Field - -from ddev.ai.tools.core.base import BaseToolInput - -from .base import CmdTool - - -class ReadFileInput(BaseToolInput): - path: Annotated[str, Field(description="Absolute or relative path to the file to read")] - offset: Annotated[ - int, Field(description="Line number to start reading from (0-indexed, default: 0). Must be >= 0.", ge=0) - ] = 0 - limit: Annotated[ - int | None, Field(description="Number of lines to read (default: all remaining lines). Must be >= 1.", ge=1) - ] = None - - -class ReadFileTool(CmdTool[ReadFileInput]): - """Reads contents of a text file from the host filesystem. - Use to inspect config files, logs, source code. Do not use for binary files. - Supports offset/limit for paging through large files.""" - - @property - def name(self) -> str: - return "read_file" - - def cmd(self, tool_input: ReadFileInput) -> list[str]: - path = tool_input.path - offset = tool_input.offset - limit = tool_input.limit - if offset == 0 and limit is None: - return ["cat", path] - start = offset + 1 - if limit is not None: - return ["awk", f"NR>={start} && NR<={start + limit - 1}", path] - return ["awk", f"NR>={start}", path] diff --git a/ddev/tests/ai/tools/fs/__init__.py b/ddev/tests/ai/tools/fs/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/tools/fs/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/tools/fs/conftest.py b/ddev/tests/ai/tools/fs/conftest.py new file mode 100644 index 0000000000000..8d6677b98c398 --- /dev/null +++ b/ddev/tests/ai/tools/fs/conftest.py @@ -0,0 +1,45 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio + +import pytest + +from ddev.ai.tools.fs.append_file import AppendFileTool +from ddev.ai.tools.fs.create_file import CreateFileTool +from ddev.ai.tools.fs.edit_file import EditFileTool +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.fs.read_file import ReadFileTool + + +@pytest.fixture +def registry() -> FileRegistry: + return FileRegistry() + + +@pytest.fixture +def read_tool(registry: FileRegistry) -> ReadFileTool: + return ReadFileTool(registry) + + +@pytest.fixture +def create_tool(registry: FileRegistry) -> CreateFileTool: + return CreateFileTool(registry) + + +@pytest.fixture +def edit_tool(registry: FileRegistry) -> EditFileTool: + return EditFileTool(registry) + + +@pytest.fixture +def append_tool(registry: FileRegistry) -> AppendFileTool: + return AppendFileTool(registry) + + +@pytest.fixture +def known_file(tmp_path, create_tool: CreateFileTool): + """A temp file registered in the registry via create.""" + f = tmp_path / "file.txt" + asyncio.run(create_tool.run({"path": str(f), "content": "line one\nline two\nline three\n"})) + return f diff --git a/ddev/tests/ai/tools/fs/test_append_file.py b/ddev/tests/ai/tools/fs/test_append_file.py new file mode 100644 index 0000000000000..2b669572d30bb --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_append_file.py @@ -0,0 +1,91 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio +from unittest.mock import patch + +import pytest + +from ddev.ai.tools.fs.append_file import AppendFileTool +from ddev.ai.tools.fs.create_file import CreateFileTool +from ddev.ai.tools.fs.file_registry import FileRegistry + + +def test_tool_name(registry: FileRegistry) -> None: + assert AppendFileTool(registry).name == "append_file" + + +@pytest.mark.parametrize( + "content,expected_in,expected_not_in", + [ + ("line four\n", "line four\n", None), + ("appended", "three\nappended", None), + ("A\r\nB\r\n", "A\nB\n", "\r"), + ], +) +def test_append_file_success(append_tool: AppendFileTool, known_file, content, expected_in, expected_not_in) -> None: + result = asyncio.run(append_tool.run({"path": str(known_file), "content": content})) + + assert result.success is True + text = known_file.read_text(encoding="utf-8") + assert expected_in in text + if expected_not_in is not None: + assert expected_not_in not in text + + +def test_append_file_fails_for_unregistered_file(append_tool: AppendFileTool, tmp_path) -> None: + f = tmp_path / "unread.txt" + f.write_text("content", encoding="utf-8") + + result = asyncio.run(append_tool.run({"path": str(f), "content": "more"})) + + assert result.success is False + assert "Not authorized" in result.error + + +@pytest.mark.parametrize( + "initial,appended,expected", + [ + ("no newline", "appended", "no newline\nappended"), + ("", "first line", "first line"), + ], +) +def test_append_file_separator( + append_tool: AppendFileTool, create_tool: CreateFileTool, tmp_path, initial, appended, expected +) -> None: + f = tmp_path / "file.txt" + asyncio.run(create_tool.run({"path": str(f), "content": initial})) + + result = asyncio.run(append_tool.run({"path": str(f), "content": appended})) + + assert result.success is True + assert f.read_text(encoding="utf-8") == expected + + +def test_append_file_fails_if_file_changed_externally(append_tool: AppendFileTool, known_file) -> None: + known_file.write_text("externally modified\n", encoding="utf-8") + + result = asyncio.run(append_tool.run({"path": str(known_file), "content": "more"})) + + assert result.success is False + assert "Re-read and retry" in result.error + + +def test_append_file_updates_registry(append_tool: AppendFileTool, registry: FileRegistry, known_file) -> None: + asyncio.run(append_tool.run({"path": str(known_file), "content": "extra\n"})) + + new_content = known_file.read_text(encoding="utf-8") + assert registry.verify(str(known_file), new_content) is True + + +def test_append_file_oserror_on_write(append_tool: AppendFileTool, registry: FileRegistry, known_file) -> None: + original_content = known_file.read_text(encoding="utf-8") + + with patch("pathlib.Path.write_text", side_effect=PermissionError("permission denied")): + result = asyncio.run(append_tool.run({"path": str(known_file), "content": "new line"})) + + assert result.success is False + assert result.error is not None + # File must be untouched and registry must still reflect the original content + assert known_file.read_text(encoding="utf-8") == original_content + assert registry.verify(str(known_file), original_content) is True diff --git a/ddev/tests/ai/tools/fs/test_base.py b/ddev/tests/ai/tools/fs/test_base.py new file mode 100644 index 0000000000000..d19d71092322d --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_base.py @@ -0,0 +1,121 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +import pytest +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.fs.base import FileRegistryTool +from ddev.ai.tools.fs.file_registry import FileRegistry + +# --------------------------------------------------------------------------- +# Minimal concrete subclass for testing +# --------------------------------------------------------------------------- + + +class DummyInput(BaseToolInput): + path: Annotated[str, Field(description="Path")] + + +class DummyTool(FileRegistryTool[DummyInput]): + """Dummy tool to test TextEdit base behavior.""" + + @property + def name(self) -> str: + return "dummy" + + async def __call__(self, tool_input: DummyInput) -> ToolResult: + return ToolResult(success=True) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def registry() -> FileRegistry: + return FileRegistry() + + +@pytest.fixture +def tool(registry: FileRegistry) -> DummyTool: + return DummyTool(registry) + + +# --------------------------------------------------------------------------- +# _read_verified +# --------------------------------------------------------------------------- + + +def test_read_verified_fails_if_not_known(tool: DummyTool, tmp_path) -> None: + path = str(tmp_path / "file.txt") + content, error = tool._read_verified(path) + + assert content == "" + assert error is not None + assert error.success is False + assert "Not authorized" in error.error + + +def test_read_verified_fails_if_file_changed_externally(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: + f = tmp_path / "file.txt" + f.write_text("original", encoding="utf-8") + registry.record(str(f), "original") + + f.write_text("modified", encoding="utf-8") + + content, error = tool._read_verified(str(f)) + + assert content == "" + assert error is not None + assert error.success is False + assert "Re-read and retry" in error.error + + +def test_read_verified_succeeds_if_content_matches(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: + f = tmp_path / "file.txt" + f.write_text("hello", encoding="utf-8") + registry.record(str(f), "hello") + + content, error = tool._read_verified(str(f)) + + assert error is None + assert content == "hello" + + +def test_read_verified_handles_oserror(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "ghost.txt") + # Record the path so it passes the is_known check, but never create the file + registry.record(path, "anything") + + content, error = tool._read_verified(path) + + assert content == "" + assert error is not None + assert error.success is False + + +# --------------------------------------------------------------------------- +# _register +# --------------------------------------------------------------------------- + + +def test_register_registers_path(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + tool._register(path, "written") + + assert registry.is_known(path) is True + assert registry.verify(path, "written") is True + + +def test_register_updates_hash_after_register(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + tool._register(path, "old") + tool._register(path, "new") + + assert registry.verify(path, "new") is True + assert registry.verify(path, "old") is False diff --git a/ddev/tests/ai/tools/fs/test_create_file.py b/ddev/tests/ai/tools/fs/test_create_file.py new file mode 100644 index 0000000000000..2714ef5bb06aa --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_create_file.py @@ -0,0 +1,85 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio +from unittest.mock import patch + +from ddev.ai.tools.fs.create_file import CreateFileTool +from ddev.ai.tools.fs.file_registry import FileRegistry + + +def test_tool_name(registry: FileRegistry) -> None: + assert CreateFileTool(registry).name == "create_file" + + +def test_create_file_success(create_tool: CreateFileTool, tmp_path) -> None: + f = tmp_path / "new.txt" + + result = asyncio.run(create_tool.run({"path": str(f), "content": "hello"})) + + assert result.success is True + assert f.read_text(encoding="utf-8") == "hello" + + +def test_create_file_default_empty_content(create_tool: CreateFileTool, tmp_path) -> None: + f = tmp_path / "empty.txt" + + result = asyncio.run(create_tool.run({"path": str(f)})) + + assert result.success is True + assert f.read_text(encoding="utf-8") == "" + + +def test_create_file_creates_missing_parent_dirs(create_tool: CreateFileTool, tmp_path) -> None: + f = tmp_path / "a" / "b" / "c" / "file.txt" + + result = asyncio.run(create_tool.run({"path": str(f), "content": "nested"})) + + assert result.success is True + assert f.exists() + assert f.read_text(encoding="utf-8") == "nested" + + +def test_create_file_fails_if_file_already_exists( + create_tool: CreateFileTool, registry: FileRegistry, tmp_path +) -> None: + f = tmp_path / "existing.txt" + f.write_text("original", encoding="utf-8") + + result = asyncio.run(create_tool.run({"path": str(f), "content": "new"})) + + assert result.success is False + assert result.error is not None + assert f.read_text(encoding="utf-8") == "original" + assert not registry.is_known(str(f)) + + +def test_create_tool_registers_in_registry(create_tool: CreateFileTool, registry: FileRegistry, tmp_path) -> None: + f = tmp_path / "file.txt" + asyncio.run(create_tool.run({"path": str(f), "content": "hi"})) + + assert registry.is_known(str(f)) is True + assert registry.verify(str(f), "hi") is True + + +def test_create_file_oserror_on_mkdir(create_tool: CreateFileTool, registry: FileRegistry, tmp_path) -> None: + f = tmp_path / "a" / "b" / "new.txt" + + with patch("pathlib.Path.mkdir", side_effect=PermissionError("permission denied")): + result = asyncio.run(create_tool.run({"path": str(f), "content": "hi"})) + + assert result.success is False + assert result.error is not None + assert not f.exists() + assert not registry.is_known(str(f)) + + +def test_create_file_oserror_on_write(create_tool: CreateFileTool, registry: FileRegistry, tmp_path) -> None: + f = tmp_path / "new.txt" + + with patch("pathlib.Path.write_text", side_effect=PermissionError("permission denied")): + result = asyncio.run(create_tool.run({"path": str(f), "content": "hi"})) + + assert result.success is False + assert result.error is not None + assert not registry.is_known(str(f)) diff --git a/ddev/tests/ai/tools/fs/test_edit_file.py b/ddev/tests/ai/tools/fs/test_edit_file.py new file mode 100644 index 0000000000000..cbfd48a78c193 --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_edit_file.py @@ -0,0 +1,110 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio +from unittest.mock import patch + +import pytest + +from ddev.ai.tools.fs.create_file import CreateFileTool +from ddev.ai.tools.fs.edit_file import EditFileTool +from ddev.ai.tools.fs.file_registry import FileRegistry + + +def test_tool_name(registry: FileRegistry) -> None: + assert EditFileTool(registry).name == "edit_file" + + +def test_edit_file_replaces_string(edit_tool: EditFileTool, known_file) -> None: + result = asyncio.run(edit_tool.run({"path": str(known_file), "old_string": "line two", "new_string": "line TWO"})) + + assert result.success is True + content = known_file.read_text(encoding="utf-8") + assert "line TWO" in content + assert "line two" not in content + + +def test_edit_file_deletes_line(edit_tool: EditFileTool, known_file) -> None: + result = asyncio.run(edit_tool.run({"path": str(known_file), "old_string": "line two\n", "new_string": ""})) + + assert result.success is True + assert "line two" not in known_file.read_text(encoding="utf-8") + + +def test_edit_file_fails_for_unregistered_file(edit_tool: EditFileTool, tmp_path) -> None: + f = tmp_path / "unread.txt" + f.write_text("content", encoding="utf-8") + + result = asyncio.run(edit_tool.run({"path": str(f), "old_string": "content", "new_string": "new"})) + + assert result.success is False + assert "Not authorized" in result.error + + +@pytest.mark.parametrize("old_string", ["does not exist", ""]) +def test_edit_file_fails_if_old_string_not_found_or_empty(edit_tool: EditFileTool, known_file, old_string) -> None: + result = asyncio.run(edit_tool.run({"path": str(known_file), "old_string": old_string, "new_string": "x"})) + + assert result.success is False + + +def test_edit_file_fails_if_old_string_ambiguous( + edit_tool: EditFileTool, create_tool: CreateFileTool, tmp_path +) -> None: + f = tmp_path / "dup.txt" + asyncio.run(create_tool.run({"path": str(f), "content": "foo\nfoo\nfoo\n"})) + + result = asyncio.run(edit_tool.run({"path": str(f), "old_string": "foo", "new_string": "bar"})) + + assert result.success is False + assert "3" in result.error + assert result.hint is not None + + +def test_edit_file_fails_if_file_changed_externally(edit_tool: EditFileTool, known_file) -> None: + known_file.write_text("externally modified\n", encoding="utf-8") + + result = asyncio.run(edit_tool.run({"path": str(known_file), "old_string": "line one", "new_string": "x"})) + + assert result.success is False + assert "Re-read and retry" in result.error + + +def test_edit_file_updates_registry(edit_tool: EditFileTool, registry: FileRegistry, known_file) -> None: + asyncio.run(edit_tool.run({"path": str(known_file), "old_string": "line one", "new_string": "LINE ONE"})) + + new_content = known_file.read_text(encoding="utf-8") + assert registry.verify(str(known_file), new_content) is True + assert registry.verify(str(known_file), "line one\nline two\nline three\n") is False + + +@pytest.mark.parametrize( + "file_content,old_string,new_string,expected", + [ + ("line one\nline two\n", "line one\r\nline two", "replaced", "replaced\n"), # CRLF in old_string + ("line one\n", "line one", "A\r\nB", "A\nB\n"), # CRLF in new_string + ], +) +def test_edit_file_normalizes_crlf( + edit_tool: EditFileTool, create_tool: CreateFileTool, tmp_path, file_content, old_string, new_string, expected +) -> None: + f = tmp_path / "file.txt" + asyncio.run(create_tool.run({"path": str(f), "content": file_content})) + + result = asyncio.run(edit_tool.run({"path": str(f), "old_string": old_string, "new_string": new_string})) + + assert result.success is True + assert f.read_text(encoding="utf-8") == expected + + +def test_edit_file_oserror_on_write(edit_tool: EditFileTool, registry: FileRegistry, known_file) -> None: + original_content = known_file.read_text(encoding="utf-8") + + with patch("pathlib.Path.write_text", side_effect=PermissionError("permission denied")): + result = asyncio.run(edit_tool.run({"path": str(known_file), "old_string": "line one", "new_string": "x"})) + + assert result.success is False + assert result.error is not None + # File must be untouched and registry must still reflect the original content + assert known_file.read_text(encoding="utf-8") == original_content + assert registry.verify(str(known_file), original_content) is True diff --git a/ddev/tests/ai/tools/fs/test_file_registry.py b/ddev/tests/ai/tools/fs/test_file_registry.py new file mode 100644 index 0000000000000..17ab79dc73909 --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_file_registry.py @@ -0,0 +1,109 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ddev.ai.tools.fs.file_registry import FileRegistry + + +@pytest.fixture +def registry() -> FileRegistry: + return FileRegistry() + + +# --------------------------------------------------------------------------- +# is_known +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "record,expected", + [ + (False, False), + (True, True), + ], +) +def test_is_known(registry: FileRegistry, tmp_path, record, expected) -> None: + path = str(tmp_path / "file.txt") + if record: + registry.record(path, "hello") + assert registry.is_known(path) is expected + + +def test_is_known_different_path(registry: FileRegistry, tmp_path) -> None: + registry.record(str(tmp_path / "other.txt"), "hello") + assert registry.is_known(str(tmp_path / "file.txt")) is False + + +# --------------------------------------------------------------------------- +# verify +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "recorded_content,verify_content,expected", + [ + ("hello", "hello", True), + ("hello", "world", False), + (None, "any content", False), + ], +) +def test_verify(registry: FileRegistry, tmp_path, recorded_content, verify_content, expected) -> None: + path = str(tmp_path / "file.txt") + if recorded_content is not None: + registry.record(path, recorded_content) + assert registry.verify(path, verify_content) is expected + + +# --------------------------------------------------------------------------- +# record overwrites +# --------------------------------------------------------------------------- + + +def test_record_overwrites_previous_hash(registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + registry.record(path, "old") + registry.record(path, "new") + + assert registry.verify(path, "new") is True + assert registry.verify(path, "old") is False + + +# --------------------------------------------------------------------------- +# path normalization +# --------------------------------------------------------------------------- + + +def test_normalize_relative_and_absolute_are_same_key(registry: FileRegistry, tmp_path, monkeypatch) -> None: + # Make tmp_path the cwd so that a relative path resolves to the same absolute path + monkeypatch.chdir(tmp_path) + + abs_path = str(tmp_path / "file.txt") + rel_path = "file.txt" + + registry.record(abs_path, "hello") + assert registry.is_known(rel_path) is True + assert registry.verify(rel_path, "hello") is True + + +# --------------------------------------------------------------------------- +# lock_for +# --------------------------------------------------------------------------- + + +def test_lock_for_same_path_returns_same_instance(registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + assert registry.lock_for(path) is registry.lock_for(path) + + +def test_lock_for_different_paths_return_different_instances(registry: FileRegistry, tmp_path) -> None: + path_a = str(tmp_path / "a.txt") + path_b = str(tmp_path / "b.txt") + assert registry.lock_for(path_a) is not registry.lock_for(path_b) + + +def test_lock_for_normalizes_path(registry: FileRegistry, tmp_path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + abs_path = str(tmp_path / "file.txt") + rel_path = "file.txt" + assert registry.lock_for(abs_path) is registry.lock_for(rel_path) diff --git a/ddev/tests/ai/tools/fs/test_read_file.py b/ddev/tests/ai/tools/fs/test_read_file.py new file mode 100644 index 0000000000000..f1b8da06d91ed --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_read_file.py @@ -0,0 +1,105 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio +from unittest.mock import patch + +import pytest + +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.fs.read_file import ReadFileTool + + +def test_tool_name(registry: FileRegistry) -> None: + assert ReadFileTool(registry).name == "read_file" + + +def test_read_file_success(read_tool: ReadFileTool, tmp_path) -> None: + f = tmp_path / "config.txt" + f.write_text("hello\nworld\n", encoding="utf-8") + + result = asyncio.run(read_tool.run({"path": str(f)})) + + assert result.success is True + assert result.data == "0: hello\n1: world\n" + + +def test_read_registers_unknown_file(read_tool: ReadFileTool, registry: FileRegistry, tmp_path) -> None: + f = tmp_path / "file.txt" + f.write_text("content", encoding="utf-8") + asyncio.run(read_tool.run({"path": str(f)})) + + assert registry.is_known(str(f)) is True + + +def test_read_file_missing_file(read_tool: ReadFileTool, tmp_path) -> None: + result = asyncio.run(read_tool.run({"path": str(tmp_path / "ghost.txt")})) + + assert result.success is False + assert str(tmp_path / "ghost.txt") in result.error + + +def test_read_file_permission_error(read_tool: ReadFileTool, tmp_path) -> None: + f = tmp_path / "secret.txt" + f.write_text("secret", encoding="utf-8") + + with patch("pathlib.Path.read_text", side_effect=PermissionError("permission denied")): + result = asyncio.run(read_tool.run({"path": str(f)})) + + assert result.success is False + assert result.error is not None + + +def test_read_file_binary_file(read_tool: ReadFileTool, tmp_path) -> None: + f = tmp_path / "binary.bin" + f.write_bytes(b"\xff\xfe\x00binary") + + result = asyncio.run(read_tool.run({"path": str(f)})) + + assert result.success is False + assert result.error is not None + + +@pytest.mark.parametrize( + "offset, limit, expected", + [ + (1, None, "1: b\n2: c\n"), + (0, 2, "0: a\n1: b\n"), + (1, 2, "1: b\n2: c\n"), + (1, 1, "1: b\n"), + (2, 10, "2: c\n"), # limit exceeds remaining lines + (100, None, ""), # offset beyond EOF + ], +) +def test_read_file_with_offset_and_limit(read_tool: ReadFileTool, tmp_path, offset, limit, expected) -> None: + f = tmp_path / "file.txt" + f.write_text("a\nb\nc\n", encoding="utf-8") + + result = asyncio.run(read_tool.run({"path": str(f), "offset": offset, "limit": limit})) + + assert result.success is True + assert result.data == expected + + +def test_read_file_truncated(read_tool: ReadFileTool, tmp_path) -> None: + from ddev.ai.tools.core.truncation import MAX_CHARS + + f = tmp_path / "large.txt" + f.write_text("x" * (MAX_CHARS + 1000), encoding="utf-8") + + result = asyncio.run(read_tool.run({"path": str(f)})) + + assert result.success is True + assert result.truncated is True + assert result.total_size is not None + assert result.hint is not None + + +def test_read_file_no_trailing_newline(read_tool: ReadFileTool, tmp_path) -> None: + f = tmp_path / "file.txt" + f.write_text("no newline at end", encoding="utf-8") + + result = asyncio.run(read_tool.run({"path": str(f)})) + + assert result.success is True + assert result.data == "0: no newline at end" diff --git a/ddev/tests/ai/tools/fs/test_workflow.py b/ddev/tests/ai/tools/fs/test_workflow.py new file mode 100644 index 0000000000000..077f63189bf91 --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_workflow.py @@ -0,0 +1,63 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio + +from ddev.ai.tools.fs.append_file import AppendFileTool +from ddev.ai.tools.fs.create_file import CreateFileTool +from ddev.ai.tools.fs.edit_file import EditFileTool +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.fs.read_file import ReadFileTool + + +def test_workflow_create_read_edit_append( + create_tool: CreateFileTool, + read_tool: ReadFileTool, + edit_tool: EditFileTool, + append_tool: AppendFileTool, + registry: FileRegistry, + tmp_path, +) -> None: + f = tmp_path / "workflow.txt" + + # Step 1: create + r = asyncio.run(create_tool.run({"path": str(f), "content": "version: 1\n"})) + assert r.success is True + + # Step 2: read (registers current content) + r = asyncio.run(read_tool.run({"path": str(f)})) + assert r.success is True + + # Step 3: edit + r = asyncio.run(edit_tool.run({"path": str(f), "old_string": "version: 1", "new_string": "version: 2"})) + assert r.success is True + assert "version: 2" in f.read_text(encoding="utf-8") + + # Step 4: append + r = asyncio.run(append_tool.run({"path": str(f), "content": "# updated\n"})) + assert r.success is True + assert f.read_text(encoding="utf-8").endswith("# updated\n") + + # Registry must reflect the final state + assert registry.verify(str(f), f.read_text(encoding="utf-8")) is True + + +def test_workflow_stale_file( + create_tool: CreateFileTool, + read_tool: ReadFileTool, + edit_tool: EditFileTool, + tmp_path, +) -> None: + f = tmp_path / "shared.txt" + asyncio.run(create_tool.run({"path": str(f), "content": "original\n"})) + f.write_text("updated externally\n", encoding="utf-8") + + result = asyncio.run(edit_tool.run({"path": str(f), "old_string": "original", "new_string": "my edit"})) + assert result.success is False + assert "Re-read and retry" in result.error + + asyncio.run(read_tool.run({"path": str(f)})) + + result = asyncio.run(edit_tool.run({"path": str(f), "old_string": "updated externally", "new_string": "final"})) + assert result.success is True + assert f.read_text(encoding="utf-8") == "final\n" diff --git a/ddev/tests/ai/tools/http/__init__.py b/ddev/tests/ai/tools/http/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/tools/http/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/tools/http/test_http_get.py b/ddev/tests/ai/tools/http/test_http_get.py new file mode 100644 index 0000000000000..d2e8c06220fa1 --- /dev/null +++ b/ddev/tests/ai/tools/http/test_http_get.py @@ -0,0 +1,130 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +from ddev.ai.tools.http.http_get import HttpGetTool + +# --------------------------------------------------------------------------- +# Fixtures / helpers +# --------------------------------------------------------------------------- + + +@pytest.fixture +def http_tool() -> HttpGetTool: + return HttpGetTool() + + +def fake_response(status_code: int, text: str = "") -> MagicMock: + """Fake a HTTP response.""" + resp = MagicMock() + resp.status_code = status_code + resp.text = text + resp.is_success = 200 <= status_code < 300 + return resp + + +def patch_httpx(response=None, *, side_effect=None): + """Patch httpx.AsyncClient so tests never hit the network.""" + mock_get = AsyncMock(return_value=response, side_effect=side_effect) + mock_client = AsyncMock() + mock_client.__aenter__.return_value.get = mock_get + return patch("ddev.ai.tools.http.http_get.httpx.AsyncClient", return_value=mock_client) + + +# --------------------------------------------------------------------------- +# Metadata +# --------------------------------------------------------------------------- + + +def test_tool_meta(http_tool: HttpGetTool) -> None: + assert http_tool.name == "http_get" + + +# --------------------------------------------------------------------------- +# URL validation +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("url", ["ftp://example.com", "example.com", "", "//example.com"]) +def test_invalid_url(http_tool: HttpGetTool, url: str) -> None: + result = asyncio.run(http_tool.run({"url": url})) + + assert result.success is False + assert "http" in result.error and "https" in result.error + + +# --------------------------------------------------------------------------- +# HTTP responses +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "status_code,body", + [ + (200, "# HELP requests_total counter\nrequests_total 42"), + (201, "created"), + (204, ""), + ], +) +def test_request_success(http_tool: HttpGetTool, status_code: int, body: str) -> None: + with patch_httpx(fake_response(status_code, body)): + result = asyncio.run(http_tool.run({"url": "http://localhost:9090/metrics"})) + + assert result.success is True + assert f"Status: {status_code}" in result.data + assert body in result.data + + +@pytest.mark.parametrize("status_code", [400, 404, 500, 503]) +def test_request_non_success_status(http_tool: HttpGetTool, status_code: int) -> None: + with patch_httpx(fake_response(status_code, "error body")): + result = asyncio.run(http_tool.run({"url": "http://localhost:9090/metrics"})) + + assert result.success is True + assert f"Status: {status_code}" in result.data + + +# --------------------------------------------------------------------------- +# Network errors +# --------------------------------------------------------------------------- + + +def test_request_timeout(http_tool: HttpGetTool) -> None: + with patch_httpx(side_effect=httpx.TimeoutException("timed out")): + result = asyncio.run(http_tool.run({"url": "http://localhost:9090/metrics", "timeout": 1.0})) + + assert result.success is False + assert "timed out after 1.0s" in result.error + + +def test_request_error(http_tool: HttpGetTool) -> None: + with patch_httpx(side_effect=httpx.RequestError("connection refused")): + result = asyncio.run(http_tool.run({"url": "http://localhost:9090/metrics"})) + + assert result.success is False + assert "Request failed" in result.error + + +# --------------------------------------------------------------------------- +# Truncation +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("status_code", [200, 500]) +def test_response_truncated(http_tool: HttpGetTool, status_code: int) -> None: + from ddev.ai.tools.core.truncation import MAX_CHARS + + large_body = "x" * (MAX_CHARS + 1000) + with patch_httpx(fake_response(status_code, large_body)): + result = asyncio.run(http_tool.run({"url": "http://localhost:9090/metrics"})) + + assert result.success is True + assert result.truncated is True + assert result.total_size is not None + assert result.hint is not None + assert f"Status: {status_code}" in result.data diff --git a/ddev/tests/ai/tools/shell/test_tools.py b/ddev/tests/ai/tools/shell/test_tools.py index 573946e34bae3..81fcb45d3d3b1 100644 --- a/ddev/tests/ai/tools/shell/test_tools.py +++ b/ddev/tests/ai/tools/shell/test_tools.py @@ -9,7 +9,6 @@ from ddev.ai.tools.shell.grep import GrepInput, GrepTool from ddev.ai.tools.shell.list_files import ListFilesInput, ListFilesTool from ddev.ai.tools.shell.mkdir import MkdirInput, MkdirTool -from ddev.ai.tools.shell.read_file import ReadFileInput, ReadFileTool # --------------------------------------------------------------------------- # Tool metadata @@ -22,7 +21,6 @@ (GrepTool, "grep", 30), (ListFilesTool, "list_files", 30), (MkdirTool, "mkdir", 5), - (ReadFileTool, "read_file", 10), ], ) def test_tool_meta(tool_cls, expected_name, expected_timeout): @@ -115,41 +113,3 @@ def mkdir_tool() -> MkdirTool: def test_mkdir_cmd(mkdir_tool: MkdirTool): assert mkdir_tool.cmd(MkdirInput(path="/a/b/c")) == ["mkdir", "-p", "/a/b/c"] assert mkdir_tool.cmd(MkdirInput(path="/my dir/sub dir")) == ["mkdir", "-p", "/my dir/sub dir"] - - -# --------------------------------------------------------------------------- -# ReadFileTool -# --------------------------------------------------------------------------- - - -@pytest.fixture -def read_file_tool() -> ReadFileTool: - return ReadFileTool() - - -@pytest.fixture -def path() -> str: - return "/etc/config.conf" - - -@pytest.mark.parametrize( - "offset,limit,expected_cmd", - [ - (None, None, ["cat", "/etc/config.conf"]), - (0, None, ["cat", "/etc/config.conf"]), - (1, None, ["awk", "NR>=2", "/etc/config.conf"]), - (5, None, ["awk", "NR>=6", "/etc/config.conf"]), - (0, 10, ["awk", "NR>=1 && NR<=10", "/etc/config.conf"]), - (0, 1, ["awk", "NR>=1 && NR<=1", "/etc/config.conf"]), - (1, 10, ["awk", "NR>=2 && NR<=11", "/etc/config.conf"]), - (5, 1, ["awk", "NR>=6 && NR<=6", "/etc/config.conf"]), - (10, 5, ["awk", "NR>=11 && NR<=15", "/etc/config.conf"]), - (3, 5, ["awk", "NR>=4 && NR<=8", "/etc/config.conf"]), - ], -) -def test_read_file_cmd(read_file_tool: ReadFileTool, offset, limit, expected_cmd): - if offset is None and limit is None: - inp = ReadFileInput(path="/etc/config.conf") - else: - inp = ReadFileInput(path="/etc/config.conf", offset=offset or 0, limit=limit) - assert read_file_tool.cmd(inp) == expected_cmd From 8723f721e611269621db50d382a098458c3b169e Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Fri, 27 Mar 2026 11:24:40 +0100 Subject: [PATCH 24/44] Agent layer (#23053) * Implement AnthropicAgent, defined types and created tests * Fix some bugs and improved tests * Rename agents to client and added \n btw TextBlocks * Add ContextUsage and modify tools' allowed_callers * Add docstrings and pytest-asyncio --- ddev/hatch.toml | 1 + ddev/pyproject.toml | 5 +- ddev/src/ddev/ai/agent/__init__.py | 3 + ddev/src/ddev/ai/agent/client.py | 219 ++++++++++ ddev/src/ddev/ai/agent/exceptions.py | 29 ++ ddev/src/ddev/ai/tools/core/registry.py | 9 +- ddev/tests/ai/agent/__init__.py | 3 + ddev/tests/ai/agent/test_client.py | 453 +++++++++++++++++++++ ddev/tests/ai/tools/core/test_base.py | 13 +- ddev/tests/ai/tools/core/test_registry.py | 25 +- ddev/tests/ai/tools/fs/conftest.py | 5 +- ddev/tests/ai/tools/fs/test_append_file.py | 29 +- ddev/tests/ai/tools/fs/test_create_file.py | 29 +- ddev/tests/ai/tools/fs/test_edit_file.py | 43 +- ddev/tests/ai/tools/fs/test_read_file.py | 33 +- ddev/tests/ai/tools/fs/test_workflow.py | 21 +- ddev/tests/ai/tools/http/test_http_get.py | 25 +- ddev/tests/ai/tools/shell/test_base.py | 48 +-- ddev/tests/ai/tools/shell/test_tools.py | 5 +- 19 files changed, 848 insertions(+), 150 deletions(-) create mode 100644 ddev/src/ddev/ai/agent/__init__.py create mode 100644 ddev/src/ddev/ai/agent/client.py create mode 100644 ddev/src/ddev/ai/agent/exceptions.py create mode 100644 ddev/tests/ai/agent/__init__.py create mode 100644 ddev/tests/ai/agent/test_client.py diff --git a/ddev/hatch.toml b/ddev/hatch.toml index 16d9b3980fa02..9550fb4a939c3 100644 --- a/ddev/hatch.toml +++ b/ddev/hatch.toml @@ -9,6 +9,7 @@ python = "3.13" e2e-env = false dependencies = [ "pyyaml", + "pytest-asyncio", "vcrpy", ] # TODO: remove this when the old CLI is gone diff --git a/ddev/pyproject.toml b/ddev/pyproject.toml index b82b8db3bc028..f2baa037e9570 100644 --- a/ddev/pyproject.toml +++ b/ddev/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ "Programming Language :: Python :: 3.13", ] dependencies = [ - "anthropic>=0.18.0", + "anthropic>=0.86.0", "click~=8.1.6", "coverage", "datadog-api-client==2.20.0", @@ -140,3 +140,6 @@ ban-relative-imports = "parents" [tool.ruff.lint.per-file-ignores] #Tests can use assertions and relative imports "**/tests/**/*" = ["I252"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" diff --git a/ddev/src/ddev/ai/agent/__init__.py b/ddev/src/ddev/ai/agent/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/agent/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/agent/client.py b/ddev/src/ddev/ai/agent/client.py new file mode 100644 index 0000000000000..d576429015cef --- /dev/null +++ b/ddev/src/ddev/ai/agent/client.py @@ -0,0 +1,219 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from copy import deepcopy +from dataclasses import dataclass +from enum import StrEnum +from typing import Any, Final + +import anthropic +from anthropic.types import MessageParam, ToolParam, ToolResultBlockParam + +from ddev.ai.tools.core.registry import ToolRegistry + +from .exceptions import ( + AgentAPIError, + AgentConnectionError, + AgentError, + AgentRateLimitError, +) + +DEFAULT_MODEL: Final[str] = "claude-sonnet-4-6" +DEFAULT_MAX_TOKENS: Final[int] = 8192 # max tokens per response +ALLOWED_TOOL_CALLERS: Final = ["code_execution_20260120"] + + +class StopReason(StrEnum): + """Maps Anthropic API stop_reason strings to a typed enum.""" + + END_TURN = "end_turn" + MAX_TOKENS = "max_tokens" + STOP_SEQUENCE = "stop_sequence" + TOOL_USE = "tool_use" + PAUSE_TURN = "pause_turn" + REFUSAL = "refusal" + + +@dataclass(frozen=True) +class ToolCall: + """A single tool invocation requested by the model.""" + + id: str + name: str + input: dict[str, Any] + + +@dataclass(frozen=True) +class ContextUsage: + """Context window accounting for a single API call.""" + + window_size: int + used_tokens: int + + @property + def context_pct(self) -> float: + return self.used_tokens / self.window_size * 100 + + @property + def remaining_tokens(self) -> int: + return self.window_size - self.used_tokens + + +@dataclass(frozen=True) +class TokenUsage: + """Token accounting from a single API call.""" + + input_tokens: int # tokens sent to the model (system_prompt + history) + output_tokens: int # tokens the model generated + cache_read_input_tokens: int # tokens read from prompt cache + cache_creation_input_tokens: int # tokens written to prompt cache + context: ContextUsage + + +@dataclass(frozen=True) +class AgentResponse: + """The complete response from a single AnthropicAgent.send() call. + Adds useful metadata to the response of the Anthropic API.""" + + stop_reason: StopReason + text: str + tool_calls: list[ToolCall] + usage: TokenUsage + + +class AnthropicAgent: + """A wrapper around the Anthropic API that provides a simple interface for interacting with agents.""" + + def __init__( + self, + client: anthropic.AsyncAnthropic, + tools: ToolRegistry, + system_prompt: str, + name: str, + model: str = DEFAULT_MODEL, + max_tokens: int = DEFAULT_MAX_TOKENS, + programmatic_tool_calling: bool = False, + ) -> None: + """Initialize an AnthropicAgent. + Args: + client: The Anthropic client to use. + tools: The ToolRegistry to use (might not be used in every call if allowed_tools in send() is provided) + system_prompt: The system prompt to use. + name: The name of the agent. + model: The model to use. + max_tokens: The max tokens per response. + programmatic_tool_calling: Whether to allow programmatic tool calling. + """ + + self._client = client + self._tools = tools + self._system_prompt = system_prompt + self.name = name + self._model = model + self._max_tokens = max_tokens + self._programmatic_tool_calling = programmatic_tool_calling + self._history: list[MessageParam] = [] + self._context_window: int | None = None + + @property + def history(self) -> list[MessageParam]: + """Read-only snapshot of the conversation history.""" + return deepcopy(self._history) + + def reset(self) -> None: + """Clear conversation history to start a new conversation.""" + self._history = [] + + async def _get_context_window(self) -> int: + if self._context_window is None: + info = await self._client.models.retrieve(self._model) + self._context_window = info.max_input_tokens + return self._context_window + + def _get_tool_definitions(self, allowed_tools: list[str] | None) -> list[ToolParam]: + """Filter tool definitions by allowlist. None means all tools.""" + definitions = self._tools.definitions + if allowed_tools is not None: + allowed = set(allowed_tools) + definitions = [d for d in definitions if d["name"] in allowed] + if not self._programmatic_tool_calling: + definitions = [{**d, "allowed_callers": ALLOWED_TOOL_CALLERS} for d in definitions] + return definitions + + async def send( + self, + content: str | list[ToolResultBlockParam], + allowed_tools: list[str] | None = None, + ) -> AgentResponse: + """Send a message to the agent and return the response. + Args: + content: The content to send to the agent. + allowed_tools: The tools in the ToolRegistry to allow the agent to use. + Returns: + An AgentResponse object containing the response from the agent. + """ + tool_defs = self._get_tool_definitions(allowed_tools) + + user_msg: MessageParam = {"role": "user", "content": content} + messages = [*self._history, user_msg] + + try: + response = await self._client.messages.create( + model=self._model, + max_tokens=self._max_tokens, + system=self._system_prompt, + messages=messages, + tools=tool_defs if tool_defs else anthropic.NOT_GIVEN, + ) + except anthropic.APIConnectionError as e: + raise AgentConnectionError(f"Connection failed: {e}") from e + except anthropic.RateLimitError as e: + raise AgentRateLimitError(f"Rate limit exceeded: {e}") from e + except anthropic.APIStatusError as e: + raise AgentAPIError(e.status_code, e.message) from e + except anthropic.APIResponseValidationError as e: + raise AgentError(f"Response validation failed: {e}") from e + + # stop_reason is None only in streaming responses; we use non-streaming, so None is unexpected + if response.stop_reason is None: + raise AgentError("Received null stop_reason from API") + + try: + stop_reason = StopReason(response.stop_reason) + except ValueError as e: + raise AgentError(f"Unknown stop_reason: {response.stop_reason!r}") from e + + text_parts: list[str] = [] + tool_calls: list[ToolCall] = [] + + for block in response.content: + if isinstance(block, anthropic.types.TextBlock): + text_parts.append(block.text) + elif isinstance(block, anthropic.types.ToolUseBlock): + tool_calls.append(ToolCall(id=block.id, name=block.name, input=dict(block.input))) + # ThinkingBlock and RedactedThinkingBlock are intentionally ignored. + # Extended thinking support can add a `thinking: str` field to AgentResponse later. + + cache_read = response.usage.cache_read_input_tokens or 0 + cache_creation = response.usage.cache_creation_input_tokens or 0 + used_tokens = response.usage.input_tokens + cache_read + cache_creation + usage = TokenUsage( + input_tokens=response.usage.input_tokens, + output_tokens=response.usage.output_tokens, + cache_read_input_tokens=cache_read, + cache_creation_input_tokens=cache_creation, + context=ContextUsage(window_size=await self._get_context_window(), used_tokens=used_tokens), + ) + + agent_response = AgentResponse( + stop_reason=stop_reason, + text="\n".join(text_parts), + tool_calls=tool_calls, + usage=usage, + ) + + # Save to history only after a successful response. + self._history.extend([user_msg, {"role": "assistant", "content": response.content}]) + + return agent_response diff --git a/ddev/src/ddev/ai/agent/exceptions.py b/ddev/src/ddev/ai/agent/exceptions.py new file mode 100644 index 0000000000000..d0d25d3665239 --- /dev/null +++ b/ddev/src/ddev/ai/agent/exceptions.py @@ -0,0 +1,29 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + + +class AgentError(Exception): + """Base class for all errors raised by AnthropicAgent.""" + + pass + + +class AgentConnectionError(AgentError): + """Network failure — the API was unreachable.""" + + pass + + +class AgentRateLimitError(AgentError): + """Rate limit hit — the request may be retried after a delay.""" + + pass + + +class AgentAPIError(AgentError): + """The API returned an error status code.""" + + def __init__(self, status_code: int, message: str) -> None: + super().__init__(message) + self.status_code = status_code diff --git a/ddev/src/ddev/ai/tools/core/registry.py b/ddev/src/ddev/ai/tools/core/registry.py index 29c6f92fb8801..240e969a81843 100644 --- a/ddev/src/ddev/ai/tools/core/registry.py +++ b/ddev/src/ddev/ai/tools/core/registry.py @@ -2,15 +2,11 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from typing import Final - from anthropic.types import ToolParam from .protocol import ToolProtocol from .types import ToolResult -ALLOWED_TOOL_CALLERS: Final = ["code_execution_20260120"] - class ToolRegistry: """Registry holding all available tools.""" @@ -20,9 +16,8 @@ def __init__(self, tools: list[ToolProtocol]) -> None: @property def definitions(self) -> list[ToolParam]: - """Return Anthropic SDK tool definitions for all registered tools. - Each tool definition dict is not mutated, but a new dict is returned with the allowed_callers key added.""" - return [{**tool.definition, "allowed_callers": ALLOWED_TOOL_CALLERS} for tool in self._tools.values()] + """Return Anthropic SDK tool definitions for all registered tools.""" + return [tool.definition for tool in self._tools.values()] async def run(self, name: str, raw: dict[str, object]) -> ToolResult: """Execute a tool by name, returning an error result if not found.""" diff --git a/ddev/tests/ai/agent/__init__.py b/ddev/tests/ai/agent/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/agent/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/agent/test_client.py b/ddev/tests/ai/agent/test_client.py new file mode 100644 index 0000000000000..f4d1b9f5e8c96 --- /dev/null +++ b/ddev/tests/ai/agent/test_client.py @@ -0,0 +1,453 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import anthropic +import pytest + +from ddev.ai.agent.client import AnthropicAgent, StopReason +from ddev.ai.agent.exceptions import ( + AgentAPIError, + AgentConnectionError, + AgentError, + AgentRateLimitError, +) +from ddev.ai.tools.core.registry import ToolRegistry +from ddev.ai.tools.core.types import ToolResult + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def make_usage( + input_tokens: int = 10, + output_tokens: int = 20, + cache_read: int | None = None, + cache_creation: int | None = None, +) -> SimpleNamespace: + return SimpleNamespace( + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_read_input_tokens=cache_read, + cache_creation_input_tokens=cache_creation, + ) + + +def make_text_block(text: str) -> anthropic.types.TextBlock: + return anthropic.types.TextBlock(type="text", text=text) + + +def make_tool_use_block( + id: str = "toolu_01", + name: str = "read_file", + input: dict | None = None, +) -> anthropic.types.ToolUseBlock: + return anthropic.types.ToolUseBlock( + type="tool_use", + id=id, + name=name, + input=input or {"path": "/tmp/file.txt"}, + ) + + +def make_response( + stop_reason: str | None, + content: list, + usage: SimpleNamespace | None = None, +) -> SimpleNamespace: + return SimpleNamespace( + stop_reason=stop_reason, + content=content, + usage=usage or make_usage(), + ) + + +FAKE_CONTEXT_WINDOW = 200_000 + + +def make_agent( + tools: ToolRegistry | None = None, + mock_response: SimpleNamespace | None = None, +) -> tuple[AnthropicAgent, AsyncMock]: + client = MagicMock(spec=anthropic.AsyncAnthropic) + client.messages = MagicMock() + client.messages.create = AsyncMock(return_value=mock_response or make_response("end_turn", [])) + client.models = MagicMock() + client.models.retrieve = AsyncMock(return_value=SimpleNamespace(max_input_tokens=FAKE_CONTEXT_WINDOW)) + registry = tools or ToolRegistry([]) + agent = AnthropicAgent( + client=client, + tools=registry, + system_prompt="You are helpful.", + name="test-agent", + ) + return agent, client.messages.create + + +# --------------------------------------------------------------------------- +# end_turn with a single TextBlock +# --------------------------------------------------------------------------- + + +async def test_end_turn_single_text_block() -> None: + content = [make_text_block("Hello!")] + resp = make_response("end_turn", content) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Hi") + + assert result.stop_reason is StopReason.END_TURN + assert result.text == "Hello!" + assert result.tool_calls == [] + assert len(agent.history) == 2 + assert agent.history[0] == {"role": "user", "content": "Hi"} + assert agent.history[1] == {"role": "assistant", "content": content} + + +# --------------------------------------------------------------------------- +# tool_use +# --------------------------------------------------------------------------- + + +async def test_tool_use_single_block() -> None: + block = make_tool_use_block(id="toolu_42", name="read_file", input={"path": "/etc/hosts"}) + resp = make_response("tool_use", [block]) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Read hosts") + + assert result.stop_reason is StopReason.TOOL_USE + assert len(result.tool_calls) == 1 + tc = result.tool_calls[0] + assert tc.id == "toolu_42" + assert tc.name == "read_file" + assert tc.input == {"path": "/etc/hosts"} + + +# --------------------------------------------------------------------------- +# mixed TextBlock + ToolUseBlock +# --------------------------------------------------------------------------- + + +async def test_mixed_text_and_tool_use() -> None: + content = [ + make_text_block("I'll read the file for you."), + make_tool_use_block(id="toolu_01", name="read_file"), + ] + resp = make_response("tool_use", content) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Read a file") + + assert result.text == "I'll read the file for you." + assert len(result.tool_calls) == 1 + + +# --------------------------------------------------------------------------- +# Multiple TextBlocks are concatenated +# --------------------------------------------------------------------------- + + +async def test_multiple_text_blocks_are_concatenated() -> None: + content = [make_text_block("Hello, "), make_text_block("world!")] + resp = make_response("end_turn", content) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Hi") + + assert result.text == "Hello, \nworld!" + + +# --------------------------------------------------------------------------- +# max_tokens is a normal response (not an error) +# --------------------------------------------------------------------------- + + +async def test_max_tokens_is_not_an_error() -> None: + resp = make_response("max_tokens", [make_text_block("Truncated...")]) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Tell me everything") + + assert result.stop_reason is StopReason.MAX_TOKENS + assert len(agent.history) == 2 + + +# --------------------------------------------------------------------------- +# allowed_tools filtering +# --------------------------------------------------------------------------- + + +class FakeTool: + def __init__(self, name: str) -> None: + self._name = name + + @property + def name(self) -> str: + return self._name + + @property + def description(self) -> str: + return "" + + @property + def definition(self) -> dict: + return {"name": self._name, "description": "", "input_schema": {}} + + async def run(self, raw: dict) -> ToolResult: + pass + + +async def test_allowed_tools_filters_to_subset() -> None: + registry = ToolRegistry([FakeTool(n) for n in ["read_file", "grep", "mkdir"]]) + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(tools=registry, mock_response=resp) + + await agent.send("Hi", allowed_tools=["read_file"]) + + sent_names = [t["name"] for t in create_mock.call_args.kwargs["tools"]] + assert sent_names == ["read_file"] + + +async def test_allowed_tools_none_passes_all() -> None: + registry = ToolRegistry([FakeTool(n) for n in ["a", "b"]]) + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(tools=registry, mock_response=resp) + + await agent.send("Hi", allowed_tools=None) + + sent_names = [t["name"] for t in create_mock.call_args.kwargs["tools"]] + assert sent_names == ["a", "b"] + + +@pytest.mark.parametrize("allowed_tools", [[], ["nonexistent_tool"]]) +async def test_allowed_tools_passes_not_given(allowed_tools: list[str]) -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(mock_response=resp) + + await agent.send("Hi", allowed_tools=allowed_tools) + + assert create_mock.call_args.kwargs["tools"] is anthropic.NOT_GIVEN + + +# --------------------------------------------------------------------------- +# API errors map to the correct AgentError subclass +# --------------------------------------------------------------------------- + + +def _make_error_agent(side_effect: Exception) -> AnthropicAgent: + client = MagicMock(spec=anthropic.AsyncAnthropic) + client.messages = MagicMock() + client.messages.create = AsyncMock(side_effect=side_effect) + return AnthropicAgent(client=client, tools=ToolRegistry([]), system_prompt="", name="t") + + +async def test_connection_error_maps_to_agent_connection_error() -> None: + agent = _make_error_agent(anthropic.APIConnectionError(request=MagicMock())) + + with pytest.raises(AgentConnectionError) as exc_info: + await agent.send("Hi") + + assert "Connection failed" in str(exc_info.value) + assert agent.history == [] + + +async def test_rate_limit_error_maps_to_agent_rate_limit_error() -> None: + agent = _make_error_agent( + anthropic.RateLimitError( + message="rate limit", + response=MagicMock(status_code=429, headers={}), + body=None, + ) + ) + + with pytest.raises(AgentRateLimitError) as exc_info: + await agent.send("Hi") + + assert "Rate limit exceeded" in str(exc_info.value) + assert agent.history == [] + + +async def test_api_status_error_maps_to_agent_api_error() -> None: + agent = _make_error_agent( + anthropic.APIStatusError( + message="internal server error", + response=MagicMock(status_code=500), + body=None, + ) + ) + + with pytest.raises(AgentAPIError) as exc_info: + await agent.send("Hi") + + assert exc_info.value.status_code == 500 + assert agent.history == [] + + +async def test_response_validation_error_maps_to_agent_error() -> None: + agent = _make_error_agent(anthropic.APIResponseValidationError(response=MagicMock(), body=None)) + + with pytest.raises(AgentError) as exc_info: + await agent.send("Hi") + + assert "Response validation failed" in str(exc_info.value) + assert agent.history == [] + + +# --------------------------------------------------------------------------- +# Unknown stop_reason raises AgentError, history unchanged +# --------------------------------------------------------------------------- + + +async def test_unknown_stop_reason_raises_agent_error() -> None: + resp = make_response("totally_unknown_reason", []) + agent, _ = make_agent(mock_response=resp) + + with pytest.raises(AgentError) as exc_info: + await agent.send("Hi") + + assert agent.history == [] + assert "Unknown stop_reason" in str(exc_info.value) + assert "totally_unknown_reason" in str(exc_info.value) + + +# --------------------------------------------------------------------------- +# cache_read_input_tokens=None defaults to 0 +# --------------------------------------------------------------------------- + + +async def test_cache_tokens_none_defaults_to_zero() -> None: + usage = make_usage(cache_read=None, cache_creation=None) + resp = make_response("end_turn", [make_text_block("ok")], usage=usage) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Hi") + + assert result.usage.cache_read_input_tokens == 0 + assert result.usage.cache_creation_input_tokens == 0 + + +# --------------------------------------------------------------------------- +# ContextUsage fields +# --------------------------------------------------------------------------- + + +async def test_context_usage_fields() -> None: + usage = make_usage(input_tokens=1000, cache_read=500, cache_creation=200) + resp = make_response("end_turn", [make_text_block("ok")], usage=usage) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Hi") + + ctx = result.usage.context + assert ctx.window_size == FAKE_CONTEXT_WINDOW + assert ctx.used_tokens == 1700 # 1000 + 500 + 200 + assert ctx.context_pct == pytest.approx(1700 / FAKE_CONTEXT_WINDOW * 100) + assert ctx.remaining_tokens == FAKE_CONTEXT_WINDOW - 1700 + + +# --------------------------------------------------------------------------- +# context_window is fetched once and cached across multiple sends +# --------------------------------------------------------------------------- + + +async def test_context_window_fetched_once() -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, _ = make_agent(mock_response=resp) + agent._client.messages.create = AsyncMock(return_value=resp) + + await agent.send("First") + await agent.send("Second") + + agent._client.models.retrieve.assert_awaited_once() + + +# --------------------------------------------------------------------------- +# Multi-turn — send str then send tool results → history has 4 entries +# --------------------------------------------------------------------------- + + +async def test_multi_turn_history_grows_correctly() -> None: + tool_resp = make_response("tool_use", [make_tool_use_block(id="toolu_01")]) + text_resp = make_response("end_turn", [make_text_block("Done.")]) + + client = MagicMock(spec=anthropic.AsyncAnthropic) + client.messages = MagicMock() + client.messages.create = AsyncMock(side_effect=[tool_resp, text_resp]) + client.models = MagicMock() + client.models.retrieve = AsyncMock(return_value=SimpleNamespace(max_input_tokens=FAKE_CONTEXT_WINDOW)) + agent = AnthropicAgent(client=client, tools=ToolRegistry([]), system_prompt="", name="t") + + first = await agent.send("Do X") + assert first.stop_reason is StopReason.TOOL_USE + assert len(agent.history) == 2 + + tool_results = [{"type": "tool_result", "tool_use_id": "toolu_01", "content": "result"}] + second = await agent.send(tool_results) + assert second.stop_reason is StopReason.END_TURN + assert len(agent.history) == 4 + assert agent.history[2]["role"] == "user" + assert agent.history[3]["role"] == "assistant" + + +# --------------------------------------------------------------------------- +# history property returns a copy +# --------------------------------------------------------------------------- + + +async def test_history_property_returns_copy() -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, _ = make_agent(mock_response=resp) + await agent.send("Hi") + + snapshot = agent.history + snapshot.clear() + + assert len(agent.history) == 2 + + +# --------------------------------------------------------------------------- +# reset() clears history +# --------------------------------------------------------------------------- + + +async def test_reset_clears_history() -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, _ = make_agent(mock_response=resp) + await agent.send("Hi") + assert len(agent.history) == 2 + + agent.reset() + assert agent.history == [] + + +# --------------------------------------------------------------------------- +# Error mid-conversation leaves history unchanged +# --------------------------------------------------------------------------- + + +async def test_error_mid_conversation_leaves_history_unchanged() -> None: + ok_resp = make_response("end_turn", [make_text_block("ok")]) + client = MagicMock(spec=anthropic.AsyncAnthropic) + client.messages = MagicMock() + client.messages.create = AsyncMock( + side_effect=[ + ok_resp, + anthropic.APIConnectionError(request=MagicMock()), + ] + ) + client.models = MagicMock() + client.models.retrieve = AsyncMock(return_value=SimpleNamespace(max_input_tokens=FAKE_CONTEXT_WINDOW)) + agent = AnthropicAgent(client=client, tools=ToolRegistry([]), system_prompt="", name="t") + + await agent.send("First message") + history_after_first = agent.history[:] + + with pytest.raises(AgentConnectionError): + await agent.send("Second message") + + assert agent.history == history_after_first diff --git a/ddev/tests/ai/tools/core/test_base.py b/ddev/tests/ai/tools/core/test_base.py index 96cd0f8b07d0c..35e94f750a69e 100644 --- a/ddev/tests/ai/tools/core/test_base.py +++ b/ddev/tests/ai/tools/core/test_base.py @@ -1,7 +1,6 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import asyncio from typing import Annotated import pytest @@ -194,8 +193,8 @@ async def __call__(self, tool_input: SimpleInput) -> ToolResult: # --- run(): happy path --- -def test_run_valid_input_returns_success(echo_tool: EchoTool): - result = asyncio.run(echo_tool.run({"message": "hello"})) +async def test_run_valid_input_returns_success(echo_tool: EchoTool): + result = await echo_tool.run({"message": "hello"}) assert result.success is True assert result.data == "hello" @@ -210,8 +209,8 @@ def test_run_valid_input_returns_success(echo_tool: EchoTool): {"message": "hi", "extra": "oops"}, ], ) -def test_run_invalid_input_returns_failure(echo_tool: EchoTool, raw: dict): - result = asyncio.run(echo_tool.run(raw)) +async def test_run_invalid_input_returns_failure(echo_tool: EchoTool, raw: dict): + result = await echo_tool.run(raw) assert result.success is False assert result.error is not None @@ -219,8 +218,8 @@ def test_run_invalid_input_returns_failure(echo_tool: EchoTool, raw: dict): # --- run(): __call__ exception handling --- -def test_run_captures_exception_from_call(failing_tool: FailingTool): - result = asyncio.run(failing_tool.run({"message": "boom"})) +async def test_run_captures_exception_from_call(failing_tool: FailingTool): + result = await failing_tool.run({"message": "boom"}) assert isinstance(result, ToolResult) assert result.success is False assert "RuntimeError" in result.error diff --git a/ddev/tests/ai/tools/core/test_registry.py b/ddev/tests/ai/tools/core/test_registry.py index fdd42714b6ed4..1366a9d8b5be8 100644 --- a/ddev/tests/ai/tools/core/test_registry.py +++ b/ddev/tests/ai/tools/core/test_registry.py @@ -1,11 +1,10 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import asyncio import pytest -from ddev.ai.tools.core.registry import ALLOWED_TOOL_CALLERS, ToolRegistry +from ddev.ai.tools.core.registry import ToolRegistry from ddev.ai.tools.core.types import ToolResult # --------------------------------------------------------------------------- @@ -76,8 +75,6 @@ def test_empty_registry_returns_empty_list(): def test_tool_registry_definitions_returns_all_tool_definitions(): registry = ToolRegistry([FakeTool("a"), FakeTool("b")]) assert len(registry.definitions) == 2 - for defn in registry.definitions: - assert defn["allowed_callers"] == ALLOWED_TOOL_CALLERS def test_definition_contains_tool_name(): @@ -90,41 +87,41 @@ def test_definition_contains_tool_name(): # --------------------------------------------------------------------------- -def test_run_dispatches_to_correct_tool(): +async def test_run_dispatches_to_correct_tool(): tool_a = FakeTool("a", ToolResult(success=True, data="from a")) tool_b = FakeTool("b", ToolResult(success=True, data="from b")) registry = ToolRegistry([tool_a, tool_b]) - result = asyncio.run(registry.run("b", {})) + result = await registry.run("b", {}) assert result.success is True assert result.data == "from b" -def test_passes_raw_dict_to_tool_unchanged(): +async def test_passes_raw_dict_to_tool_unchanged(): tool = FakeTool("t") registry = ToolRegistry([tool]) raw = {"key": "value", "num": 42} - asyncio.run(registry.run("t", raw)) + await registry.run("t", raw) assert tool.last_raw == raw -def test_returns_tool_result_on_tool_failure(): +async def test_returns_tool_result_on_tool_failure(): registry = ToolRegistry([FakeTool("t", ToolResult(success=False, error="bad input"))]) - result = asyncio.run(registry.run("t", {})) + result = await registry.run("t", {}) assert result.success is False assert result.error == "bad input" -def test_unknown_tool_returns_failure(): +async def test_unknown_tool_returns_failure(): registry = ToolRegistry([FakeTool("known_tool")]) - result = asyncio.run(registry.run("unknown_tool", {})) + result = await registry.run("unknown_tool", {}) assert result.success is False assert "Unknown tool: 'unknown_tool'" in result.error -def test_empty_registry_always_returns_unknown_error(): +async def test_empty_registry_always_returns_unknown_error(): registry = ToolRegistry([]) - result = asyncio.run(registry.run("anything", {})) + result = await registry.run("anything", {}) assert result.success is False assert result.error is not None diff --git a/ddev/tests/ai/tools/fs/conftest.py b/ddev/tests/ai/tools/fs/conftest.py index 8d6677b98c398..12ae9e34eb1d5 100644 --- a/ddev/tests/ai/tools/fs/conftest.py +++ b/ddev/tests/ai/tools/fs/conftest.py @@ -1,7 +1,6 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import asyncio import pytest @@ -38,8 +37,8 @@ def append_tool(registry: FileRegistry) -> AppendFileTool: @pytest.fixture -def known_file(tmp_path, create_tool: CreateFileTool): +async def known_file(tmp_path, create_tool: CreateFileTool): """A temp file registered in the registry via create.""" f = tmp_path / "file.txt" - asyncio.run(create_tool.run({"path": str(f), "content": "line one\nline two\nline three\n"})) + await create_tool.run({"path": str(f), "content": "line one\nline two\nline three\n"}) return f diff --git a/ddev/tests/ai/tools/fs/test_append_file.py b/ddev/tests/ai/tools/fs/test_append_file.py index 2b669572d30bb..289142e378191 100644 --- a/ddev/tests/ai/tools/fs/test_append_file.py +++ b/ddev/tests/ai/tools/fs/test_append_file.py @@ -1,7 +1,6 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import asyncio from unittest.mock import patch import pytest @@ -23,8 +22,10 @@ def test_tool_name(registry: FileRegistry) -> None: ("A\r\nB\r\n", "A\nB\n", "\r"), ], ) -def test_append_file_success(append_tool: AppendFileTool, known_file, content, expected_in, expected_not_in) -> None: - result = asyncio.run(append_tool.run({"path": str(known_file), "content": content})) +async def test_append_file_success( + append_tool: AppendFileTool, known_file, content, expected_in, expected_not_in +) -> None: + result = await append_tool.run({"path": str(known_file), "content": content}) assert result.success is True text = known_file.read_text(encoding="utf-8") @@ -33,11 +34,11 @@ def test_append_file_success(append_tool: AppendFileTool, known_file, content, e assert expected_not_in not in text -def test_append_file_fails_for_unregistered_file(append_tool: AppendFileTool, tmp_path) -> None: +async def test_append_file_fails_for_unregistered_file(append_tool: AppendFileTool, tmp_path) -> None: f = tmp_path / "unread.txt" f.write_text("content", encoding="utf-8") - result = asyncio.run(append_tool.run({"path": str(f), "content": "more"})) + result = await append_tool.run({"path": str(f), "content": "more"}) assert result.success is False assert "Not authorized" in result.error @@ -50,39 +51,39 @@ def test_append_file_fails_for_unregistered_file(append_tool: AppendFileTool, tm ("", "first line", "first line"), ], ) -def test_append_file_separator( +async def test_append_file_separator( append_tool: AppendFileTool, create_tool: CreateFileTool, tmp_path, initial, appended, expected ) -> None: f = tmp_path / "file.txt" - asyncio.run(create_tool.run({"path": str(f), "content": initial})) + await create_tool.run({"path": str(f), "content": initial}) - result = asyncio.run(append_tool.run({"path": str(f), "content": appended})) + result = await append_tool.run({"path": str(f), "content": appended}) assert result.success is True assert f.read_text(encoding="utf-8") == expected -def test_append_file_fails_if_file_changed_externally(append_tool: AppendFileTool, known_file) -> None: +async def test_append_file_fails_if_file_changed_externally(append_tool: AppendFileTool, known_file) -> None: known_file.write_text("externally modified\n", encoding="utf-8") - result = asyncio.run(append_tool.run({"path": str(known_file), "content": "more"})) + result = await append_tool.run({"path": str(known_file), "content": "more"}) assert result.success is False assert "Re-read and retry" in result.error -def test_append_file_updates_registry(append_tool: AppendFileTool, registry: FileRegistry, known_file) -> None: - asyncio.run(append_tool.run({"path": str(known_file), "content": "extra\n"})) +async def test_append_file_updates_registry(append_tool: AppendFileTool, registry: FileRegistry, known_file) -> None: + await append_tool.run({"path": str(known_file), "content": "extra\n"}) new_content = known_file.read_text(encoding="utf-8") assert registry.verify(str(known_file), new_content) is True -def test_append_file_oserror_on_write(append_tool: AppendFileTool, registry: FileRegistry, known_file) -> None: +async def test_append_file_oserror_on_write(append_tool: AppendFileTool, registry: FileRegistry, known_file) -> None: original_content = known_file.read_text(encoding="utf-8") with patch("pathlib.Path.write_text", side_effect=PermissionError("permission denied")): - result = asyncio.run(append_tool.run({"path": str(known_file), "content": "new line"})) + result = await append_tool.run({"path": str(known_file), "content": "new line"}) assert result.success is False assert result.error is not None diff --git a/ddev/tests/ai/tools/fs/test_create_file.py b/ddev/tests/ai/tools/fs/test_create_file.py index 2714ef5bb06aa..8b0c0296fa38a 100644 --- a/ddev/tests/ai/tools/fs/test_create_file.py +++ b/ddev/tests/ai/tools/fs/test_create_file.py @@ -1,7 +1,6 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import asyncio from unittest.mock import patch from ddev.ai.tools.fs.create_file import CreateFileTool @@ -12,41 +11,41 @@ def test_tool_name(registry: FileRegistry) -> None: assert CreateFileTool(registry).name == "create_file" -def test_create_file_success(create_tool: CreateFileTool, tmp_path) -> None: +async def test_create_file_success(create_tool: CreateFileTool, tmp_path) -> None: f = tmp_path / "new.txt" - result = asyncio.run(create_tool.run({"path": str(f), "content": "hello"})) + result = await create_tool.run({"path": str(f), "content": "hello"}) assert result.success is True assert f.read_text(encoding="utf-8") == "hello" -def test_create_file_default_empty_content(create_tool: CreateFileTool, tmp_path) -> None: +async def test_create_file_default_empty_content(create_tool: CreateFileTool, tmp_path) -> None: f = tmp_path / "empty.txt" - result = asyncio.run(create_tool.run({"path": str(f)})) + result = await create_tool.run({"path": str(f)}) assert result.success is True assert f.read_text(encoding="utf-8") == "" -def test_create_file_creates_missing_parent_dirs(create_tool: CreateFileTool, tmp_path) -> None: +async def test_create_file_creates_missing_parent_dirs(create_tool: CreateFileTool, tmp_path) -> None: f = tmp_path / "a" / "b" / "c" / "file.txt" - result = asyncio.run(create_tool.run({"path": str(f), "content": "nested"})) + result = await create_tool.run({"path": str(f), "content": "nested"}) assert result.success is True assert f.exists() assert f.read_text(encoding="utf-8") == "nested" -def test_create_file_fails_if_file_already_exists( +async def test_create_file_fails_if_file_already_exists( create_tool: CreateFileTool, registry: FileRegistry, tmp_path ) -> None: f = tmp_path / "existing.txt" f.write_text("original", encoding="utf-8") - result = asyncio.run(create_tool.run({"path": str(f), "content": "new"})) + result = await create_tool.run({"path": str(f), "content": "new"}) assert result.success is False assert result.error is not None @@ -54,19 +53,19 @@ def test_create_file_fails_if_file_already_exists( assert not registry.is_known(str(f)) -def test_create_tool_registers_in_registry(create_tool: CreateFileTool, registry: FileRegistry, tmp_path) -> None: +async def test_create_tool_registers_in_registry(create_tool: CreateFileTool, registry: FileRegistry, tmp_path) -> None: f = tmp_path / "file.txt" - asyncio.run(create_tool.run({"path": str(f), "content": "hi"})) + await create_tool.run({"path": str(f), "content": "hi"}) assert registry.is_known(str(f)) is True assert registry.verify(str(f), "hi") is True -def test_create_file_oserror_on_mkdir(create_tool: CreateFileTool, registry: FileRegistry, tmp_path) -> None: +async def test_create_file_oserror_on_mkdir(create_tool: CreateFileTool, registry: FileRegistry, tmp_path) -> None: f = tmp_path / "a" / "b" / "new.txt" with patch("pathlib.Path.mkdir", side_effect=PermissionError("permission denied")): - result = asyncio.run(create_tool.run({"path": str(f), "content": "hi"})) + result = await create_tool.run({"path": str(f), "content": "hi"}) assert result.success is False assert result.error is not None @@ -74,11 +73,11 @@ def test_create_file_oserror_on_mkdir(create_tool: CreateFileTool, registry: Fil assert not registry.is_known(str(f)) -def test_create_file_oserror_on_write(create_tool: CreateFileTool, registry: FileRegistry, tmp_path) -> None: +async def test_create_file_oserror_on_write(create_tool: CreateFileTool, registry: FileRegistry, tmp_path) -> None: f = tmp_path / "new.txt" with patch("pathlib.Path.write_text", side_effect=PermissionError("permission denied")): - result = asyncio.run(create_tool.run({"path": str(f), "content": "hi"})) + result = await create_tool.run({"path": str(f), "content": "hi"}) assert result.success is False assert result.error is not None diff --git a/ddev/tests/ai/tools/fs/test_edit_file.py b/ddev/tests/ai/tools/fs/test_edit_file.py index cbfd48a78c193..27c8b87cedce2 100644 --- a/ddev/tests/ai/tools/fs/test_edit_file.py +++ b/ddev/tests/ai/tools/fs/test_edit_file.py @@ -1,7 +1,6 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import asyncio from unittest.mock import patch import pytest @@ -15,8 +14,8 @@ def test_tool_name(registry: FileRegistry) -> None: assert EditFileTool(registry).name == "edit_file" -def test_edit_file_replaces_string(edit_tool: EditFileTool, known_file) -> None: - result = asyncio.run(edit_tool.run({"path": str(known_file), "old_string": "line two", "new_string": "line TWO"})) +async def test_edit_file_replaces_string(edit_tool: EditFileTool, known_file) -> None: + result = await edit_tool.run({"path": str(known_file), "old_string": "line two", "new_string": "line TWO"}) assert result.success is True content = known_file.read_text(encoding="utf-8") @@ -24,54 +23,56 @@ def test_edit_file_replaces_string(edit_tool: EditFileTool, known_file) -> None: assert "line two" not in content -def test_edit_file_deletes_line(edit_tool: EditFileTool, known_file) -> None: - result = asyncio.run(edit_tool.run({"path": str(known_file), "old_string": "line two\n", "new_string": ""})) +async def test_edit_file_deletes_line(edit_tool: EditFileTool, known_file) -> None: + result = await edit_tool.run({"path": str(known_file), "old_string": "line two\n", "new_string": ""}) assert result.success is True assert "line two" not in known_file.read_text(encoding="utf-8") -def test_edit_file_fails_for_unregistered_file(edit_tool: EditFileTool, tmp_path) -> None: +async def test_edit_file_fails_for_unregistered_file(edit_tool: EditFileTool, tmp_path) -> None: f = tmp_path / "unread.txt" f.write_text("content", encoding="utf-8") - result = asyncio.run(edit_tool.run({"path": str(f), "old_string": "content", "new_string": "new"})) + result = await edit_tool.run({"path": str(f), "old_string": "content", "new_string": "new"}) assert result.success is False assert "Not authorized" in result.error @pytest.mark.parametrize("old_string", ["does not exist", ""]) -def test_edit_file_fails_if_old_string_not_found_or_empty(edit_tool: EditFileTool, known_file, old_string) -> None: - result = asyncio.run(edit_tool.run({"path": str(known_file), "old_string": old_string, "new_string": "x"})) +async def test_edit_file_fails_if_old_string_not_found_or_empty( + edit_tool: EditFileTool, known_file, old_string +) -> None: + result = await edit_tool.run({"path": str(known_file), "old_string": old_string, "new_string": "x"}) assert result.success is False -def test_edit_file_fails_if_old_string_ambiguous( +async def test_edit_file_fails_if_old_string_ambiguous( edit_tool: EditFileTool, create_tool: CreateFileTool, tmp_path ) -> None: f = tmp_path / "dup.txt" - asyncio.run(create_tool.run({"path": str(f), "content": "foo\nfoo\nfoo\n"})) + await create_tool.run({"path": str(f), "content": "foo\nfoo\nfoo\n"}) - result = asyncio.run(edit_tool.run({"path": str(f), "old_string": "foo", "new_string": "bar"})) + result = await edit_tool.run({"path": str(f), "old_string": "foo", "new_string": "bar"}) assert result.success is False assert "3" in result.error assert result.hint is not None -def test_edit_file_fails_if_file_changed_externally(edit_tool: EditFileTool, known_file) -> None: +async def test_edit_file_fails_if_file_changed_externally(edit_tool: EditFileTool, known_file) -> None: known_file.write_text("externally modified\n", encoding="utf-8") - result = asyncio.run(edit_tool.run({"path": str(known_file), "old_string": "line one", "new_string": "x"})) + result = await edit_tool.run({"path": str(known_file), "old_string": "line one", "new_string": "x"}) assert result.success is False assert "Re-read and retry" in result.error -def test_edit_file_updates_registry(edit_tool: EditFileTool, registry: FileRegistry, known_file) -> None: - asyncio.run(edit_tool.run({"path": str(known_file), "old_string": "line one", "new_string": "LINE ONE"})) +async def test_edit_file_updates_registry(edit_tool: EditFileTool, registry: FileRegistry, known_file) -> None: + await edit_tool.run({"path": str(known_file), "old_string": "line one", "new_string": "LINE ONE"}) new_content = known_file.read_text(encoding="utf-8") assert registry.verify(str(known_file), new_content) is True @@ -85,23 +86,23 @@ def test_edit_file_updates_registry(edit_tool: EditFileTool, registry: FileRegis ("line one\n", "line one", "A\r\nB", "A\nB\n"), # CRLF in new_string ], ) -def test_edit_file_normalizes_crlf( +async def test_edit_file_normalizes_crlf( edit_tool: EditFileTool, create_tool: CreateFileTool, tmp_path, file_content, old_string, new_string, expected ) -> None: f = tmp_path / "file.txt" - asyncio.run(create_tool.run({"path": str(f), "content": file_content})) + await create_tool.run({"path": str(f), "content": file_content}) - result = asyncio.run(edit_tool.run({"path": str(f), "old_string": old_string, "new_string": new_string})) + result = await edit_tool.run({"path": str(f), "old_string": old_string, "new_string": new_string}) assert result.success is True assert f.read_text(encoding="utf-8") == expected -def test_edit_file_oserror_on_write(edit_tool: EditFileTool, registry: FileRegistry, known_file) -> None: +async def test_edit_file_oserror_on_write(edit_tool: EditFileTool, registry: FileRegistry, known_file) -> None: original_content = known_file.read_text(encoding="utf-8") with patch("pathlib.Path.write_text", side_effect=PermissionError("permission denied")): - result = asyncio.run(edit_tool.run({"path": str(known_file), "old_string": "line one", "new_string": "x"})) + result = await edit_tool.run({"path": str(known_file), "old_string": "line one", "new_string": "x"}) assert result.success is False assert result.error is not None diff --git a/ddev/tests/ai/tools/fs/test_read_file.py b/ddev/tests/ai/tools/fs/test_read_file.py index f1b8da06d91ed..f2497e6c09a18 100644 --- a/ddev/tests/ai/tools/fs/test_read_file.py +++ b/ddev/tests/ai/tools/fs/test_read_file.py @@ -1,7 +1,6 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import asyncio from unittest.mock import patch import pytest @@ -14,47 +13,47 @@ def test_tool_name(registry: FileRegistry) -> None: assert ReadFileTool(registry).name == "read_file" -def test_read_file_success(read_tool: ReadFileTool, tmp_path) -> None: +async def test_read_file_success(read_tool: ReadFileTool, tmp_path) -> None: f = tmp_path / "config.txt" f.write_text("hello\nworld\n", encoding="utf-8") - result = asyncio.run(read_tool.run({"path": str(f)})) + result = await read_tool.run({"path": str(f)}) assert result.success is True assert result.data == "0: hello\n1: world\n" -def test_read_registers_unknown_file(read_tool: ReadFileTool, registry: FileRegistry, tmp_path) -> None: +async def test_read_registers_unknown_file(read_tool: ReadFileTool, registry: FileRegistry, tmp_path) -> None: f = tmp_path / "file.txt" f.write_text("content", encoding="utf-8") - asyncio.run(read_tool.run({"path": str(f)})) + await read_tool.run({"path": str(f)}) assert registry.is_known(str(f)) is True -def test_read_file_missing_file(read_tool: ReadFileTool, tmp_path) -> None: - result = asyncio.run(read_tool.run({"path": str(tmp_path / "ghost.txt")})) +async def test_read_file_missing_file(read_tool: ReadFileTool, tmp_path) -> None: + result = await read_tool.run({"path": str(tmp_path / "ghost.txt")}) assert result.success is False assert str(tmp_path / "ghost.txt") in result.error -def test_read_file_permission_error(read_tool: ReadFileTool, tmp_path) -> None: +async def test_read_file_permission_error(read_tool: ReadFileTool, tmp_path) -> None: f = tmp_path / "secret.txt" f.write_text("secret", encoding="utf-8") with patch("pathlib.Path.read_text", side_effect=PermissionError("permission denied")): - result = asyncio.run(read_tool.run({"path": str(f)})) + result = await read_tool.run({"path": str(f)}) assert result.success is False assert result.error is not None -def test_read_file_binary_file(read_tool: ReadFileTool, tmp_path) -> None: +async def test_read_file_binary_file(read_tool: ReadFileTool, tmp_path) -> None: f = tmp_path / "binary.bin" f.write_bytes(b"\xff\xfe\x00binary") - result = asyncio.run(read_tool.run({"path": str(f)})) + result = await read_tool.run({"path": str(f)}) assert result.success is False assert result.error is not None @@ -71,23 +70,23 @@ def test_read_file_binary_file(read_tool: ReadFileTool, tmp_path) -> None: (100, None, ""), # offset beyond EOF ], ) -def test_read_file_with_offset_and_limit(read_tool: ReadFileTool, tmp_path, offset, limit, expected) -> None: +async def test_read_file_with_offset_and_limit(read_tool: ReadFileTool, tmp_path, offset, limit, expected) -> None: f = tmp_path / "file.txt" f.write_text("a\nb\nc\n", encoding="utf-8") - result = asyncio.run(read_tool.run({"path": str(f), "offset": offset, "limit": limit})) + result = await read_tool.run({"path": str(f), "offset": offset, "limit": limit}) assert result.success is True assert result.data == expected -def test_read_file_truncated(read_tool: ReadFileTool, tmp_path) -> None: +async def test_read_file_truncated(read_tool: ReadFileTool, tmp_path) -> None: from ddev.ai.tools.core.truncation import MAX_CHARS f = tmp_path / "large.txt" f.write_text("x" * (MAX_CHARS + 1000), encoding="utf-8") - result = asyncio.run(read_tool.run({"path": str(f)})) + result = await read_tool.run({"path": str(f)}) assert result.success is True assert result.truncated is True @@ -95,11 +94,11 @@ def test_read_file_truncated(read_tool: ReadFileTool, tmp_path) -> None: assert result.hint is not None -def test_read_file_no_trailing_newline(read_tool: ReadFileTool, tmp_path) -> None: +async def test_read_file_no_trailing_newline(read_tool: ReadFileTool, tmp_path) -> None: f = tmp_path / "file.txt" f.write_text("no newline at end", encoding="utf-8") - result = asyncio.run(read_tool.run({"path": str(f)})) + result = await read_tool.run({"path": str(f)}) assert result.success is True assert result.data == "0: no newline at end" diff --git a/ddev/tests/ai/tools/fs/test_workflow.py b/ddev/tests/ai/tools/fs/test_workflow.py index 077f63189bf91..a45ad9d937e26 100644 --- a/ddev/tests/ai/tools/fs/test_workflow.py +++ b/ddev/tests/ai/tools/fs/test_workflow.py @@ -1,7 +1,6 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import asyncio from ddev.ai.tools.fs.append_file import AppendFileTool from ddev.ai.tools.fs.create_file import CreateFileTool @@ -10,7 +9,7 @@ from ddev.ai.tools.fs.read_file import ReadFileTool -def test_workflow_create_read_edit_append( +async def test_workflow_create_read_edit_append( create_tool: CreateFileTool, read_tool: ReadFileTool, edit_tool: EditFileTool, @@ -21,20 +20,20 @@ def test_workflow_create_read_edit_append( f = tmp_path / "workflow.txt" # Step 1: create - r = asyncio.run(create_tool.run({"path": str(f), "content": "version: 1\n"})) + r = await create_tool.run({"path": str(f), "content": "version: 1\n"}) assert r.success is True # Step 2: read (registers current content) - r = asyncio.run(read_tool.run({"path": str(f)})) + r = await read_tool.run({"path": str(f)}) assert r.success is True # Step 3: edit - r = asyncio.run(edit_tool.run({"path": str(f), "old_string": "version: 1", "new_string": "version: 2"})) + r = await edit_tool.run({"path": str(f), "old_string": "version: 1", "new_string": "version: 2"}) assert r.success is True assert "version: 2" in f.read_text(encoding="utf-8") # Step 4: append - r = asyncio.run(append_tool.run({"path": str(f), "content": "# updated\n"})) + r = await append_tool.run({"path": str(f), "content": "# updated\n"}) assert r.success is True assert f.read_text(encoding="utf-8").endswith("# updated\n") @@ -42,22 +41,22 @@ def test_workflow_create_read_edit_append( assert registry.verify(str(f), f.read_text(encoding="utf-8")) is True -def test_workflow_stale_file( +async def test_workflow_stale_file( create_tool: CreateFileTool, read_tool: ReadFileTool, edit_tool: EditFileTool, tmp_path, ) -> None: f = tmp_path / "shared.txt" - asyncio.run(create_tool.run({"path": str(f), "content": "original\n"})) + await create_tool.run({"path": str(f), "content": "original\n"}) f.write_text("updated externally\n", encoding="utf-8") - result = asyncio.run(edit_tool.run({"path": str(f), "old_string": "original", "new_string": "my edit"})) + result = await edit_tool.run({"path": str(f), "old_string": "original", "new_string": "my edit"}) assert result.success is False assert "Re-read and retry" in result.error - asyncio.run(read_tool.run({"path": str(f)})) + await read_tool.run({"path": str(f)}) - result = asyncio.run(edit_tool.run({"path": str(f), "old_string": "updated externally", "new_string": "final"})) + result = await edit_tool.run({"path": str(f), "old_string": "updated externally", "new_string": "final"}) assert result.success is True assert f.read_text(encoding="utf-8") == "final\n" diff --git a/ddev/tests/ai/tools/http/test_http_get.py b/ddev/tests/ai/tools/http/test_http_get.py index d2e8c06220fa1..2cb871bdfd62a 100644 --- a/ddev/tests/ai/tools/http/test_http_get.py +++ b/ddev/tests/ai/tools/http/test_http_get.py @@ -1,7 +1,6 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import asyncio from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -51,8 +50,8 @@ def test_tool_meta(http_tool: HttpGetTool) -> None: @pytest.mark.parametrize("url", ["ftp://example.com", "example.com", "", "//example.com"]) -def test_invalid_url(http_tool: HttpGetTool, url: str) -> None: - result = asyncio.run(http_tool.run({"url": url})) +async def test_invalid_url(http_tool: HttpGetTool, url: str) -> None: + result = await http_tool.run({"url": url}) assert result.success is False assert "http" in result.error and "https" in result.error @@ -71,9 +70,9 @@ def test_invalid_url(http_tool: HttpGetTool, url: str) -> None: (204, ""), ], ) -def test_request_success(http_tool: HttpGetTool, status_code: int, body: str) -> None: +async def test_request_success(http_tool: HttpGetTool, status_code: int, body: str) -> None: with patch_httpx(fake_response(status_code, body)): - result = asyncio.run(http_tool.run({"url": "http://localhost:9090/metrics"})) + result = await http_tool.run({"url": "http://localhost:9090/metrics"}) assert result.success is True assert f"Status: {status_code}" in result.data @@ -81,9 +80,9 @@ def test_request_success(http_tool: HttpGetTool, status_code: int, body: str) -> @pytest.mark.parametrize("status_code", [400, 404, 500, 503]) -def test_request_non_success_status(http_tool: HttpGetTool, status_code: int) -> None: +async def test_request_non_success_status(http_tool: HttpGetTool, status_code: int) -> None: with patch_httpx(fake_response(status_code, "error body")): - result = asyncio.run(http_tool.run({"url": "http://localhost:9090/metrics"})) + result = await http_tool.run({"url": "http://localhost:9090/metrics"}) assert result.success is True assert f"Status: {status_code}" in result.data @@ -94,17 +93,17 @@ def test_request_non_success_status(http_tool: HttpGetTool, status_code: int) -> # --------------------------------------------------------------------------- -def test_request_timeout(http_tool: HttpGetTool) -> None: +async def test_request_timeout(http_tool: HttpGetTool) -> None: with patch_httpx(side_effect=httpx.TimeoutException("timed out")): - result = asyncio.run(http_tool.run({"url": "http://localhost:9090/metrics", "timeout": 1.0})) + result = await http_tool.run({"url": "http://localhost:9090/metrics", "timeout": 1.0}) assert result.success is False assert "timed out after 1.0s" in result.error -def test_request_error(http_tool: HttpGetTool) -> None: +async def test_request_error(http_tool: HttpGetTool) -> None: with patch_httpx(side_effect=httpx.RequestError("connection refused")): - result = asyncio.run(http_tool.run({"url": "http://localhost:9090/metrics"})) + result = await http_tool.run({"url": "http://localhost:9090/metrics"}) assert result.success is False assert "Request failed" in result.error @@ -116,12 +115,12 @@ def test_request_error(http_tool: HttpGetTool) -> None: @pytest.mark.parametrize("status_code", [200, 500]) -def test_response_truncated(http_tool: HttpGetTool, status_code: int) -> None: +async def test_response_truncated(http_tool: HttpGetTool, status_code: int) -> None: from ddev.ai.tools.core.truncation import MAX_CHARS large_body = "x" * (MAX_CHARS + 1000) with patch_httpx(fake_response(status_code, large_body)): - result = asyncio.run(http_tool.run({"url": "http://localhost:9090/metrics"})) + result = await http_tool.run({"url": "http://localhost:9090/metrics"}) assert result.success is True assert result.truncated is True diff --git a/ddev/tests/ai/tools/shell/test_base.py b/ddev/tests/ai/tools/shell/test_base.py index 5d7431239a5e7..3568170b9092d 100644 --- a/ddev/tests/ai/tools/shell/test_base.py +++ b/ddev/tests/ai/tools/shell/test_base.py @@ -79,42 +79,42 @@ def slow_greet_tool() -> SlowGreetTool: # --------------------------------------------------------------------------- -def test_run_command_success(proc): +async def test_run_command_success(proc): with patch_proc(proc): - result = asyncio.run(run_command(["echo", "hello"])) + result = await run_command(["echo", "hello"]) assert result.success is True assert result.data == "hello\n" assert result.truncated is False -def test_run_command_failure_combines_stdout_and_stderr(): +async def test_run_command_failure_combines_stdout_and_stderr(): proc = make_proc(returncode=1, stdout=b"partial\n", stderr=b"error\n") with patch_proc(proc): - result = asyncio.run(run_command(["cmd"])) + result = await run_command(["cmd"]) assert result.success is False assert "partial" in result.data assert "error" in result.data -def test_run_command_failure_stderr_only_when_no_stdout(): +async def test_run_command_failure_stderr_only_when_no_stdout(): proc = make_proc(returncode=1, stdout=b"", stderr=b"fatal error\n") with patch_proc(proc): - result = asyncio.run(run_command(["cmd"])) + result = await run_command(["cmd"]) assert result.success is False and result.data == "fatal error\n" -def test_run_command_ignores_stderr_on_zero_exit(): +async def test_run_command_ignores_stderr_on_zero_exit(): proc = make_proc(returncode=0, stdout=b"out\n", stderr=b"warning\n") with patch_proc(proc): - result = asyncio.run(run_command(["cmd"])) + result = await run_command(["cmd"]) assert result.success is True assert "warning" not in result.data -def test_run_command_stderr_included_when_stdout_empty_on_success(): +async def test_run_command_stderr_included_when_stdout_empty_on_success(): proc = make_proc(returncode=0, stdout=b"", stderr=b"info: initialized\n") with patch_proc(proc): - result = asyncio.run(run_command(["cmd"])) + result = await run_command(["cmd"]) assert result.success is True assert result.data == "info: initialized\n" @@ -127,10 +127,10 @@ def test_run_command_stderr_included_when_stdout_empty_on_success(): (1, b"", b""), ], ) -def test_run_command_empty_output(returncode, stdout, stderr): +async def test_run_command_empty_output(returncode, stdout, stderr): proc = make_proc(returncode=returncode, stdout=stdout, stderr=stderr) with patch_proc(proc): - result = asyncio.run(run_command(["cmd"])) + result = await run_command(["cmd"]) assert result.data == "(no output)" @@ -139,27 +139,27 @@ def test_run_command_empty_output(returncode, stdout, stderr): # --------------------------------------------------------------------------- -def test_run_command_not_found(): +async def test_run_command_not_found(): with patch("asyncio.create_subprocess_exec", side_effect=FileNotFoundError()): - result = asyncio.run(run_command(["nonexistent"])) + result = await run_command(["nonexistent"]) assert result.success is False assert "Command not found" in result.error assert "nonexistent" in result.error -def test_run_command_timeout(): +async def test_run_command_timeout(): proc = make_proc() with patch_proc(proc): with patch("asyncio.wait_for", new=_raise_timeout): - result = asyncio.run(run_command(["sleep", "100"], timeout=5)) + result = await run_command(["sleep", "100"], timeout=5) assert result.success is False assert "5s" in result.error proc.kill.assert_called_once() -def test_run_command_unexpected_exception(): +async def test_run_command_unexpected_exception(): with patch("asyncio.create_subprocess_exec", side_effect=OSError("permission denied")): - result = asyncio.run(run_command(["cmd"])) + result = await run_command(["cmd"]) assert result.success is False assert "OSError" in result.error assert "permission denied" in result.error @@ -170,21 +170,21 @@ def test_run_command_unexpected_exception(): # --------------------------------------------------------------------------- -def test_run_command_truncation(): +async def test_run_command_truncation(): large = ("x" * 80 + "\n") * 700 proc = make_proc(stdout=large.encode()) with patch_proc(proc): - result = asyncio.run(run_command(["cmd"])) + result = await run_command(["cmd"]) assert result.truncated is True assert result.total_size == len(large) assert result.shown_size == len(result.data) assert result.hint is not None -def test_run_command_no_truncation_at_limit(): +async def test_run_command_no_truncation_at_limit(): proc = make_proc(stdout=("x" * MAX_CHARS).encode()) with patch_proc(proc): - result = asyncio.run(run_command(["cmd"])) + result = await run_command(["cmd"]) assert result.truncated is False assert result.total_size is None assert result.hint is None @@ -200,10 +200,10 @@ def test_cmd_tool_timeouts(greet_tool: GreetTool, slow_greet_tool: SlowGreetTool assert SlowGreetTool.timeout == 60 # custom timeout -def test_cmd_tool_dispatches_with_correct_timeout(greet_tool: GreetTool, slow_greet_tool: SlowGreetTool): +async def test_cmd_tool_dispatches_with_correct_timeout(greet_tool: GreetTool, slow_greet_tool: SlowGreetTool): for tool, expected_timeout in [(greet_tool, 10), (slow_greet_tool, 60)]: with patch( "ddev.ai.tools.shell.base.run_command", new=AsyncMock(return_value=ToolResult(success=True)) ) as mock_run: - asyncio.run(tool.run({"name": "world"})) + await tool.run({"name": "world"}) mock_run.assert_called_once_with(["echo", "hello world"], timeout=expected_timeout) diff --git a/ddev/tests/ai/tools/shell/test_tools.py b/ddev/tests/ai/tools/shell/test_tools.py index 81fcb45d3d3b1..05084acc97e9e 100644 --- a/ddev/tests/ai/tools/shell/test_tools.py +++ b/ddev/tests/ai/tools/shell/test_tools.py @@ -1,7 +1,6 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import asyncio from unittest.mock import AsyncMock, patch import pytest @@ -66,12 +65,12 @@ def test_grep_cmd_pattern_and_path_placement(grep_tool: GrepTool): assert cmd[-1] == "/my dir/sub dir" -def test_grep_no_matches_returns_success(grep_tool: GrepTool): +async def test_grep_no_matches_returns_success(grep_tool: GrepTool): from ddev.ai.tools.core.types import ToolResult no_match_result = ToolResult(success=False, data="(no output)", error=None) with patch("ddev.ai.tools.shell.grep.run_command", new=AsyncMock(return_value=no_match_result)): - result = asyncio.run(grep_tool(GrepInput(pattern="nomatch", path="/tmp"))) + result = await grep_tool(GrepInput(pattern="nomatch", path="/tmp")) assert result.success is True assert result.data == "(no output)" From fe52ed0102767ccc1b71d6d2cabf7b2fe66d76ab Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Thu, 9 Apr 2026 10:33:28 +0200 Subject: [PATCH 25/44] ReAct layer (#23083) * Modified Agent layer to encapsule the anthropic-specific logic * Reorder agent/ and created react/ with the ReActProcess and its tests * Reduce two zips into one * Fix some little bugs * Surround ReActCallback calls with try/catch * Fix comments and remove max iterations in react loop * Change AgentProtocol to BaseAgent to handle history logic --- ddev/src/ddev/ai/agent/base.py | 36 +++ ddev/src/ddev/ai/agent/client.py | 146 ++++----- ddev/src/ddev/ai/agent/exceptions.py | 8 +- ddev/src/ddev/ai/agent/types.py | 75 +++++ ddev/src/ddev/ai/react/__init__.py | 3 + ddev/src/ddev/ai/react/process.py | 156 +++++++++ ddev/tests/ai/agent/test_client.py | 52 ++- ddev/tests/ai/react/__init__.py | 3 + ddev/tests/ai/react/test_process.py | 455 +++++++++++++++++++++++++++ 9 files changed, 830 insertions(+), 104 deletions(-) create mode 100644 ddev/src/ddev/ai/agent/base.py create mode 100644 ddev/src/ddev/ai/agent/types.py create mode 100644 ddev/src/ddev/ai/react/__init__.py create mode 100644 ddev/src/ddev/ai/react/process.py create mode 100644 ddev/tests/ai/react/__init__.py create mode 100644 ddev/tests/ai/react/test_process.py diff --git a/ddev/src/ddev/ai/agent/base.py b/ddev/src/ddev/ai/agent/base.py new file mode 100644 index 0000000000000..7546a19747960 --- /dev/null +++ b/ddev/src/ddev/ai/agent/base.py @@ -0,0 +1,36 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from abc import ABC, abstractmethod +from copy import deepcopy + +from ddev.ai.agent.types import AgentResponse, ToolResultMessage + + +class BaseAgent[TMessage](ABC): + """Abstract base class for all agent implementations. + + Provides shared, provider-agnostic history management. The message type + TMessage is supplied by each concrete provider (e.g. MessageParam for Anthropic). + Subclasses must implement send(). + """ + + def __init__(self) -> None: + self._history: list[TMessage] = [] + + @property + def history(self) -> list[TMessage]: + """Read-only snapshot of the conversation history.""" + return deepcopy(self._history) + + def reset(self) -> None: + """Clear conversation history to start a new conversation.""" + self._history = [] + + @abstractmethod + async def send( + self, + content: str | list[ToolResultMessage], + allowed_tools: list[str] | None = None, + ) -> AgentResponse: ... diff --git a/ddev/src/ddev/ai/agent/client.py b/ddev/src/ddev/ai/agent/client.py index d576429015cef..add571ed03491 100644 --- a/ddev/src/ddev/ai/agent/client.py +++ b/ddev/src/ddev/ai/agent/client.py @@ -2,87 +2,22 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from copy import deepcopy -from dataclasses import dataclass -from enum import StrEnum -from typing import Any, Final +from typing import Final import anthropic from anthropic.types import MessageParam, ToolParam, ToolResultBlockParam +from ddev.ai.agent.base import BaseAgent +from ddev.ai.agent.exceptions import AgentAPIError, AgentConnectionError, AgentError, AgentRateLimitError +from ddev.ai.agent.types import AgentResponse, ContextUsage, StopReason, TokenUsage, ToolCall, ToolResultMessage from ddev.ai.tools.core.registry import ToolRegistry -from .exceptions import ( - AgentAPIError, - AgentConnectionError, - AgentError, - AgentRateLimitError, -) - DEFAULT_MODEL: Final[str] = "claude-sonnet-4-6" DEFAULT_MAX_TOKENS: Final[int] = 8192 # max tokens per response ALLOWED_TOOL_CALLERS: Final = ["code_execution_20260120"] -class StopReason(StrEnum): - """Maps Anthropic API stop_reason strings to a typed enum.""" - - END_TURN = "end_turn" - MAX_TOKENS = "max_tokens" - STOP_SEQUENCE = "stop_sequence" - TOOL_USE = "tool_use" - PAUSE_TURN = "pause_turn" - REFUSAL = "refusal" - - -@dataclass(frozen=True) -class ToolCall: - """A single tool invocation requested by the model.""" - - id: str - name: str - input: dict[str, Any] - - -@dataclass(frozen=True) -class ContextUsage: - """Context window accounting for a single API call.""" - - window_size: int - used_tokens: int - - @property - def context_pct(self) -> float: - return self.used_tokens / self.window_size * 100 - - @property - def remaining_tokens(self) -> int: - return self.window_size - self.used_tokens - - -@dataclass(frozen=True) -class TokenUsage: - """Token accounting from a single API call.""" - - input_tokens: int # tokens sent to the model (system_prompt + history) - output_tokens: int # tokens the model generated - cache_read_input_tokens: int # tokens read from prompt cache - cache_creation_input_tokens: int # tokens written to prompt cache - context: ContextUsage - - -@dataclass(frozen=True) -class AgentResponse: - """The complete response from a single AnthropicAgent.send() call. - Adds useful metadata to the response of the Anthropic API.""" - - stop_reason: StopReason - text: str - tool_calls: list[ToolCall] - usage: TokenUsage - - -class AnthropicAgent: +class AnthropicAgent(BaseAgent[MessageParam]): """A wrapper around the Anthropic API that provides a simple interface for interacting with agents.""" def __init__( @@ -106,6 +41,7 @@ def __init__( programmatic_tool_calling: Whether to allow programmatic tool calling. """ + super().__init__() self._client = client self._tools = tools self._system_prompt = system_prompt @@ -113,21 +49,21 @@ def __init__( self._model = model self._max_tokens = max_tokens self._programmatic_tool_calling = programmatic_tool_calling - self._history: list[MessageParam] = [] self._context_window: int | None = None - @property - def history(self) -> list[MessageParam]: - """Read-only snapshot of the conversation history.""" - return deepcopy(self._history) - - def reset(self) -> None: - """Clear conversation history to start a new conversation.""" - self._history = [] - async def _get_context_window(self) -> int: if self._context_window is None: - info = await self._client.models.retrieve(self._model) + try: + info = await self._client.models.retrieve(self._model) + except anthropic.APIConnectionError as e: + raise AgentConnectionError(f"Connection failed: {e}") from e + except anthropic.RateLimitError as e: + raise AgentRateLimitError(f"Rate limit exceeded: {e}") from e + except anthropic.APIStatusError as e: + raise AgentAPIError(e.status_code, e.message) from e + except anthropic.APIResponseValidationError as e: + raise AgentError(f"Response validation failed: {e}") from e + self._context_window = info.max_input_tokens return self._context_window @@ -141,9 +77,43 @@ def _get_tool_definitions(self, allowed_tools: list[str] | None) -> list[ToolPar definitions = [{**d, "allowed_callers": ALLOWED_TOOL_CALLERS} for d in definitions] return definitions + def _map_stop_reason(self, raw: str) -> StopReason: + """Map a raw Anthropic stop_reason string to the generic StopReason enum.""" + # pause_turn gets an explicit check to provide a more informative message than "Unknown stop_reason" + if raw == "pause_turn": + raise AgentError("pause_turn is not supported in batch mode") from None + mapping = { + "end_turn": StopReason.END_TURN, + "tool_use": StopReason.TOOL_USE, + "max_tokens": StopReason.MAX_TOKENS, + "stop_sequence": StopReason.OTHER, + "refusal": StopReason.OTHER, + } + if raw not in mapping: + raise AgentError(f"Unknown stop_reason: {raw!r}") from None + return mapping[raw] + + def _to_tool_result_params(self, messages: list[ToolResultMessage]) -> list[ToolResultBlockParam]: + """Convert model-agnostic ToolResultMessages to Anthropic SDK ToolResultBlockParams.""" + return [ + { + "type": "tool_result", + "tool_use_id": msg.tool_call_id, + "is_error": not msg.result.success, + **( + {"content": msg.result.data} + if msg.result.data is not None + else {"content": msg.result.error or "(unknown error)"} + if not msg.result.success + else {} + ), + } + for msg in messages + ] + async def send( self, - content: str | list[ToolResultBlockParam], + content: str | list[ToolResultMessage], allowed_tools: list[str] | None = None, ) -> AgentResponse: """Send a message to the agent and return the response. @@ -155,7 +125,10 @@ async def send( """ tool_defs = self._get_tool_definitions(allowed_tools) - user_msg: MessageParam = {"role": "user", "content": content} + api_content: str | list[ToolResultBlockParam] = ( + self._to_tool_result_params(content) if isinstance(content, list) else content + ) + user_msg: MessageParam = {"role": "user", "content": api_content} messages = [*self._history, user_msg] try: @@ -179,10 +152,7 @@ async def send( if response.stop_reason is None: raise AgentError("Received null stop_reason from API") - try: - stop_reason = StopReason(response.stop_reason) - except ValueError as e: - raise AgentError(f"Unknown stop_reason: {response.stop_reason!r}") from e + stop_reason = self._map_stop_reason(response.stop_reason) text_parts: list[str] = [] tool_calls: list[ToolCall] = [] @@ -203,7 +173,7 @@ async def send( output_tokens=response.usage.output_tokens, cache_read_input_tokens=cache_read, cache_creation_input_tokens=cache_creation, - context=ContextUsage(window_size=await self._get_context_window(), used_tokens=used_tokens), + context_usage=ContextUsage(window_size=await self._get_context_window(), used_tokens=used_tokens), ) agent_response = AgentResponse( diff --git a/ddev/src/ddev/ai/agent/exceptions.py b/ddev/src/ddev/ai/agent/exceptions.py index d0d25d3665239..9f19519fa85eb 100644 --- a/ddev/src/ddev/ai/agent/exceptions.py +++ b/ddev/src/ddev/ai/agent/exceptions.py @@ -4,22 +4,16 @@ class AgentError(Exception): - """Base class for all errors raised by AnthropicAgent.""" - - pass + """Base class for all errors raised by an agent.""" class AgentConnectionError(AgentError): """Network failure — the API was unreachable.""" - pass - class AgentRateLimitError(AgentError): """Rate limit hit — the request may be retried after a delay.""" - pass - class AgentAPIError(AgentError): """The API returned an error status code.""" diff --git a/ddev/src/ddev/ai/agent/types.py b/ddev/src/ddev/ai/agent/types.py new file mode 100644 index 0000000000000..84e380aea77d1 --- /dev/null +++ b/ddev/src/ddev/ai/agent/types.py @@ -0,0 +1,75 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +"""Wire types for the agent layer: enums, dataclasses, and response shapes.""" + +from dataclasses import dataclass +from enum import StrEnum +from typing import Any + +from ddev.ai.tools.core.types import ToolResult + + +class StopReason(StrEnum): + """Generic stop reasons for agent responses, independent of any provider.""" + + END_TURN = "end_turn" + TOOL_USE = "tool_use" + MAX_TOKENS = "max_tokens" + OTHER = "other" + + +@dataclass(frozen=True) +class ToolCall: + """A single tool invocation requested by the model.""" + + id: str + name: str + input: dict[str, Any] + + +@dataclass(frozen=True) +class ToolResultMessage: + """Wraps a tool result to be sent back to the agent, keyed by the originating tool call ID.""" + + tool_call_id: str # matches ToolCall.id + result: ToolResult + + +@dataclass(frozen=True) +class ContextUsage: + """Context window accounting for a single API call.""" + + window_size: int + used_tokens: int + + @property + def context_pct(self) -> float: + return self.used_tokens / self.window_size * 100 + + @property + def remaining_tokens(self) -> int: + return self.window_size - self.used_tokens + + +@dataclass(frozen=True) +class TokenUsage: + """Token accounting from a single API call.""" + + input_tokens: int # tokens sent to the model (system_prompt + history) + output_tokens: int # tokens the model generated + cache_read_input_tokens: int # tokens read from prompt cache + cache_creation_input_tokens: int # tokens written to prompt cache + context_usage: ContextUsage | None = None # None only for agents that don't provide context tracking + + +@dataclass(frozen=True) +class AgentResponse: + """The complete response from a single agent.send() call. + Adds useful metadata to the response of the agent.""" + + stop_reason: StopReason + text: str + tool_calls: list[ToolCall] + usage: TokenUsage diff --git a/ddev/src/ddev/ai/react/__init__.py b/ddev/src/ddev/ai/react/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/react/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/react/process.py b/ddev/src/ddev/ai/react/process.py new file mode 100644 index 0000000000000..e48a88dc14e0a --- /dev/null +++ b/ddev/src/ddev/ai/react/process.py @@ -0,0 +1,156 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import asyncio +from dataclasses import dataclass +from typing import Any, Protocol + +from ddev.ai.agent.base import BaseAgent +from ddev.ai.agent.exceptions import AgentError +from ddev.ai.agent.types import AgentResponse, ContextUsage, StopReason, ToolCall, ToolResultMessage +from ddev.ai.tools.core.registry import ToolRegistry +from ddev.ai.tools.core.types import ToolResult + + +@dataclass(frozen=True) +class ReActResult: + """Immutable summary of a completed ReAct loop run.""" + + final_response: AgentResponse + iterations: int + total_input_tokens: int # sum across all iterations + total_output_tokens: int # sum across all iterations + context_usage: ContextUsage | None # promoted from final_response.usage.context_usage + + +class ReActCallback(Protocol): + """Observer interface for ReActProcess lifecycle events.""" + + async def on_agent_response(self, response: AgentResponse, iteration: int) -> None: + """Called after every agent.send() returns, including the first.""" + ... + + async def on_tool_call(self, tool_call: ToolCall, result: ToolResult, iteration: int) -> None: + """Called once per (tool_call, result) pair after all tools in a batch execute.""" + ... + + async def on_complete(self, result: ReActResult) -> None: + """Called when the loop exits cleanly with a ReActResult.""" + ... + + async def on_error(self, error: BaseException) -> None: + """Called when the loop aborts — covers AgentError, KeyboardInterrupt, and CancelledError. + The exception is always re-raised after this returns.""" + ... + + +class ReActProcess: + """ + Manages the ReAct (Reason + Act) loop for a single task. + + Sends a prompt to an agent, executes any tool calls in parallel, + feeds results back, and repeats until the agent stops requesting tools. + """ + + def __init__( + self, + agent: BaseAgent[Any], + tool_registry: ToolRegistry, + callbacks: list[ReActCallback] | None = None, + ) -> None: + """ + Args: + agent: A BaseAgent subclass (e.g. AnthropicAgent). + tool_registry: Registry of tools available in this loop. + callbacks: Optional observers. Empty list means no events are fired. + """ + self._agent = agent + self._tool_registry = tool_registry + self._callbacks: list[ReActCallback] = callbacks or [] + + async def start(self, prompt: str, allowed_tools: list[str] | None = None) -> ReActResult: + """ + Run the ReAct loop for a single task. + + Args: + prompt: The initial user prompt to send to the agent. + allowed_tools: Optional subset of tools the agent may call in this run. None means all. + + Returns: + A ReActResult summarising the final response, token counts, and iteration count. + + Raises: + Every exception is forwarded after notifying callbacks. + """ + try: + response = await self._agent.send(prompt, allowed_tools) + iterations = 1 + total_input = response.usage.input_tokens + total_output = response.usage.output_tokens + + for cb in self._callbacks: + try: + await cb.on_agent_response(response, iterations) + except Exception: + pass # in the future we should log this error + + # No iteration cap — this is an interactive CLI tool; the user can Ctrl+C to stop. + while response.stop_reason == StopReason.TOOL_USE: + if not response.tool_calls: + raise AgentError("Agent returned stop_reason=TOOL_USE with no tool calls") + + raw_results = await asyncio.gather( + *[self._tool_registry.run(tc.name, tc.input) for tc in response.tool_calls], + return_exceptions=True, + ) + tool_results: list[ToolResult] = [ + r if isinstance(r, ToolResult) else ToolResult(success=False, error=f"{type(r).__name__}: {r}") + for r in raw_results + ] + + tool_call_results = list(zip(response.tool_calls, tool_results, strict=True)) + + for tc, result in tool_call_results: + for cb in self._callbacks: + try: + await cb.on_tool_call(tc, result, iterations) + except Exception: + pass + + messages = [ToolResultMessage(tool_call_id=tc.id, result=result) for tc, result in tool_call_results] + + response = await self._agent.send(messages, allowed_tools) + iterations += 1 + total_input += response.usage.input_tokens + total_output += response.usage.output_tokens + + for cb in self._callbacks: + try: + await cb.on_agent_response(response, iterations) + except Exception: + pass + + react_result = ReActResult( + final_response=response, + iterations=iterations, + total_input_tokens=total_input, + total_output_tokens=total_output, + context_usage=response.usage.context_usage, + ) + + for cb in self._callbacks: + try: + await cb.on_complete(react_result) + except Exception: + pass + + return react_result + + except BaseException as e: + for cb in self._callbacks: + try: + await cb.on_error(e) + except Exception: + pass + raise diff --git a/ddev/tests/ai/agent/test_client.py b/ddev/tests/ai/agent/test_client.py index f4d1b9f5e8c96..ae9578c5c87ad 100644 --- a/ddev/tests/ai/agent/test_client.py +++ b/ddev/tests/ai/agent/test_client.py @@ -8,13 +8,9 @@ import anthropic import pytest -from ddev.ai.agent.client import AnthropicAgent, StopReason -from ddev.ai.agent.exceptions import ( - AgentAPIError, - AgentConnectionError, - AgentError, - AgentRateLimitError, -) +from ddev.ai.agent.client import AnthropicAgent +from ddev.ai.agent.exceptions import AgentAPIError, AgentConnectionError, AgentError, AgentRateLimitError +from ddev.ai.agent.types import StopReason, ToolResultMessage from ddev.ai.tools.core.registry import ToolRegistry from ddev.ai.tools.core.types import ToolResult @@ -343,7 +339,7 @@ async def test_context_usage_fields() -> None: result = await agent.send("Hi") - ctx = result.usage.context + ctx = result.usage.context_usage assert ctx.window_size == FAKE_CONTEXT_WINDOW assert ctx.used_tokens == 1700 # 1000 + 500 + 200 assert ctx.context_pct == pytest.approx(1700 / FAKE_CONTEXT_WINDOW * 100) @@ -386,7 +382,7 @@ async def test_multi_turn_history_grows_correctly() -> None: assert first.stop_reason is StopReason.TOOL_USE assert len(agent.history) == 2 - tool_results = [{"type": "tool_result", "tool_use_id": "toolu_01", "content": "result"}] + tool_results = [ToolResultMessage(tool_call_id="toolu_01", result=ToolResult(success=True, data="result"))] second = await agent.send(tool_results) assert second.stop_reason is StopReason.END_TURN assert len(agent.history) == 4 @@ -425,6 +421,44 @@ async def test_reset_clears_history() -> None: assert agent.history == [] +# --------------------------------------------------------------------------- +# pause_turn raises AgentError +# --------------------------------------------------------------------------- + + +async def test_pause_turn_raises_agent_error() -> None: + resp = make_response("pause_turn", []) + agent, _ = make_agent(mock_response=resp) + + with pytest.raises(AgentError): + await agent.send("Hi") + + assert agent.history == [] + + +# --------------------------------------------------------------------------- +# refusal and stop_sequence map to StopReason.OTHER +# --------------------------------------------------------------------------- + + +async def test_refusal_maps_to_other() -> None: + resp = make_response("refusal", [make_text_block("I can't do that")]) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Do something bad") + + assert result.stop_reason is StopReason.OTHER + + +async def test_stop_sequence_maps_to_other() -> None: + resp = make_response("stop_sequence", [make_text_block("Stopped.")]) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Hi") + + assert result.stop_reason is StopReason.OTHER + + # --------------------------------------------------------------------------- # Error mid-conversation leaves history unchanged # --------------------------------------------------------------------------- diff --git a/ddev/tests/ai/react/__init__.py b/ddev/tests/ai/react/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/react/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/react/test_process.py b/ddev/tests/ai/react/test_process.py new file mode 100644 index 0000000000000..664e25c5505a6 --- /dev/null +++ b/ddev/tests/ai/react/test_process.py @@ -0,0 +1,455 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import asyncio +from typing import Any + +import pytest + +from ddev.ai.agent.base import BaseAgent +from ddev.ai.agent.exceptions import AgentConnectionError +from ddev.ai.agent.types import AgentResponse, ContextUsage, StopReason, TokenUsage, ToolCall, ToolResultMessage +from ddev.ai.react.process import ReActCallback, ReActProcess, ReActResult +from ddev.ai.tools.core.types import ToolResult + +# --------------------------------------------------------------------------- +# Mock helpers +# --------------------------------------------------------------------------- + + +class MockAgent(BaseAgent[Any]): + """Minimal BaseAgent implementation that replays a fixed list of responses.""" + + def __init__(self, responses: list[AgentResponse]) -> None: + super().__init__() + self._responses = iter(responses) + self.send_calls: list[str | list[ToolResultMessage]] = [] + + async def send( + self, + content: str | list[ToolResultMessage], + allowed_tools: list[str] | None = None, + ) -> AgentResponse: + self.send_calls.append(content) + return next(self._responses) + + +class MockToolRegistry: + """Minimal tool registry that always returns a configurable ToolResult.""" + + def __init__(self, result: ToolResult | None = None) -> None: + self._result = result or ToolResult(success=True, data="ok") + self.run_calls: list[tuple[str, dict]] = [] + + async def run(self, name: str, raw: dict[str, object]) -> ToolResult: + self.run_calls.append((name, raw)) + return self._result + + +class RaisingToolRegistry: + """Registry that always raises a given exception from run().""" + + def __init__(self, exc: BaseException) -> None: + self._exc = exc + self.run_calls: list[tuple[str, dict]] = [] + + async def run(self, name: str, raw: dict[str, object]) -> ToolResult: + self.run_calls.append((name, raw)) + raise self._exc + + +class PerToolRegistry: + """Registry that dispatches per tool name, raising or returning per configured behavior.""" + + def __init__(self, behaviors: dict[str, ToolResult | BaseException]) -> None: + self._behaviors = behaviors + self.run_calls: list[tuple[str, dict]] = [] + + async def run(self, name: str, raw: dict[str, object]) -> ToolResult: + self.run_calls.append((name, raw)) + behavior = self._behaviors[name] + if isinstance(behavior, BaseException): + raise behavior + return behavior + + +class MockCallback: + """Records all lifecycle events emitted by ReActProcess.""" + + def __init__(self) -> None: + self.agent_responses: list[tuple[AgentResponse, int]] = [] + self.tool_calls_seen: list[tuple[ToolCall, ToolResult, int]] = [] + self.complete_results: list[ReActResult] = [] + self.errors: list[Exception] = [] + + async def on_agent_response(self, response: AgentResponse, iteration: int) -> None: + self.agent_responses.append((response, iteration)) + + async def on_tool_call(self, tool_call: ToolCall, result: ToolResult, iteration: int) -> None: + self.tool_calls_seen.append((tool_call, result, iteration)) + + async def on_complete(self, result: ReActResult) -> None: + self.complete_results.append(result) + + async def on_error(self, error: Exception) -> None: + self.errors.append(error) + + +def make_response( + stop_reason: StopReason, + tool_calls: list[ToolCall] | None = None, + input_tokens: int = 10, + output_tokens: int = 5, + context_usage: ContextUsage | None = None, +) -> AgentResponse: + return AgentResponse( + stop_reason=stop_reason, + text="", + tool_calls=tool_calls or [], + usage=TokenUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_read_input_tokens=0, + cache_creation_input_tokens=0, + context_usage=context_usage, + ), + ) + + +def make_tool_call( + call_id: str = "tc_01", name: str = "read_file", tool_input: dict[str, Any] | None = None +) -> ToolCall: + return ToolCall(id=call_id, name=name, input=tool_input or {}) + + +def make_process( + agent: MockAgent, + registry: MockToolRegistry | None = None, + callbacks: list[ReActCallback] | None = None, +) -> ReActProcess: + return ReActProcess( + agent=agent, + tool_registry=registry or MockToolRegistry(), + callbacks=callbacks, + ) + + +# --------------------------------------------------------------------------- +# Stop reasons — parametrized single-response cases +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("stop_reason", [StopReason.END_TURN, StopReason.MAX_TOKENS, StopReason.OTHER]) +async def test_stop_reason_single_response(stop_reason) -> None: + agent = MockAgent([make_response(stop_reason)]) + + result = await make_process(agent).start("Hi") + + assert result.final_response.stop_reason == stop_reason + assert result.iterations == 1 + assert len(agent.send_calls) == 1 + assert agent.send_calls[0] == "Hi" + + +# --------------------------------------------------------------------------- +# Single tool call +# --------------------------------------------------------------------------- + + +async def test_single_tool_call_executes_tool_and_returns() -> None: + tc = make_tool_call("tc_01", "read_file") + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN), + ] + registry = MockToolRegistry() + agent = MockAgent(responses) + + result = await make_process(agent, registry=registry).start("Do something") + + assert result.final_response.stop_reason == StopReason.END_TURN + assert result.iterations == 2 + assert len(registry.run_calls) == 1 + assert registry.run_calls[0][0] == "read_file" + assert len(agent.send_calls) == 2 + assert agent.send_calls[0] == "Do something" + assert isinstance(agent.send_calls[1], list) + assert agent.send_calls[1][0].tool_call_id == "tc_01" + assert agent.send_calls[1][0].result.data == "ok" + + +# --------------------------------------------------------------------------- +# Multi-tool parallel dispatch +# --------------------------------------------------------------------------- + + +async def test_multi_tool_parallel_dispatches_all() -> None: + tool_calls = [ + make_tool_call("tc_01", "a"), + make_tool_call("tc_02", "b"), + make_tool_call("tc_03", "c"), + ] + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=tool_calls), + make_response(StopReason.END_TURN), + ] + registry = MockToolRegistry() + agent = MockAgent(responses) + + await make_process(agent, registry=registry).start("Do three things") + + assert len(registry.run_calls) == 3 + assert {name for name, _ in registry.run_calls} == {"a", "b", "c"} + assert len(agent.send_calls[1]) == 3 + + +# --------------------------------------------------------------------------- +# Tool exception resilience +# --------------------------------------------------------------------------- + + +async def test_tool_exception_loop_continues_with_failure_result() -> None: + """(a) A raising tool must not abort the loop. (b) Its ToolResultMessage must have success=False.""" + tc = make_tool_call("tc_01", "read_file") + agent = MockAgent( + [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN), + ] + ) + + result = await ReActProcess( + agent=agent, + tool_registry=RaisingToolRegistry(RuntimeError("disk error")), + ).start("Do something") + + assert result.iterations == 2 + assert result.final_response.stop_reason == StopReason.END_TURN + sent_back = agent.send_calls[1] + assert isinstance(sent_back, list) + assert sent_back[0].result.success is False + + +async def test_tool_exception_on_tool_call_callback_fires_with_error_result() -> None: + """(c) on_tool_call must fire even when the tool raised, carrying the failure ToolResult.""" + tc = make_tool_call("tc_01", "read_file") + agent = MockAgent( + [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN), + ] + ) + callback = MockCallback() + + await ReActProcess( + agent=agent, + tool_registry=RaisingToolRegistry(ValueError("oops")), + callbacks=[callback], + ).start("x") + + assert len(callback.tool_calls_seen) == 1 + _, error_result, _ = callback.tool_calls_seen[0] + assert error_result.success is False + + +@pytest.mark.parametrize( + "exc,expected_error", + [ + (RuntimeError("disk error"), "RuntimeError: disk error"), + (ValueError("bad input"), "ValueError: bad input"), + (OSError("file not found"), "OSError: file not found"), + ], +) +async def test_tool_exception_error_message_format(exc: BaseException, expected_error: str) -> None: + """Error string in the failure result must be formatted as 'ExceptionType: message'.""" + tc = make_tool_call() + agent = MockAgent( + [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN), + ] + ) + + await ReActProcess( + agent=agent, + tool_registry=RaisingToolRegistry(exc), + ).start("x") + + sent_back: list[ToolResultMessage] = agent.send_calls[1] + assert sent_back[0].result.error == expected_error + + +async def test_partial_batch_failure_only_affects_raising_tool() -> None: + """In a multi-tool batch, only the raising tool gets success=False; successful tools are unaffected.""" + tc_ok = make_tool_call("tc_01", "read_file") + tc_bad = make_tool_call("tc_02", "write_file") + agent = MockAgent( + [ + make_response(StopReason.TOOL_USE, tool_calls=[tc_ok, tc_bad]), + make_response(StopReason.END_TURN), + ] + ) + registry = PerToolRegistry( + { + "read_file": ToolResult(success=True, data="contents"), + "write_file": RuntimeError("permission denied"), + } + ) + + result = await ReActProcess(agent=agent, tool_registry=registry).start("Do both") + + assert result.iterations == 2 + sent_back: list[ToolResultMessage] = agent.send_calls[1] + assert len(sent_back) == 2 + results = {msg.tool_call_id: msg.result for msg in sent_back} + assert results["tc_01"].success is True + assert results["tc_01"].data == "contents" + assert results["tc_02"].success is False + assert "RuntimeError" in (results["tc_02"].error or "") + + +# --------------------------------------------------------------------------- +# Callbacks fired correctly +# --------------------------------------------------------------------------- + + +async def test_callbacks_invoked_correct_counts() -> None: + tool_calls = [make_tool_call("tc_01"), make_tool_call("tc_02", "grep")] + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=tool_calls), + make_response(StopReason.END_TURN), + ] + expected_result = ToolResult(success=True, data="ok") + registry = MockToolRegistry(result=expected_result) + callback = MockCallback() + agent = MockAgent(responses) + + result = await make_process(agent, registry=registry, callbacks=[callback]).start("Run tools") + + assert len(callback.agent_responses) == 2 + assert callback.agent_responses[0][1] == 1 + assert callback.agent_responses[1][1] == 2 + assert len(callback.tool_calls_seen) == 2 + assert all(iteration == 1 for _, _, iteration in callback.tool_calls_seen) + assert callback.tool_calls_seen[0][0] is tool_calls[0] + assert callback.tool_calls_seen[1][0] is tool_calls[1] + assert callback.tool_calls_seen[0][1] is expected_result + assert callback.tool_calls_seen[1][1] is expected_result + assert len(callback.complete_results) == 1 + assert callback.complete_results[0] is result + assert len(callback.errors) == 0 + + +# --------------------------------------------------------------------------- +# Error path +# --------------------------------------------------------------------------- + + +class ErrorAgent(BaseAgent[Any]): + async def send( + self, content: str | list[ToolResultMessage], allowed_tools: list[str] | None = None + ) -> AgentResponse: + raise AgentConnectionError("network down") + + +async def test_agent_error_notifies_and_reraises() -> None: + callback = MockCallback() + process = ReActProcess( + agent=ErrorAgent(), + tool_registry=MockToolRegistry(), + callbacks=[callback], + ) + + with pytest.raises(AgentConnectionError): + await process.start("Anything") + + assert len(callback.errors) == 1 + assert isinstance(callback.errors[0], AgentConnectionError) + assert len(callback.complete_results) == 0 + assert len(callback.agent_responses) == 0 + + +class InterruptAgent(BaseAgent[Any]): + async def send( + self, content: str | list[ToolResultMessage], allowed_tools: list[str] | None = None + ) -> AgentResponse: + raise KeyboardInterrupt + + +async def test_keyboard_interrupt_notifies_and_reraises() -> None: + callback = MockCallback() + process = ReActProcess( + agent=InterruptAgent(), + tool_registry=MockToolRegistry(), + callbacks=[callback], + ) + + with pytest.raises(KeyboardInterrupt): + await process.start("Anything") + + assert len(callback.errors) == 1 + assert isinstance(callback.errors[0], KeyboardInterrupt) + assert len(callback.complete_results) == 0 + + +async def test_cancelled_error_notifies_and_reraises() -> None: + class CancelledAgent(BaseAgent[Any]): + async def send( + self, content: str | list[ToolResultMessage], allowed_tools: list[str] | None = None + ) -> AgentResponse: + raise asyncio.CancelledError + + callback = MockCallback() + process = ReActProcess( + agent=CancelledAgent(), + tool_registry=MockToolRegistry(), + callbacks=[callback], + ) + + with pytest.raises(asyncio.CancelledError): + await process.start("Anything") + + assert len(callback.errors) == 1 + assert isinstance(callback.errors[0], asyncio.CancelledError) + assert len(callback.complete_results) == 0 + + +# --------------------------------------------------------------------------- +# Token accumulation +# --------------------------------------------------------------------------- + + +async def test_total_tokens_summed_across_iterations() -> None: + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[make_tool_call()], input_tokens=100, output_tokens=50), + make_response(StopReason.END_TURN, input_tokens=200, output_tokens=80), + ] + agent = MockAgent(responses) + + result = await make_process(agent).start("Task") + + assert result.total_input_tokens == 300 + assert result.total_output_tokens == 130 + assert result.iterations == 2 + + +# --------------------------------------------------------------------------- +# Context usage propagation — parametrized None vs present +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "context_usage", + [ + None, + ContextUsage(window_size=200_000, used_tokens=50_000), + ], +) +async def test_context_usage_propagated(context_usage: ContextUsage | None) -> None: + agent = MockAgent([make_response(StopReason.END_TURN, context_usage=context_usage)]) + + result = await make_process(agent).start("Hi") + + assert result.context_usage is context_usage + assert result.final_response.usage.context_usage is context_usage From 3553187af127214e58d187137d7c173c7e94220e Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Fri, 10 Apr 2026 12:12:29 +0200 Subject: [PATCH 26/44] CallbackSet API (#23243) * change client.py to anthropic_client.py * Add CallbackSet that acts as a decorator * Add callback tests * Change functions in test for pytest fixtures --- .../agent/{client.py => anthropic_client.py} | 0 ddev/src/ddev/ai/react/callbacks.py | 100 ++++++++++ ddev/src/ddev/ai/react/process.py | 80 ++------ ddev/src/ddev/ai/react/types.py | 18 ++ ...est_client.py => test_anthropic_client.py} | 2 +- ddev/tests/ai/react/test_callbacks.py | 184 ++++++++++++++++++ ddev/tests/ai/react/test_process.py | 127 ++++++------ 7 files changed, 392 insertions(+), 119 deletions(-) rename ddev/src/ddev/ai/agent/{client.py => anthropic_client.py} (100%) create mode 100644 ddev/src/ddev/ai/react/callbacks.py create mode 100644 ddev/src/ddev/ai/react/types.py rename ddev/tests/ai/agent/{test_client.py => test_anthropic_client.py} (99%) create mode 100644 ddev/tests/ai/react/test_callbacks.py diff --git a/ddev/src/ddev/ai/agent/client.py b/ddev/src/ddev/ai/agent/anthropic_client.py similarity index 100% rename from ddev/src/ddev/ai/agent/client.py rename to ddev/src/ddev/ai/agent/anthropic_client.py diff --git a/ddev/src/ddev/ai/react/callbacks.py b/ddev/src/ddev/ai/react/callbacks.py new file mode 100644 index 0000000000000..59ef6687ff67d --- /dev/null +++ b/ddev/src/ddev/ai/react/callbacks.py @@ -0,0 +1,100 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from typing import Protocol + +from ddev.ai.agent.types import AgentResponse, ToolCall +from ddev.ai.react.types import ReActResult +from ddev.ai.tools.core.types import ToolResult + + +class OnAgentResponseCallback(Protocol): + """Called after every agent.send() returns, including the first.""" + + async def __call__(self, response: AgentResponse, iteration: int) -> None: ... + + +class OnToolCallCallback(Protocol): + """Called once per (tool_call, result) pair after all tools in a batch execute.""" + + async def __call__(self, tool_call: ToolCall, result: ToolResult, iteration: int) -> None: ... + + +class OnCompleteCallback(Protocol): + """Called when the loop exits cleanly.""" + + async def __call__(self, result: ReActResult) -> None: ... + + +class OnErrorCallback(Protocol): + """Called when the loop aborts. The exception is always re-raised after this returns.""" + + async def __call__(self, error: BaseException) -> None: ... + + +class CallbackSet: + """Decorator-based registry for ReAct lifecycle event handlers. + + Usage:: + + cb = CallbackSet() + + @cb.on_complete + async def log_done(result: ReActResult) -> None: + print(f"Done in {result.iterations} iterations") + """ + + def __init__(self) -> None: + self._on_agent_response: list[OnAgentResponseCallback] = [] + self._on_tool_call: list[OnToolCallCallback] = [] + self._on_complete: list[OnCompleteCallback] = [] + self._on_error: list[OnErrorCallback] = [] + + def on_agent_response(self, func: OnAgentResponseCallback) -> OnAgentResponseCallback: + """Register a handler fired after every agent response.""" + self._on_agent_response.append(func) + return func + + def on_tool_call(self, func: OnToolCallCallback) -> OnToolCallCallback: + """Register a handler fired after each tool in a batch executes.""" + self._on_tool_call.append(func) + return func + + def on_complete(self, func: OnCompleteCallback) -> OnCompleteCallback: + """Register a handler fired when the loop exits cleanly.""" + self._on_complete.append(func) + return func + + def on_error(self, func: OnErrorCallback) -> OnErrorCallback: + """Register a handler fired when the loop aborts.""" + self._on_error.append(func) + return func + + async def fire_agent_response(self, response: AgentResponse, iteration: int) -> None: + for handler in self._on_agent_response: + try: + await handler(response, iteration) + except Exception: + pass # we will see in the future what to do with this + + async def fire_tool_call(self, tool_call: ToolCall, result: ToolResult, iteration: int) -> None: + for handler in self._on_tool_call: + try: + await handler(tool_call, result, iteration) + except Exception: + pass + + async def fire_complete(self, result: ReActResult) -> None: + for handler in self._on_complete: + try: + await handler(result) + except Exception: + pass + + async def fire_error(self, error: BaseException) -> None: + for handler in self._on_error: + try: + await handler(error) + except Exception: + pass diff --git a/ddev/src/ddev/ai/react/process.py b/ddev/src/ddev/ai/react/process.py index e48a88dc14e0a..1b2f509bea8e0 100644 --- a/ddev/src/ddev/ai/react/process.py +++ b/ddev/src/ddev/ai/react/process.py @@ -3,48 +3,17 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import asyncio -from dataclasses import dataclass -from typing import Any, Protocol +from typing import Any from ddev.ai.agent.base import BaseAgent from ddev.ai.agent.exceptions import AgentError -from ddev.ai.agent.types import AgentResponse, ContextUsage, StopReason, ToolCall, ToolResultMessage +from ddev.ai.agent.types import StopReason, ToolResultMessage +from ddev.ai.react.callbacks import CallbackSet +from ddev.ai.react.types import ReActResult from ddev.ai.tools.core.registry import ToolRegistry from ddev.ai.tools.core.types import ToolResult -@dataclass(frozen=True) -class ReActResult: - """Immutable summary of a completed ReAct loop run.""" - - final_response: AgentResponse - iterations: int - total_input_tokens: int # sum across all iterations - total_output_tokens: int # sum across all iterations - context_usage: ContextUsage | None # promoted from final_response.usage.context_usage - - -class ReActCallback(Protocol): - """Observer interface for ReActProcess lifecycle events.""" - - async def on_agent_response(self, response: AgentResponse, iteration: int) -> None: - """Called after every agent.send() returns, including the first.""" - ... - - async def on_tool_call(self, tool_call: ToolCall, result: ToolResult, iteration: int) -> None: - """Called once per (tool_call, result) pair after all tools in a batch execute.""" - ... - - async def on_complete(self, result: ReActResult) -> None: - """Called when the loop exits cleanly with a ReActResult.""" - ... - - async def on_error(self, error: BaseException) -> None: - """Called when the loop aborts — covers AgentError, KeyboardInterrupt, and CancelledError. - The exception is always re-raised after this returns.""" - ... - - class ReActProcess: """ Manages the ReAct (Reason + Act) loop for a single task. @@ -57,17 +26,17 @@ def __init__( self, agent: BaseAgent[Any], tool_registry: ToolRegistry, - callbacks: list[ReActCallback] | None = None, + callback_sets: list[CallbackSet] | None = None, ) -> None: """ Args: agent: A BaseAgent subclass (e.g. AnthropicAgent). tool_registry: Registry of tools available in this loop. - callbacks: Optional observers. Empty list means no events are fired. + callback_sets: Optional CallbackSet instances to observe loop events. """ self._agent = agent self._tool_registry = tool_registry - self._callbacks: list[ReActCallback] = callbacks or [] + self._callback_sets: list[CallbackSet] = callback_sets or [] async def start(self, prompt: str, allowed_tools: list[str] | None = None) -> ReActResult: """ @@ -89,11 +58,8 @@ async def start(self, prompt: str, allowed_tools: list[str] | None = None) -> Re total_input = response.usage.input_tokens total_output = response.usage.output_tokens - for cb in self._callbacks: - try: - await cb.on_agent_response(response, iterations) - except Exception: - pass # in the future we should log this error + for cb_set in self._callback_sets: + await cb_set.fire_agent_response(response, iterations) # No iteration cap — this is an interactive CLI tool; the user can Ctrl+C to stop. while response.stop_reason == StopReason.TOOL_USE: @@ -112,11 +78,8 @@ async def start(self, prompt: str, allowed_tools: list[str] | None = None) -> Re tool_call_results = list(zip(response.tool_calls, tool_results, strict=True)) for tc, result in tool_call_results: - for cb in self._callbacks: - try: - await cb.on_tool_call(tc, result, iterations) - except Exception: - pass + for cb_set in self._callback_sets: + await cb_set.fire_tool_call(tc, result, iterations) messages = [ToolResultMessage(tool_call_id=tc.id, result=result) for tc, result in tool_call_results] @@ -125,11 +88,8 @@ async def start(self, prompt: str, allowed_tools: list[str] | None = None) -> Re total_input += response.usage.input_tokens total_output += response.usage.output_tokens - for cb in self._callbacks: - try: - await cb.on_agent_response(response, iterations) - except Exception: - pass + for cb_set in self._callback_sets: + await cb_set.fire_agent_response(response, iterations) react_result = ReActResult( final_response=response, @@ -139,18 +99,12 @@ async def start(self, prompt: str, allowed_tools: list[str] | None = None) -> Re context_usage=response.usage.context_usage, ) - for cb in self._callbacks: - try: - await cb.on_complete(react_result) - except Exception: - pass + for cb_set in self._callback_sets: + await cb_set.fire_complete(react_result) return react_result except BaseException as e: - for cb in self._callbacks: - try: - await cb.on_error(e) - except Exception: - pass + for cb_set in self._callback_sets: + await cb_set.fire_error(e) raise diff --git a/ddev/src/ddev/ai/react/types.py b/ddev/src/ddev/ai/react/types.py new file mode 100644 index 0000000000000..4a6b2345433dc --- /dev/null +++ b/ddev/src/ddev/ai/react/types.py @@ -0,0 +1,18 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from dataclasses import dataclass + +from ddev.ai.agent.types import AgentResponse, ContextUsage + + +@dataclass(frozen=True) +class ReActResult: + """Immutable summary of a completed ReAct loop run.""" + + final_response: AgentResponse + iterations: int + total_input_tokens: int # sum across all iterations + total_output_tokens: int # sum across all iterations + context_usage: ContextUsage | None # promoted from final_response.usage.context_usage diff --git a/ddev/tests/ai/agent/test_client.py b/ddev/tests/ai/agent/test_anthropic_client.py similarity index 99% rename from ddev/tests/ai/agent/test_client.py rename to ddev/tests/ai/agent/test_anthropic_client.py index ae9578c5c87ad..5b68c0ff9aca5 100644 --- a/ddev/tests/ai/agent/test_client.py +++ b/ddev/tests/ai/agent/test_anthropic_client.py @@ -8,7 +8,7 @@ import anthropic import pytest -from ddev.ai.agent.client import AnthropicAgent +from ddev.ai.agent.anthropic_client import AnthropicAgent from ddev.ai.agent.exceptions import AgentAPIError, AgentConnectionError, AgentError, AgentRateLimitError from ddev.ai.agent.types import StopReason, ToolResultMessage from ddev.ai.tools.core.registry import ToolRegistry diff --git a/ddev/tests/ai/react/test_callbacks.py b/ddev/tests/ai/react/test_callbacks.py new file mode 100644 index 0000000000000..da93f6c159e87 --- /dev/null +++ b/ddev/tests/ai/react/test_callbacks.py @@ -0,0 +1,184 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import pytest + +from ddev.ai.agent.types import AgentResponse, StopReason, TokenUsage, ToolCall +from ddev.ai.react.callbacks import CallbackSet +from ddev.ai.react.types import ReActResult +from ddev.ai.tools.core.types import ToolResult + +# --------------------------------------------------------------------------- +# Minimal fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def response() -> AgentResponse: + return AgentResponse( + stop_reason=StopReason.END_TURN, + text="", + tool_calls=[], + usage=TokenUsage(input_tokens=10, output_tokens=5, cache_read_input_tokens=0, cache_creation_input_tokens=0), + ) + + +@pytest.fixture +def tool_call() -> ToolCall: + return ToolCall(id="tc_01", name="read_file", input={}) + + +@pytest.fixture +def react_result(response: AgentResponse) -> ReActResult: + return ReActResult( + final_response=response, + iterations=1, + total_input_tokens=10, + total_output_tokens=5, + context_usage=None, + ) + + +# --------------------------------------------------------------------------- +# Registration via decorators +# --------------------------------------------------------------------------- + + +async def test_decorator_returns_original_function() -> None: + cb = CallbackSet() + + async def handler(response: AgentResponse, iteration: int) -> None: + pass + + assert cb.on_agent_response(handler) is handler + + +async def test_decorator_registers_handler_in_internal_list() -> None: + cb = CallbackSet() + + @cb.on_agent_response + async def h1(response: AgentResponse, iteration: int) -> None: ... + + @cb.on_agent_response + async def h2(response: AgentResponse, iteration: int) -> None: ... + + assert cb._on_agent_response == [h1, h2] + + +# --------------------------------------------------------------------------- +# Dispatch ordering and isolation +# --------------------------------------------------------------------------- + + +async def test_empty_callback_set_is_noop( + response: AgentResponse, tool_call: ToolCall, react_result: ReActResult +) -> None: + cb = CallbackSet() + await cb.fire_agent_response(response, 1) + await cb.fire_tool_call(tool_call, ToolResult(success=True, data="ok"), 1) + await cb.fire_complete(react_result) + await cb.fire_error(RuntimeError("boom")) + + +async def test_multiple_handlers_same_event_all_fire(response: AgentResponse) -> None: + cb = CallbackSet() + fired: list[int] = [] + + @cb.on_agent_response + async def first(response: AgentResponse, iteration: int) -> None: + fired.append(1) + + @cb.on_agent_response + async def second(response: AgentResponse, iteration: int) -> None: + fired.append(2) + + @cb.on_agent_response + async def third(response: AgentResponse, iteration: int) -> None: + fired.append(3) + + await cb.fire_agent_response(response, 5) + + assert fired == [1, 2, 3] + + +async def test_handlers_receive_correct_arguments(response: AgentResponse) -> None: + cb = CallbackSet() + received: list[tuple] = [] + + @cb.on_agent_response + async def h(response: AgentResponse, iteration: int) -> None: + received.append((response, iteration)) + + await cb.fire_agent_response(response, 7) + + assert received == [(response, 7)] + + +# --------------------------------------------------------------------------- +# Exception-swallowing guarantee +# --------------------------------------------------------------------------- + + +async def test_fire_swallows_handler_exception(response: AgentResponse) -> None: + cb = CallbackSet() + fired: list[int] = [] + + @cb.on_agent_response + async def bad(response: AgentResponse, iteration: int) -> None: + raise RuntimeError("boom") + + @cb.on_agent_response + async def good(response: AgentResponse, iteration: int) -> None: + fired.append(iteration) + + await cb.fire_agent_response(response, 1) + assert fired == [1] + + +async def test_fire_tool_call_swallows_handler_exception(tool_call: ToolCall) -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_tool_call + async def bad(tool_call: ToolCall, result: ToolResult, iteration: int) -> None: + raise RuntimeError("boom") + + @cb.on_tool_call + async def good(tool_call: ToolCall, result: ToolResult, iteration: int) -> None: + fired.append(True) + + await cb.fire_tool_call(tool_call, ToolResult(success=True, data="ok"), 1) + assert fired == [True] + + +async def test_fire_complete_swallows_handler_exception(react_result: ReActResult) -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_complete + async def bad(result: ReActResult) -> None: + raise RuntimeError("boom") + + @cb.on_complete + async def good(result: ReActResult) -> None: + fired.append(True) + + await cb.fire_complete(react_result) + assert fired == [True] + + +async def test_fire_error_swallows_handler_exception() -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_error + async def bad(error: BaseException) -> None: + raise RuntimeError("boom") + + @cb.on_error + async def good(error: BaseException) -> None: + fired.append(True) + + await cb.fire_error(ValueError("original error")) + assert fired == [True] diff --git a/ddev/tests/ai/react/test_process.py b/ddev/tests/ai/react/test_process.py index 664e25c5505a6..8424d6096d14e 100644 --- a/ddev/tests/ai/react/test_process.py +++ b/ddev/tests/ai/react/test_process.py @@ -10,7 +10,9 @@ from ddev.ai.agent.base import BaseAgent from ddev.ai.agent.exceptions import AgentConnectionError from ddev.ai.agent.types import AgentResponse, ContextUsage, StopReason, TokenUsage, ToolCall, ToolResultMessage -from ddev.ai.react.process import ReActCallback, ReActProcess, ReActResult +from ddev.ai.react.callbacks import CallbackSet +from ddev.ai.react.process import ReActProcess +from ddev.ai.react.types import ReActResult from ddev.ai.tools.core.types import ToolResult # --------------------------------------------------------------------------- @@ -74,26 +76,32 @@ async def run(self, name: str, raw: dict[str, object]) -> ToolResult: return behavior -class MockCallback: - """Records all lifecycle events emitted by ReActProcess.""" +class CallbackRecorder: + """Test helper that wires a CallbackSet to record all lifecycle events.""" def __init__(self) -> None: self.agent_responses: list[tuple[AgentResponse, int]] = [] self.tool_calls_seen: list[tuple[ToolCall, ToolResult, int]] = [] self.complete_results: list[ReActResult] = [] - self.errors: list[Exception] = [] + self.errors: list[BaseException] = [] - async def on_agent_response(self, response: AgentResponse, iteration: int) -> None: - self.agent_responses.append((response, iteration)) + self.callback_set = CallbackSet() - async def on_tool_call(self, tool_call: ToolCall, result: ToolResult, iteration: int) -> None: - self.tool_calls_seen.append((tool_call, result, iteration)) + @self.callback_set.on_agent_response + async def _record_response(response: AgentResponse, iteration: int) -> None: + self.agent_responses.append((response, iteration)) - async def on_complete(self, result: ReActResult) -> None: - self.complete_results.append(result) + @self.callback_set.on_tool_call + async def _record_tool_call(tool_call: ToolCall, result: ToolResult, iteration: int) -> None: + self.tool_calls_seen.append((tool_call, result, iteration)) - async def on_error(self, error: Exception) -> None: - self.errors.append(error) + @self.callback_set.on_complete + async def _record_complete(result: ReActResult) -> None: + self.complete_results.append(result) + + @self.callback_set.on_error + async def _record_error(error: BaseException) -> None: + self.errors.append(error) def make_response( @@ -126,12 +134,12 @@ def make_tool_call( def make_process( agent: MockAgent, registry: MockToolRegistry | None = None, - callbacks: list[ReActCallback] | None = None, + callback_sets: list[CallbackSet] | None = None, ) -> ReActProcess: return ReActProcess( agent=agent, tool_registry=registry or MockToolRegistry(), - callbacks=callbacks, + callback_sets=callback_sets, ) @@ -141,7 +149,7 @@ def make_process( @pytest.mark.parametrize("stop_reason", [StopReason.END_TURN, StopReason.MAX_TOKENS, StopReason.OTHER]) -async def test_stop_reason_single_response(stop_reason) -> None: +async def test_stop_reason_single_response(stop_reason: StopReason) -> None: agent = MockAgent([make_response(stop_reason)]) result = await make_process(agent).start("Hi") @@ -240,16 +248,16 @@ async def test_tool_exception_on_tool_call_callback_fires_with_error_result() -> make_response(StopReason.END_TURN), ] ) - callback = MockCallback() + recorder = CallbackRecorder() await ReActProcess( agent=agent, tool_registry=RaisingToolRegistry(ValueError("oops")), - callbacks=[callback], + callback_sets=[recorder.callback_set], ).start("x") - assert len(callback.tool_calls_seen) == 1 - _, error_result, _ = callback.tool_calls_seen[0] + assert len(recorder.tool_calls_seen) == 1 + _, error_result, _ = recorder.tool_calls_seen[0] assert error_result.success is False @@ -322,23 +330,31 @@ async def test_callbacks_invoked_correct_counts() -> None: ] expected_result = ToolResult(success=True, data="ok") registry = MockToolRegistry(result=expected_result) - callback = MockCallback() + recorder = CallbackRecorder() agent = MockAgent(responses) - result = await make_process(agent, registry=registry, callbacks=[callback]).start("Run tools") + result = await make_process(agent, registry=registry, callback_sets=[recorder.callback_set]).start("Run tools") + + assert len(recorder.agent_responses) == 2 + assert recorder.agent_responses[0][1] == 1 + assert recorder.agent_responses[1][1] == 2 + assert len(recorder.tool_calls_seen) == 2 + assert all(iteration == 1 for _, _, iteration in recorder.tool_calls_seen) + assert recorder.tool_calls_seen[0][0] is tool_calls[0] + assert recorder.tool_calls_seen[1][0] is tool_calls[1] + assert recorder.tool_calls_seen[0][1] is expected_result + assert recorder.tool_calls_seen[1][1] is expected_result + assert len(recorder.complete_results) == 1 + assert recorder.complete_results[0] is result + assert len(recorder.errors) == 0 + - assert len(callback.agent_responses) == 2 - assert callback.agent_responses[0][1] == 1 - assert callback.agent_responses[1][1] == 2 - assert len(callback.tool_calls_seen) == 2 - assert all(iteration == 1 for _, _, iteration in callback.tool_calls_seen) - assert callback.tool_calls_seen[0][0] is tool_calls[0] - assert callback.tool_calls_seen[1][0] is tool_calls[1] - assert callback.tool_calls_seen[0][1] is expected_result - assert callback.tool_calls_seen[1][1] is expected_result - assert len(callback.complete_results) == 1 - assert callback.complete_results[0] is result - assert len(callback.errors) == 0 +async def test_two_callback_sets_both_notified() -> None: + agent = MockAgent([make_response(StopReason.END_TURN)]) + rec_a, rec_b = CallbackRecorder(), CallbackRecorder() + await make_process(agent, callback_sets=[rec_a.callback_set, rec_b.callback_set]).start("x") + assert len(rec_a.complete_results) == 1 + assert len(rec_b.complete_results) == 1 # --------------------------------------------------------------------------- @@ -354,20 +370,20 @@ async def send( async def test_agent_error_notifies_and_reraises() -> None: - callback = MockCallback() + recorder = CallbackRecorder() process = ReActProcess( agent=ErrorAgent(), tool_registry=MockToolRegistry(), - callbacks=[callback], + callback_sets=[recorder.callback_set], ) with pytest.raises(AgentConnectionError): await process.start("Anything") - assert len(callback.errors) == 1 - assert isinstance(callback.errors[0], AgentConnectionError) - assert len(callback.complete_results) == 0 - assert len(callback.agent_responses) == 0 + assert len(recorder.errors) == 1 + assert isinstance(recorder.errors[0], AgentConnectionError) + assert len(recorder.complete_results) == 0 + assert len(recorder.agent_responses) == 0 class InterruptAgent(BaseAgent[Any]): @@ -378,41 +394,42 @@ async def send( async def test_keyboard_interrupt_notifies_and_reraises() -> None: - callback = MockCallback() + recorder = CallbackRecorder() process = ReActProcess( agent=InterruptAgent(), tool_registry=MockToolRegistry(), - callbacks=[callback], + callback_sets=[recorder.callback_set], ) with pytest.raises(KeyboardInterrupt): await process.start("Anything") - assert len(callback.errors) == 1 - assert isinstance(callback.errors[0], KeyboardInterrupt) - assert len(callback.complete_results) == 0 + assert len(recorder.errors) == 1 + assert isinstance(recorder.errors[0], KeyboardInterrupt) + assert len(recorder.complete_results) == 0 -async def test_cancelled_error_notifies_and_reraises() -> None: - class CancelledAgent(BaseAgent[Any]): - async def send( - self, content: str | list[ToolResultMessage], allowed_tools: list[str] | None = None - ) -> AgentResponse: - raise asyncio.CancelledError +class CancelledAgent(BaseAgent[Any]): + async def send( + self, content: str | list[ToolResultMessage], allowed_tools: list[str] | None = None + ) -> AgentResponse: + raise asyncio.CancelledError - callback = MockCallback() + +async def test_cancelled_error_notifies_and_reraises() -> None: + recorder = CallbackRecorder() process = ReActProcess( agent=CancelledAgent(), tool_registry=MockToolRegistry(), - callbacks=[callback], + callback_sets=[recorder.callback_set], ) with pytest.raises(asyncio.CancelledError): await process.start("Anything") - assert len(callback.errors) == 1 - assert isinstance(callback.errors[0], asyncio.CancelledError) - assert len(callback.complete_results) == 0 + assert len(recorder.errors) == 1 + assert isinstance(recorder.errors[0], asyncio.CancelledError) + assert len(recorder.complete_results) == 0 # --------------------------------------------------------------------------- From 92a6b94c0448545680278149e893290ad3746f03 Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Wed, 15 Apr 2026 12:07:16 +0200 Subject: [PATCH 27/44] Add context compaction to AI framework (#23307) * New compact context functionality * Few changes on naming and tests * Changed compaction inside ReAct loop * Change ReAct.compact for it to return token usage * ReAct loop only preserves last turn if tool use * Merge compact functions into one --- ddev/src/ddev/ai/agent/anthropic_client.py | 5 +- ddev/src/ddev/ai/agent/base.py | 73 ++++++- ddev/src/ddev/ai/react/callbacks.py | 38 ++++ ddev/src/ddev/ai/react/process.py | 47 +++- ddev/tests/ai/agent/test_base.py | 236 +++++++++++++++++++++ ddev/tests/ai/react/test_callbacks.py | 70 ++++++ ddev/tests/ai/react/test_process.py | 164 +++++++++++++- 7 files changed, 621 insertions(+), 12 deletions(-) create mode 100644 ddev/tests/ai/agent/test_base.py diff --git a/ddev/src/ddev/ai/agent/anthropic_client.py b/ddev/src/ddev/ai/agent/anthropic_client.py index add571ed03491..a2efcb1c25e38 100644 --- a/ddev/src/ddev/ai/agent/anthropic_client.py +++ b/ddev/src/ddev/ai/agent/anthropic_client.py @@ -41,11 +41,8 @@ def __init__( programmatic_tool_calling: Whether to allow programmatic tool calling. """ - super().__init__() + super().__init__(name=name, system_prompt=system_prompt, tools=tools) self._client = client - self._tools = tools - self._system_prompt = system_prompt - self.name = name self._model = model self._max_tokens = max_tokens self._programmatic_tool_calling = programmatic_tool_calling diff --git a/ddev/src/ddev/ai/agent/base.py b/ddev/src/ddev/ai/agent/base.py index 7546a19747960..814f6bedf7429 100644 --- a/ddev/src/ddev/ai/agent/base.py +++ b/ddev/src/ddev/ai/agent/base.py @@ -4,20 +4,41 @@ from abc import ABC, abstractmethod from copy import deepcopy +from typing import Final from ddev.ai.agent.types import AgentResponse, ToolResultMessage +from ddev.ai.tools.core.registry import ToolRegistry + +_COMPACT_SYSTEM_PROMPT: Final[str] = """\ +You are summarizing an agentic conversation to free up context space. +Produce a dense, structured summary that covers ALL of the following: + 1. The original task given to the agent + 2. Every tool call made and the key finding or result from each + 3. Any decisions, conclusions, or hypotheses the agent reached + 4. What has been completed and what work remains + +Rules: +- Be exhaustive on facts and findings; omit raw data already consumed +- Use bullet points, not prose +- The agent will read ONLY this summary to continue — it must be self-sufficient +""" + +_COMPACT_REQUEST: Final[str] = "Summarize the conversation so far following your instructions." class BaseAgent[TMessage](ABC): """Abstract base class for all agent implementations. - Provides shared, provider-agnostic history management. The message type - TMessage is supplied by each concrete provider (e.g. MessageParam for Anthropic). - Subclasses must implement send(). + Provides shared, provider-agnostic history management and compaction. + The message type TMessage is supplied by each concrete provider + (e.g. MessageParam for Anthropic). Subclasses must implement send(). """ - def __init__(self) -> None: + def __init__(self, name: str, system_prompt: str, tools: ToolRegistry) -> None: self._history: list[TMessage] = [] + self.name = name + self._system_prompt = system_prompt + self._tools = tools @property def history(self) -> list[TMessage]: @@ -28,6 +49,50 @@ def reset(self) -> None: """Clear conversation history to start a new conversation.""" self._history = [] + async def compact(self) -> AgentResponse | None: + """Collapse history to 2 messages: original task + LLM summary. + + Returns the AgentResponse from the compaction call so callers can + account for its token usage. Returns None if history is already ≤ 2. + """ + if len(self._history) <= 2: + return None + + original_prompt = self._history[0] + original_system = self._system_prompt + + self._system_prompt = _COMPACT_SYSTEM_PROMPT + try: + response = await self.send(_COMPACT_REQUEST, allowed_tools=[]) + finally: + self._system_prompt = original_system # restore even if send() raises + + compact_response = self._history[-1] # summary message added by send() + + self.reset() + self._history = [original_prompt, compact_response] + return response + + async def compact_preserving_last_turn(self) -> AgentResponse | None: + """Compact history while keeping the last user+assistant pair intact. + + Used mid-ReAct loop where the last assistant message contains unresolved + tool calls that still need a tool-result response. After compaction the + preserved pair re-anchors the pending turn so the next send(tool_results) + produces a valid alternating message sequence. No-op if history is ≤ 3 + messages (too short to compact without corrupting the sequence). + + Returns the AgentResponse from the compaction call, or None if no + compaction occurred. + """ + if len(self._history) <= 3: + return None + + last_turn = self._history[-2:] # [user(tool_results_N), assistant(tool_use_N+1)] + response = await self.compact() + self._history.extend(last_turn) + return response + @abstractmethod async def send( self, diff --git a/ddev/src/ddev/ai/react/callbacks.py b/ddev/src/ddev/ai/react/callbacks.py index 59ef6687ff67d..43b25a2d5f52e 100644 --- a/ddev/src/ddev/ai/react/callbacks.py +++ b/ddev/src/ddev/ai/react/callbacks.py @@ -33,6 +33,18 @@ class OnErrorCallback(Protocol): async def __call__(self, error: BaseException) -> None: ... +class BeforeCompactCallback(Protocol): + """Called immediately before the agent's history is compacted.""" + + async def __call__(self) -> None: ... + + +class AfterCompactCallback(Protocol): + """Called immediately after the agent's history has been compacted.""" + + async def __call__(self) -> None: ... + + class CallbackSet: """Decorator-based registry for ReAct lifecycle event handlers. @@ -50,6 +62,8 @@ def __init__(self) -> None: self._on_tool_call: list[OnToolCallCallback] = [] self._on_complete: list[OnCompleteCallback] = [] self._on_error: list[OnErrorCallback] = [] + self._before_compact: list[BeforeCompactCallback] = [] + self._after_compact: list[AfterCompactCallback] = [] def on_agent_response(self, func: OnAgentResponseCallback) -> OnAgentResponseCallback: """Register a handler fired after every agent response.""" @@ -71,6 +85,16 @@ def on_error(self, func: OnErrorCallback) -> OnErrorCallback: self._on_error.append(func) return func + def on_before_compact(self, func: BeforeCompactCallback) -> BeforeCompactCallback: + """Register a handler fired just before compaction runs.""" + self._before_compact.append(func) + return func + + def on_after_compact(self, func: AfterCompactCallback) -> AfterCompactCallback: + """Register a handler fired just after compaction completes.""" + self._after_compact.append(func) + return func + async def fire_agent_response(self, response: AgentResponse, iteration: int) -> None: for handler in self._on_agent_response: try: @@ -98,3 +122,17 @@ async def fire_error(self, error: BaseException) -> None: await handler(error) except Exception: pass + + async def fire_before_compact(self) -> None: + for handler in self._before_compact: + try: + await handler() + except Exception: + pass + + async def fire_after_compact(self) -> None: + for handler in self._after_compact: + try: + await handler() + except Exception: + pass diff --git a/ddev/src/ddev/ai/react/process.py b/ddev/src/ddev/ai/react/process.py index 1b2f509bea8e0..a4e5476dfffda 100644 --- a/ddev/src/ddev/ai/react/process.py +++ b/ddev/src/ddev/ai/react/process.py @@ -7,7 +7,7 @@ from ddev.ai.agent.base import BaseAgent from ddev.ai.agent.exceptions import AgentError -from ddev.ai.agent.types import StopReason, ToolResultMessage +from ddev.ai.agent.types import AgentResponse, StopReason, ToolResultMessage from ddev.ai.react.callbacks import CallbackSet from ddev.ai.react.types import ReActResult from ddev.ai.tools.core.registry import ToolRegistry @@ -27,16 +27,56 @@ def __init__( agent: BaseAgent[Any], tool_registry: ToolRegistry, callback_sets: list[CallbackSet] | None = None, + compact_threshold_pct: float | None = 75.0, ) -> None: """ Args: agent: A BaseAgent subclass (e.g. AnthropicAgent). tool_registry: Registry of tools available in this loop. callback_sets: Optional CallbackSet instances to observe loop events. + compact_threshold_pct: Context usage percentage at which the loop auto-compacts. + None disables auto-compaction entirely. """ self._agent = agent self._tool_registry = tool_registry self._callback_sets: list[CallbackSet] = callback_sets or [] + self._compact_threshold_pct = compact_threshold_pct + + def reset(self) -> None: + """Clear the agent's conversation history.""" + self._agent.reset() + + async def compact(self, response: AgentResponse | None = None) -> tuple[int, int]: + """Compact the agent's conversation history unconditionally. + + Args: + response: The last agent response. If None, compaction is unconditional. + + Returns (input_tokens, output_tokens) from the compaction API call. + Returns (0, 0) if history was already compact and no API call was made. + """ + for cb_set in self._callback_sets: + await cb_set.fire_before_compact() + + compact_response = None + if response is None or response.stop_reason != StopReason.TOOL_USE: + compact_response = await self._agent.compact() + else: + compact_response = await self._agent.compact_preserving_last_turn() + + for cb_set in self._callback_sets: + await cb_set.fire_after_compact() + if compact_response is None: + return 0, 0 + return compact_response.usage.input_tokens, compact_response.usage.output_tokens + + def _is_compact_needed(self, response: AgentResponse) -> bool: + if self._compact_threshold_pct is None: + return False + ctx = response.usage.context_usage + if ctx is None or ctx.context_pct < self._compact_threshold_pct: + return False + return True async def start(self, prompt: str, allowed_tools: list[str] | None = None) -> ReActResult: """ @@ -91,6 +131,11 @@ async def start(self, prompt: str, allowed_tools: list[str] | None = None) -> Re for cb_set in self._callback_sets: await cb_set.fire_agent_response(response, iterations) + if self._is_compact_needed(response): + compact_in, compact_out = await self.compact(response) + total_input += compact_in + total_output += compact_out + react_result = ReActResult( final_response=response, iterations=iterations, diff --git a/ddev/tests/ai/agent/test_base.py b/ddev/tests/ai/agent/test_base.py new file mode 100644 index 0000000000000..757674ac43446 --- /dev/null +++ b/ddev/tests/ai/agent/test_base.py @@ -0,0 +1,236 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import pytest + +from ddev.ai.agent.base import _COMPACT_SYSTEM_PROMPT, BaseAgent +from ddev.ai.agent.types import AgentResponse, StopReason, TokenUsage, ToolResultMessage +from ddev.ai.tools.core.registry import ToolRegistry + +_AGENT_NAME: str = "test" +_AGENT_SYSTEM_PROMPT: str = "original" + +# --------------------------------------------------------------------------- +# Minimal concrete agent for testing BaseAgent +# --------------------------------------------------------------------------- + + +class ConcreteAgent(BaseAgent[dict]): + """Minimal BaseAgent subclass that records send() calls and replays configured responses.""" + + def __init__(self, responses: list[str | Exception] | None = None) -> None: + super().__init__(name=_AGENT_NAME, system_prompt=_AGENT_SYSTEM_PROMPT, tools=ToolRegistry([])) + self._responses = list(responses or []) + self._idx = 0 + self.send_calls: list[dict] = [] + + async def send( + self, + content: str | list[ToolResultMessage], + allowed_tools: list[str] | None = None, + ) -> AgentResponse: + self.send_calls.append( + {"content": content, "allowed_tools": allowed_tools, "system_prompt": self._system_prompt} + ) + if self._idx < len(self._responses): + resp = self._responses[self._idx] + self._idx += 1 + if isinstance(resp, Exception): + raise resp + text = resp + else: + text = "default summary" + self._idx += 1 + + # Simulate what a real provider does: append user + assistant messages. + self._history.extend( + [ + {"role": "user", "content": content}, + {"role": "assistant", "content": text}, + ] + ) + return AgentResponse( + stop_reason=StopReason.END_TURN, + text=text, + tool_calls=[], + usage=TokenUsage(input_tokens=0, output_tokens=0, cache_read_input_tokens=0, cache_creation_input_tokens=0), + ) + + +def make_history(n_messages: int) -> list[dict]: + """Build n_messages alternating user/assistant dicts for seeding _history directly.""" + return [{"role": "user" if i % 2 == 0 else "assistant", "content": f"msg-{i}"} for i in range(n_messages)] + + +# --------------------------------------------------------------------------- +# compact() — history length after compact (guard cases + collapse) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "n_messages, responses, expected_len", + [ + (0, None, 0), + (1, None, 1), + (2, None, 2), + (4, ["summary"], 2), + ], +) +async def test_compact_history_length( + n_messages: int, + responses: list[str] | None, + expected_len: int, +) -> None: + agent = ConcreteAgent(responses=responses) + agent._history = make_history(n_messages) + await agent.compact() + assert len(agent.history) == expected_len + + +async def test_compact_first_message_is_original_task() -> None: + agent = ConcreteAgent(responses=["summary"]) + original = make_history(4)[0] + agent._history = make_history(4) + await agent.compact() + assert agent.history[0] == original + + +async def test_compact_second_message_is_summary_response() -> None: + agent = ConcreteAgent(responses=["the summary text"]) + agent._history = make_history(4) + await agent.compact() + summary_msg = agent.history[1] + assert summary_msg["role"] == "assistant" + assert summary_msg["content"] == "the summary text" + + +# --------------------------------------------------------------------------- +# compact() — system prompt swap +# --------------------------------------------------------------------------- + + +async def test_compact_uses_compaction_system_prompt() -> None: + agent = ConcreteAgent(responses=["summary"]) + agent._history = make_history(4) + await agent.compact() + assert agent.send_calls[0]["system_prompt"] == _COMPACT_SYSTEM_PROMPT + + +async def test_compact_restores_original_system_prompt() -> None: + agent = ConcreteAgent(responses=["summary"]) + agent._history = make_history(4) + await agent.compact() + assert agent._system_prompt == _AGENT_SYSTEM_PROMPT + + +async def test_compact_restores_system_prompt_on_send_error() -> None: + agent = ConcreteAgent(responses=[RuntimeError("api failure")]) + agent._history = make_history(4) + with pytest.raises(RuntimeError): + await agent.compact() + assert agent._system_prompt == _AGENT_SYSTEM_PROMPT + + +# --------------------------------------------------------------------------- +# compact() — send() called with no tools +# --------------------------------------------------------------------------- + + +async def test_compact_send_uses_no_tools() -> None: + agent = ConcreteAgent(responses=["summary"]) + agent._history = make_history(4) + await agent.compact() + assert agent.send_calls[0]["allowed_tools"] == [] + + +# --------------------------------------------------------------------------- +# compact() — error leaves history unchanged +# --------------------------------------------------------------------------- + + +async def test_compact_leaves_history_unchanged_on_send_error() -> None: + agent = ConcreteAgent(responses=[RuntimeError("api failure")]) + original_history = make_history(4) + agent._history = list(original_history) + with pytest.raises(RuntimeError): + await agent.compact() + assert agent._history == original_history + + +# --------------------------------------------------------------------------- +# compact() — return value +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("n_messages", [0, 1, 2]) +async def test_compact_returns_none_when_history_too_short(n_messages: int) -> None: + agent = ConcreteAgent() + agent._history = make_history(n_messages) + result = await agent.compact() + assert result is None + + +async def test_compact_returns_response_when_compaction_occurs() -> None: + agent = ConcreteAgent(responses=["the summary"]) + agent._history = make_history(4) + result = await agent.compact() + assert result is not None + assert result.text == "the summary" + + +# --------------------------------------------------------------------------- +# compact_preserving_last_turn() — guard: history too short +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("n_messages", [0, 1, 2, 3]) +async def test_compact_preserving_last_turn_does_nothing_when_history_is_short(n_messages: int) -> None: + agent = ConcreteAgent() + agent._history = make_history(n_messages) + await agent.compact_preserving_last_turn() + assert len(agent.send_calls) == 0 + assert len(agent._history) == n_messages + + +# --------------------------------------------------------------------------- +# compact_preserving_last_turn() — collapses middle, keeps last two +# --------------------------------------------------------------------------- + + +async def test_compact_preserving_last_turn_keeps_last_two_messages() -> None: + agent = ConcreteAgent(responses=["summary"]) + agent._history = make_history(6) + last_two = agent._history[-2:] + await agent.compact_preserving_last_turn() + assert agent.history[-2:] == last_two + + +async def test_compact_preserving_last_turn_first_message_is_original_task() -> None: + agent = ConcreteAgent(responses=["summary"]) + original = make_history(6)[0] + agent._history = make_history(6) + await agent.compact_preserving_last_turn() + assert agent.history[0] == original + + +async def test_compact_preserving_last_turn_produces_four_messages() -> None: + agent = ConcreteAgent(responses=["summary"]) + agent._history = make_history(6) + await agent.compact_preserving_last_turn() + # original + summary + last user + last assistant + assert len(agent.history) == 4 + + +# --------------------------------------------------------------------------- +# compact_preserving_last_turn() — error leaves history unchanged +# --------------------------------------------------------------------------- + + +async def test_compact_preserving_last_turn_leaves_history_unchanged_on_error() -> None: + agent = ConcreteAgent(responses=[RuntimeError("api failure")]) + original_history = make_history(6) + agent._history = list(original_history) + with pytest.raises(RuntimeError): + await agent.compact_preserving_last_turn() + assert agent._history == original_history diff --git a/ddev/tests/ai/react/test_callbacks.py b/ddev/tests/ai/react/test_callbacks.py index da93f6c159e87..4b6f3618411df 100644 --- a/ddev/tests/ai/react/test_callbacks.py +++ b/ddev/tests/ai/react/test_callbacks.py @@ -182,3 +182,73 @@ async def good(error: BaseException) -> None: await cb.fire_error(ValueError("original error")) assert fired == [True] + + +# --------------------------------------------------------------------------- +# before_compact and after_compact +# --------------------------------------------------------------------------- + + +async def test_before_compact_registered_and_fired() -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_before_compact + async def h() -> None: + fired.append(True) + + await cb.fire_before_compact() + assert fired == [True] + + +async def test_after_compact_registered_and_fired() -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_after_compact + async def h() -> None: + fired.append(True) + + await cb.fire_after_compact() + assert fired == [True] + + +async def test_compact_callback_exception_is_swallowed() -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_before_compact + async def bad() -> None: + raise RuntimeError("boom") + + @cb.on_before_compact + async def good() -> None: + fired.append(True) + + await cb.fire_before_compact() + assert fired == [True] + + +async def test_multiple_compact_handlers_all_fired() -> None: + cb = CallbackSet() + fired: list[str] = [] + + @cb.on_before_compact + async def b1() -> None: + fired.append("before-1") + + @cb.on_before_compact + async def b2() -> None: + fired.append("before-2") + + @cb.on_after_compact + async def a1() -> None: + fired.append("after-1") + + @cb.on_after_compact + async def a2() -> None: + fired.append("after-2") + + await cb.fire_before_compact() + await cb.fire_after_compact() + assert fired == ["before-1", "before-2", "after-1", "after-2"] diff --git a/ddev/tests/ai/react/test_process.py b/ddev/tests/ai/react/test_process.py index 8424d6096d14e..07dc4fce7b3bc 100644 --- a/ddev/tests/ai/react/test_process.py +++ b/ddev/tests/ai/react/test_process.py @@ -13,8 +13,11 @@ from ddev.ai.react.callbacks import CallbackSet from ddev.ai.react.process import ReActProcess from ddev.ai.react.types import ReActResult +from ddev.ai.tools.core.registry import ToolRegistry from ddev.ai.tools.core.types import ToolResult +_TOOL_RESULT_DATA: str = "ok" + # --------------------------------------------------------------------------- # Mock helpers # --------------------------------------------------------------------------- @@ -24,9 +27,14 @@ class MockAgent(BaseAgent[Any]): """Minimal BaseAgent implementation that replays a fixed list of responses.""" def __init__(self, responses: list[AgentResponse]) -> None: - super().__init__() + super().__init__(name="mock", system_prompt="", tools=ToolRegistry([])) self._responses = iter(responses) self.send_calls: list[str | list[ToolResultMessage]] = [] + self.compact_calls: int = 0 + self.compact_preserving_turn_calls: int = 0 + self.compact_response: AgentResponse | None = None + self.compact_token_response: AgentResponse | None = None + self.reset_calls: int = 0 async def send( self, @@ -36,12 +44,24 @@ async def send( self.send_calls.append(content) return next(self._responses) + async def compact(self) -> AgentResponse | None: + self.compact_calls += 1 + return self.compact_response + + async def compact_preserving_last_turn(self) -> AgentResponse | None: + self.compact_preserving_turn_calls += 1 + return self.compact_token_response + + def reset(self) -> None: + super().reset() + self.reset_calls += 1 + class MockToolRegistry: """Minimal tool registry that always returns a configurable ToolResult.""" def __init__(self, result: ToolResult | None = None) -> None: - self._result = result or ToolResult(success=True, data="ok") + self._result = result or ToolResult(success=True, data=_TOOL_RESULT_DATA) self.run_calls: list[tuple[str, dict]] = [] async def run(self, name: str, raw: dict[str, object]) -> ToolResult: @@ -84,6 +104,8 @@ def __init__(self) -> None: self.tool_calls_seen: list[tuple[ToolCall, ToolResult, int]] = [] self.complete_results: list[ReActResult] = [] self.errors: list[BaseException] = [] + self.before_compacts: int = 0 + self.after_compacts: int = 0 self.callback_set = CallbackSet() @@ -103,6 +125,14 @@ async def _record_complete(result: ReActResult) -> None: async def _record_error(error: BaseException) -> None: self.errors.append(error) + @self.callback_set.on_before_compact + async def _record_before_compact() -> None: + self.before_compacts += 1 + + @self.callback_set.on_after_compact + async def _record_after_compact() -> None: + self.after_compacts += 1 + def make_response( stop_reason: StopReason, @@ -135,11 +165,13 @@ def make_process( agent: MockAgent, registry: MockToolRegistry | None = None, callback_sets: list[CallbackSet] | None = None, + compact_threshold_pct: float | None = None, ) -> ReActProcess: return ReActProcess( agent=agent, tool_registry=registry or MockToolRegistry(), callback_sets=callback_sets, + compact_threshold_pct=compact_threshold_pct, ) @@ -184,7 +216,7 @@ async def test_single_tool_call_executes_tool_and_returns() -> None: assert agent.send_calls[0] == "Do something" assert isinstance(agent.send_calls[1], list) assert agent.send_calls[1][0].tool_call_id == "tc_01" - assert agent.send_calls[1][0].result.data == "ok" + assert agent.send_calls[1][0].result.data == _TOOL_RESULT_DATA # --------------------------------------------------------------------------- @@ -363,6 +395,9 @@ async def test_two_callback_sets_both_notified() -> None: class ErrorAgent(BaseAgent[Any]): + def __init__(self) -> None: + super().__init__(name="error", system_prompt="", tools=ToolRegistry([])) + async def send( self, content: str | list[ToolResultMessage], allowed_tools: list[str] | None = None ) -> AgentResponse: @@ -387,6 +422,9 @@ async def test_agent_error_notifies_and_reraises() -> None: class InterruptAgent(BaseAgent[Any]): + def __init__(self) -> None: + super().__init__(name="interrupt", system_prompt="", tools=ToolRegistry([])) + async def send( self, content: str | list[ToolResultMessage], allowed_tools: list[str] | None = None ) -> AgentResponse: @@ -410,6 +448,9 @@ async def test_keyboard_interrupt_notifies_and_reraises() -> None: class CancelledAgent(BaseAgent[Any]): + def __init__(self) -> None: + super().__init__(name="cancelled", system_prompt="", tools=ToolRegistry([])) + async def send( self, content: str | list[ToolResultMessage], allowed_tools: list[str] | None = None ) -> AgentResponse: @@ -470,3 +511,120 @@ async def test_context_usage_propagated(context_usage: ContextUsage | None) -> N assert result.context_usage is context_usage assert result.final_response.usage.context_usage is context_usage + + +# --------------------------------------------------------------------------- +# reset() and compact() — delegation +# --------------------------------------------------------------------------- + + +async def test_reset_delegates_to_agent() -> None: + agent = MockAgent([]) + make_process(agent).reset() + assert agent.reset_calls == 1 + assert agent.history == [] + + +async def test_compact_delegates_to_agent_returns_zero_when_no_op() -> None: + agent = MockAgent([]) # compact_response is None — no compaction occurred + compact_in, compact_out = await make_process(agent).compact() + assert agent.compact_calls == 1 + assert compact_in == 0 + assert compact_out == 0 + + +async def test_compact_returns_tokens_when_compaction_occurs() -> None: + agent = MockAgent([]) + agent.compact_response = make_response(StopReason.END_TURN, input_tokens=40, output_tokens=15) + compact_in, compact_out = await make_process(agent).compact() + assert compact_in == 40 + assert compact_out == 15 + + +async def test_compact_fires_before_and_after_callbacks() -> None: + agent = MockAgent([]) + recorder = CallbackRecorder() + await make_process(agent, callback_sets=[recorder.callback_set]).compact() + assert recorder.before_compacts == 1 + assert recorder.after_compacts == 1 + + +# --------------------------------------------------------------------------- +# Auto-compact inside the ReAct loop +# --------------------------------------------------------------------------- + + +def make_context_usage(pct: float, window: int = 200_000) -> ContextUsage: + return ContextUsage(window_size=window, used_tokens=int(window * pct / 100)) + + +async def test_auto_compact_triggers_when_threshold_exceeded() -> None: + tc = make_tool_call() + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN, context_usage=make_context_usage(80.0)), + ] + agent = MockAgent(responses) + await make_process(agent, compact_threshold_pct=75.0).start("task") + assert agent.compact_calls == 1 + + +async def test_auto_compact_fires_callbacks() -> None: + tc = make_tool_call() + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN, context_usage=make_context_usage(80.0)), + ] + agent = MockAgent(responses) + recorder = CallbackRecorder() + await make_process(agent, callback_sets=[recorder.callback_set], compact_threshold_pct=75.0).start("task") + assert recorder.before_compacts == 1 + assert recorder.after_compacts == 1 + + +async def test_auto_compact_does_not_trigger_below_threshold() -> None: + tc = make_tool_call() + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN, context_usage=make_context_usage(50.0)), + ] + agent = MockAgent(responses) + await make_process(agent, compact_threshold_pct=75.0).start("task") + assert agent.compact_preserving_turn_calls == 0 + + +async def test_auto_compact_disabled_when_threshold_is_none() -> None: + tc = make_tool_call() + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN, context_usage=make_context_usage(99.9)), + ] + agent = MockAgent(responses) + await make_process(agent, compact_threshold_pct=None).start("task") + assert agent.compact_preserving_turn_calls == 0 + + +async def test_auto_compact_skipped_when_context_usage_is_none() -> None: + tc = make_tool_call() + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN, context_usage=None), + ] + agent = MockAgent(responses) + await make_process(agent, compact_threshold_pct=75.0).start("task") + assert agent.compact_preserving_turn_calls == 0 + + +async def test_auto_compact_tokens_included_in_result() -> None: + tc = make_tool_call() + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[tc], input_tokens=100, output_tokens=50), + make_response(StopReason.END_TURN, context_usage=make_context_usage(80.0), input_tokens=200, output_tokens=80), + ] + agent = MockAgent(responses) + agent.compact_response = make_response(StopReason.END_TURN, input_tokens=30, output_tokens=10) + + result = await make_process(agent, compact_threshold_pct=75.0).start("task") + + assert result.total_input_tokens == 100 + 200 + 30 + assert result.total_output_tokens == 50 + 80 + 10 From e0709dd456d283937bb3c9a7c31a332b91327c58 Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Thu, 16 Apr 2026 15:26:58 +0200 Subject: [PATCH 28/44] Add ddev tools for some ddev commands (#23347) * Add ddev tools for some ddev commands * Correct some bugs * Fix ddev_create to avoid interactive prompt for snake_case names * Add ddev_env_show and other little changes --- ddev/src/ddev/ai/tools/shell/ddev/__init__.py | 3 + ddev/src/ddev/ai/tools/shell/ddev/create.py | 39 +++++ .../src/ddev/ai/tools/shell/ddev/ddev_test.py | 43 +++++ ddev/src/ddev/ai/tools/shell/ddev/env_show.py | 27 +++ .../src/ddev/ai/tools/shell/ddev/env_start.py | 33 ++++ ddev/src/ddev/ai/tools/shell/ddev/env_stop.py | 27 +++ ddev/src/ddev/ai/tools/shell/ddev/env_test.py | 34 ++++ .../ai/tools/shell/ddev/release_changelog.py | 41 +++++ ddev/tests/ai/tools/shell/ddev/__init__.py | 3 + .../ai/tools/shell/ddev/test_ddev_tools.py | 165 ++++++++++++++++++ 10 files changed, 415 insertions(+) create mode 100644 ddev/src/ddev/ai/tools/shell/ddev/__init__.py create mode 100644 ddev/src/ddev/ai/tools/shell/ddev/create.py create mode 100644 ddev/src/ddev/ai/tools/shell/ddev/ddev_test.py create mode 100644 ddev/src/ddev/ai/tools/shell/ddev/env_show.py create mode 100644 ddev/src/ddev/ai/tools/shell/ddev/env_start.py create mode 100644 ddev/src/ddev/ai/tools/shell/ddev/env_stop.py create mode 100644 ddev/src/ddev/ai/tools/shell/ddev/env_test.py create mode 100644 ddev/src/ddev/ai/tools/shell/ddev/release_changelog.py create mode 100644 ddev/tests/ai/tools/shell/ddev/__init__.py create mode 100644 ddev/tests/ai/tools/shell/ddev/test_ddev_tools.py diff --git a/ddev/src/ddev/ai/tools/shell/ddev/__init__.py b/ddev/src/ddev/ai/tools/shell/ddev/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/tools/shell/ddev/create.py b/ddev/src/ddev/ai/tools/shell/ddev/create.py new file mode 100644 index 0000000000000..7e77e8ec19e0c --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/create.py @@ -0,0 +1,39 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated, Literal + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + +IntegrationType = Literal["check", "check_only", "event", "jmx", "logs", "metrics_crawler", "snmp_tile", "tile"] + + +class CreateInput(BaseToolInput): + integration: Annotated[str, Field(description="Name of the new integration (snake_case)")] + integration_type: Annotated[ + IntegrationType, + Field( + description="Template type: 'check' (standard Agent check), 'check_only' (no hatch env)," + " 'event', 'jmx', 'logs', 'metrics_crawler', 'snmp_tile', 'tile'" + ), + ] + + +class DdevCreateTool(CmdTool[CreateInput]): + """Scaffolds a new Datadog Agent integration with all boilerplate files and + directory structure. Creates a directory named after the integration (snake_case) + in the current working directory. Use before writing any integration code.""" + + timeout = 60 + + @property + def name(self) -> str: + return "ddev_create" + + def cmd(self, tool_input: CreateInput) -> list[str]: + # Capitalize to avoid ddev's islower() interactive prompt; normalize_package_name restores snake_case + name = tool_input.integration.capitalize() + return ["ddev", "--no-interactive", "create", "--type", tool_input.integration_type, "--skip-manifest", name] diff --git a/ddev/src/ddev/ai/tools/shell/ddev/ddev_test.py b/ddev/src/ddev/ai/tools/shell/ddev/ddev_test.py new file mode 100644 index 0000000000000..eccdd343fc284 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/ddev_test.py @@ -0,0 +1,43 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + + +class DdevTestInput(BaseToolInput): + integration: Annotated[str, Field(description="Integration name to test")] + lint: Annotated[bool, Field(description="Run linter / style checks only (-s / --lint)")] = False + fmt: Annotated[bool, Field(description="Fix formatting and linting errors (-fs / --fmt)")] = False + pytest_args: Annotated[ + list[str] | None, + Field(description='Extra pytest arguments passed after `--` (e.g. ["-k", "test_my_func", "-s"])'), + ] = None + + +class DdevTestTool(CmdTool[DdevTestInput]): + """Runs unit and integration tests for the given integration. Set `lint=true` + to run the linter only. Set `fmt=true` to fix formatting and linting errors. + Use `pytest_args` to pass extra pytest arguments (e.g. `["-k", "test_my_func"]`) + to run specific tests instead of the full suite.""" + + timeout = 600 + + @property + def name(self) -> str: + return "ddev_test" + + def cmd(self, tool_input: DdevTestInput) -> list[str]: + cmd = ["ddev", "--no-interactive", "test"] + if tool_input.lint: + cmd.append("-s") + if tool_input.fmt: + cmd.append("-fs") + cmd.append(tool_input.integration) + if tool_input.pytest_args: + cmd += ["--"] + tool_input.pytest_args + return cmd diff --git a/ddev/src/ddev/ai/tools/shell/ddev/env_show.py b/ddev/src/ddev/ai/tools/shell/ddev/env_show.py new file mode 100644 index 0000000000000..4ecb5ff8b0495 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/env_show.py @@ -0,0 +1,27 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + + +class EnvShowInput(BaseToolInput): + integration: Annotated[str, Field(description="Integration name")] + + +class DdevEnvShowTool(CmdTool[EnvShowInput]): + """Lists all available E2E environment names for an integration. Call this + before `ddev_env_test` or `ddev_env_start` to discover valid environment names.""" + + timeout = 30 + + @property + def name(self) -> str: + return "ddev_env_show" + + def cmd(self, tool_input: EnvShowInput) -> list[str]: + return ["ddev", "--no-interactive", "env", "show", tool_input.integration] diff --git a/ddev/src/ddev/ai/tools/shell/ddev/env_start.py b/ddev/src/ddev/ai/tools/shell/ddev/env_start.py new file mode 100644 index 0000000000000..45ba0d90a870f --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/env_start.py @@ -0,0 +1,33 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + + +class EnvStartInput(BaseToolInput): + integration: Annotated[str, Field(description="Integration name")] + environment: Annotated[str, Field(description="Environment name (e.g. py3.11-1.23)")] + dev: Annotated[bool, Field(description="Mount local check code into the container")] = False + + +class DdevEnvStartTool(CmdTool[EnvStartInput]): + """Starts a Docker-based E2E test environment for an integration. Use + `dev=true` to mount local check code into the container.""" + + timeout = 300 + + @property + def name(self) -> str: + return "ddev_env_start" + + def cmd(self, tool_input: EnvStartInput) -> list[str]: + cmd = ["ddev", "--no-interactive", "env", "start"] + if tool_input.dev: + cmd.append("--dev") + cmd += [tool_input.integration, tool_input.environment] + return cmd diff --git a/ddev/src/ddev/ai/tools/shell/ddev/env_stop.py b/ddev/src/ddev/ai/tools/shell/ddev/env_stop.py new file mode 100644 index 0000000000000..3f55a01db76bf --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/env_stop.py @@ -0,0 +1,27 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + + +class EnvStopInput(BaseToolInput): + integration: Annotated[str, Field(description="Integration name")] + environment: Annotated[str, Field(description="Environment name (e.g. py3.11-1.23)")] + + +class DdevEnvStopTool(CmdTool[EnvStopInput]): + """Stops and removes the Docker environment for the given integration and environment name.""" + + timeout = 120 + + @property + def name(self) -> str: + return "ddev_env_stop" + + def cmd(self, tool_input: EnvStopInput) -> list[str]: + return ["ddev", "--no-interactive", "env", "stop", tool_input.integration, tool_input.environment] diff --git a/ddev/src/ddev/ai/tools/shell/ddev/env_test.py b/ddev/src/ddev/ai/tools/shell/ddev/env_test.py new file mode 100644 index 0000000000000..915de5debdd41 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/env_test.py @@ -0,0 +1,34 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + + +class EnvTestInput(BaseToolInput): + integration: Annotated[str, Field(description="Integration name")] + environment: Annotated[str, Field(description="Environment name (e.g. py3.11-1.23)")] + dev: Annotated[bool, Field(description="Pass --dev flag (use if env was started with --dev)")] = False + + +class DdevEnvTestTool(CmdTool[EnvTestInput]): + """Runs E2E tests for the given integration in the specified environment. + `ddev env test` starts the environment, runs the tests, and stops it automatically — + no prior `ddev_env_start` is needed. Use `dev=true` to pass the `--dev` flag.""" + + timeout = 600 + + @property + def name(self) -> str: + return "ddev_env_test" + + def cmd(self, tool_input: EnvTestInput) -> list[str]: + cmd = ["ddev", "--no-interactive", "env", "test"] + if tool_input.dev: + cmd.append("--dev") + cmd += [tool_input.integration, tool_input.environment] + return cmd diff --git a/ddev/src/ddev/ai/tools/shell/ddev/release_changelog.py b/ddev/src/ddev/ai/tools/shell/ddev/release_changelog.py new file mode 100644 index 0000000000000..cf3e3006e7d89 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/release_changelog.py @@ -0,0 +1,41 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated, Literal + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + + +class ReleaseChangelogInput(BaseToolInput): + change_type: Annotated[ + Literal["fixed", "added", "changed"], + Field(description="Type of change: 'fixed' (patch), 'added' (minor), 'changed' (major)"), + ] + integration: Annotated[str, Field(description="Integration name")] + message: Annotated[str, Field(description="Human-readable changelog message")] + + +class DdevReleaseChangelogTool(CmdTool[ReleaseChangelogInput]): + """Creates a changelog entry file for the integration. `change_type` must be + `"fixed"` (patch bump), `"added"` (minor bump), or `"changed"` (major bump).""" + + timeout = 30 + + @property + def name(self) -> str: + return "ddev_release_changelog" + + def cmd(self, tool_input: ReleaseChangelogInput) -> list[str]: + return [ + "ddev", + "release", + "changelog", + "new", + tool_input.change_type, + tool_input.integration, + "-m", + tool_input.message, + ] diff --git a/ddev/tests/ai/tools/shell/ddev/__init__.py b/ddev/tests/ai/tools/shell/ddev/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/tools/shell/ddev/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/tools/shell/ddev/test_ddev_tools.py b/ddev/tests/ai/tools/shell/ddev/test_ddev_tools.py new file mode 100644 index 0000000000000..cc4eb72a7e182 --- /dev/null +++ b/ddev/tests/ai/tools/shell/ddev/test_ddev_tools.py @@ -0,0 +1,165 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest +from pydantic import ValidationError + +from ddev.ai.tools.shell.ddev.create import CreateInput, DdevCreateTool +from ddev.ai.tools.shell.ddev.ddev_test import DdevTestInput, DdevTestTool +from ddev.ai.tools.shell.ddev.env_show import DdevEnvShowTool, EnvShowInput +from ddev.ai.tools.shell.ddev.env_start import DdevEnvStartTool, EnvStartInput +from ddev.ai.tools.shell.ddev.env_stop import DdevEnvStopTool, EnvStopInput +from ddev.ai.tools.shell.ddev.env_test import DdevEnvTestTool, EnvTestInput +from ddev.ai.tools.shell.ddev.release_changelog import DdevReleaseChangelogTool, ReleaseChangelogInput + +# --- ddev create --- + + +def test_create_cmd_basic(): + tool = DdevCreateTool() + assert tool.cmd(CreateInput(integration="my_check", integration_type="check")) == [ + "ddev", + "--no-interactive", + "create", + "--type", + "check", + "--skip-manifest", + "My_check", + ] + + +@pytest.mark.parametrize( + "integration_type", ["check", "check_only", "event", "jmx", "logs", "metrics_crawler", "snmp_tile", "tile"] +) +def test_create_cmd_all_types(integration_type: str): + cmd = DdevCreateTool().cmd(CreateInput(integration="my_check", integration_type=integration_type)) + assert cmd[cmd.index("--type") + 1] == integration_type + + +def test_create_invalid_type_raises(): + with pytest.raises(ValidationError): + CreateInput(integration="my_check", integration_type="custom") + + +# --- ddev test --- + + +def test_ddev_test_cmd_no_flags(): + cmd = DdevTestTool().cmd(DdevTestInput(integration="mycheck")) + assert "--no-interactive" in cmd + assert "-s" not in cmd + assert "-fs" not in cmd + + +def test_ddev_test_cmd_lint_only(): + cmd = DdevTestTool().cmd(DdevTestInput(integration="mycheck", lint=True)) + assert "-s" in cmd + assert "-fs" not in cmd + + +def test_ddev_test_cmd_fmt_only(): + cmd = DdevTestTool().cmd(DdevTestInput(integration="mycheck", fmt=True)) + assert "-fs" in cmd + assert "-s" not in cmd + + +def test_ddev_test_cmd_fmt_and_lint(): + cmd = DdevTestTool().cmd(DdevTestInput(integration="mycheck", fmt=True, lint=True)) + assert "-fs" in cmd + assert "-s" in cmd + + +def test_ddev_test_cmd_integration_last(): + cmd = DdevTestTool().cmd(DdevTestInput(integration="mycheck", fmt=True, lint=True)) + assert cmd[-1] == "mycheck" + + +def test_ddev_test_cmd_pytest_args(): + cmd = DdevTestTool().cmd(DdevTestInput(integration="mycheck", pytest_args=["-k", "test_my_func", "-s"])) + separator_idx = cmd.index("--") + assert cmd[separator_idx + 1 :] == ["-k", "test_my_func", "-s"] + assert cmd[separator_idx - 1] == "mycheck" + + +def test_ddev_test_cmd_no_pytest_args_omits_separator(): + cmd = DdevTestTool().cmd(DdevTestInput(integration="mycheck")) + assert "--" not in cmd + + +# --- ddev env show --- + + +def test_env_show_cmd(): + assert DdevEnvShowTool().cmd(EnvShowInput(integration="mycheck")) == [ + "ddev", + "--no-interactive", + "env", + "show", + "mycheck", + ] + + +# --- ddev env start --- + + +@pytest.mark.parametrize( + "dev,expected", + [ + (False, ["ddev", "--no-interactive", "env", "start", "mycheck", "py3.11-1.23"]), + (True, ["ddev", "--no-interactive", "env", "start", "--dev", "mycheck", "py3.11-1.23"]), + ], +) +def test_env_start_cmd(dev, expected): + assert DdevEnvStartTool().cmd(EnvStartInput(integration="mycheck", environment="py3.11-1.23", dev=dev)) == expected + + +# --- ddev env test --- + + +@pytest.mark.parametrize( + "dev,expected", + [ + (False, ["ddev", "--no-interactive", "env", "test", "mycheck", "py3.11-1.23"]), + (True, ["ddev", "--no-interactive", "env", "test", "--dev", "mycheck", "py3.11-1.23"]), + ], +) +def test_env_test_cmd(dev, expected): + assert DdevEnvTestTool().cmd(EnvTestInput(integration="mycheck", environment="py3.11-1.23", dev=dev)) == expected + + +# --- ddev env stop --- + + +def test_env_stop_cmd(): + assert DdevEnvStopTool().cmd(EnvStopInput(integration="mycheck", environment="py3.11-1.23")) == [ + "ddev", + "--no-interactive", + "env", + "stop", + "mycheck", + "py3.11-1.23", + ] + + +# --- ddev release changelog --- + + +@pytest.mark.parametrize("change_type", ["fixed", "added", "changed"]) +def test_release_changelog_cmd_change_type(change_type: str): + cmd = DdevReleaseChangelogTool().cmd( + ReleaseChangelogInput(change_type=change_type, integration="mycheck", message="msg") + ) + assert cmd[4] == change_type + + +def test_release_changelog_cmd_message_placement(): + cmd = DdevReleaseChangelogTool().cmd( + ReleaseChangelogInput(change_type="fixed", integration="mycheck", message="Some message") + ) + assert cmd[-2] == "-m" + assert cmd[-1] == "Some message" + + +def test_release_changelog_invalid_change_type_raises(): + with pytest.raises(ValidationError): + ReleaseChangelogInput(change_type="patch", integration="mycheck", message="Some message") From 3b811efbc19001bf634f8a6c9c0ef6077e84e2cc Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Tue, 28 Apr 2026 11:50:30 +0200 Subject: [PATCH 29/44] ddev/ai: Add Phase and Orchestrator layer to the AI framework (#23353) * Phase layer implementation * Unify messages into PhaseTrigger, move filtering to should_process_message * Remove demo/ from tracking (local only) Co-Authored-By: Claude Sonnet 4.6 (1M context) * Few bugs fixed * Change programmatic_tool_calling memory writing bugs * Remove PTC from AnthropicAgent * Validate dependencies against scheduled flow phases * Add test to PhaseOrchestrator.on_initialize * Make PhaseRegistry instance-based * Replace tautological discovery test with real tmp_path-based test * skip type-validation fro orphan phases * Re-raise RunTimeError from on_finalize when phase fails * Replace assert with FlowConfigError guards and wrap checkpoint read in try/except * Fix Windows regex failure in checkpoint read error tests --------- Co-authored-by: Claude Sonnet 4.6 (1M context) --- ddev/src/ddev/ai/agent/anthropic_client.py | 6 - ddev/src/ddev/ai/phases/__init__.py | 3 + ddev/src/ddev/ai/phases/base.py | 265 ++++++++++ ddev/src/ddev/ai/phases/checkpoint.py | 51 ++ ddev/src/ddev/ai/phases/config.py | 145 +++++ ddev/src/ddev/ai/phases/messages.py | 18 + ddev/src/ddev/ai/phases/orchestrator.py | 118 +++++ ddev/src/ddev/ai/phases/template.py | 39 ++ ddev/src/ddev/ai/tools/core/registry.py | 84 +++ ddev/tests/ai/phases/__init__.py | 3 + ddev/tests/ai/phases/conftest.py | 114 ++++ ddev/tests/ai/phases/test_base.py | 588 +++++++++++++++++++++ ddev/tests/ai/phases/test_checkpoint.py | 117 ++++ ddev/tests/ai/phases/test_config.py | 297 +++++++++++ ddev/tests/ai/phases/test_orchestrator.py | 480 +++++++++++++++++ ddev/tests/ai/phases/test_template.py | 103 ++++ ddev/tests/ai/tools/core/test_registry.py | 58 ++ 17 files changed, 2483 insertions(+), 6 deletions(-) create mode 100644 ddev/src/ddev/ai/phases/__init__.py create mode 100644 ddev/src/ddev/ai/phases/base.py create mode 100644 ddev/src/ddev/ai/phases/checkpoint.py create mode 100644 ddev/src/ddev/ai/phases/config.py create mode 100644 ddev/src/ddev/ai/phases/messages.py create mode 100644 ddev/src/ddev/ai/phases/orchestrator.py create mode 100644 ddev/src/ddev/ai/phases/template.py create mode 100644 ddev/tests/ai/phases/__init__.py create mode 100644 ddev/tests/ai/phases/conftest.py create mode 100644 ddev/tests/ai/phases/test_base.py create mode 100644 ddev/tests/ai/phases/test_checkpoint.py create mode 100644 ddev/tests/ai/phases/test_config.py create mode 100644 ddev/tests/ai/phases/test_orchestrator.py create mode 100644 ddev/tests/ai/phases/test_template.py diff --git a/ddev/src/ddev/ai/agent/anthropic_client.py b/ddev/src/ddev/ai/agent/anthropic_client.py index a2efcb1c25e38..ccdf1d10983fb 100644 --- a/ddev/src/ddev/ai/agent/anthropic_client.py +++ b/ddev/src/ddev/ai/agent/anthropic_client.py @@ -14,7 +14,6 @@ DEFAULT_MODEL: Final[str] = "claude-sonnet-4-6" DEFAULT_MAX_TOKENS: Final[int] = 8192 # max tokens per response -ALLOWED_TOOL_CALLERS: Final = ["code_execution_20260120"] class AnthropicAgent(BaseAgent[MessageParam]): @@ -28,7 +27,6 @@ def __init__( name: str, model: str = DEFAULT_MODEL, max_tokens: int = DEFAULT_MAX_TOKENS, - programmatic_tool_calling: bool = False, ) -> None: """Initialize an AnthropicAgent. Args: @@ -38,14 +36,12 @@ def __init__( name: The name of the agent. model: The model to use. max_tokens: The max tokens per response. - programmatic_tool_calling: Whether to allow programmatic tool calling. """ super().__init__(name=name, system_prompt=system_prompt, tools=tools) self._client = client self._model = model self._max_tokens = max_tokens - self._programmatic_tool_calling = programmatic_tool_calling self._context_window: int | None = None async def _get_context_window(self) -> int: @@ -70,8 +66,6 @@ def _get_tool_definitions(self, allowed_tools: list[str] | None) -> list[ToolPar if allowed_tools is not None: allowed = set(allowed_tools) definitions = [d for d in definitions if d["name"] in allowed] - if not self._programmatic_tool_calling: - definitions = [{**d, "allowed_callers": ALLOWED_TOOL_CALLERS} for d in definitions] return definitions def _map_stop_reason(self, raw: str) -> StopReason: diff --git a/ddev/src/ddev/ai/phases/__init__.py b/ddev/src/ddev/ai/phases/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/phases/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/phases/base.py b/ddev/src/ddev/ai/phases/base.py new file mode 100644 index 0000000000000..152bbd3381426 --- /dev/null +++ b/ddev/src/ddev/ai/phases/base.py @@ -0,0 +1,265 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from collections.abc import Callable +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import anthropic + +from ddev.ai.agent.anthropic_client import AnthropicAgent +from ddev.ai.phases.checkpoint import CheckpointManager +from ddev.ai.phases.config import AgentConfig, CheckpointConfig, FlowConfigError, PhaseConfig, TaskConfig +from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger +from ddev.ai.phases.template import render_inline, render_prompt +from ddev.ai.react.callbacks import CallbackSet +from ddev.ai.react.process import ReActProcess +from ddev.ai.tools.core.registry import ToolRegistry +from ddev.event_bus.orchestrator import AsyncProcessor, BaseMessage + + +class PhaseRegistry: + def __init__(self) -> None: + self._registry: dict[str, type["Phase"]] = {} + + def register(self, name: str, phase_cls: type["Phase"]) -> None: + self._registry[name] = phase_cls + + def known_names(self) -> list[str]: + return sorted(self._registry) + + def get(self, name: str) -> type["Phase"]: + if name not in self._registry: + raise ValueError(f"Unknown phase type: {name!r}. Known: {self.known_names()}") + return self._registry[name] + + +def _make_memory_resolver(checkpoint_manager: CheckpointManager) -> Callable[[str], str]: + """Build a resolver that reads phase memory files on demand for template substitution.""" + + def resolve(key: str) -> str: + if key.endswith("_memory"): + return checkpoint_manager.get_memory(key.removesuffix("_memory")) + return f"" + + return resolve + + +def render_task_prompt( + task: TaskConfig, + config_dir: Path, + context: dict[str, Any], + resolver: Callable[[str], str] | None = None, +) -> str: + """Render a task prompt -- from file if prompt_path is set, inline otherwise.""" + if task.prompt_path is not None: + return render_prompt(config_dir / task.prompt_path, context, resolver) + if task.prompt is None: + raise FlowConfigError("TaskConfig must set either 'prompt' or 'prompt_path'") + return render_inline(task.prompt, context, resolver) + + +def render_memory_prompt(checkpoint: CheckpointConfig, config_dir: Path, context: dict[str, Any]) -> str: + """Render a checkpoint memory prompt -- from file if memory_prompt_path is set, inline otherwise.""" + if checkpoint.memory_prompt_path is not None: + return render_prompt(config_dir / checkpoint.memory_prompt_path, context) + if checkpoint.memory_prompt is None: + raise FlowConfigError("CheckpointConfig must set either 'memory_prompt' or 'memory_prompt_path'") + return render_inline(checkpoint.memory_prompt, context) + + +class Phase(AsyncProcessor[PhaseTrigger]): + """Concrete base for all phases. + + process_message() implements the immutable pipeline skeleton. + Override before_react(), after_react(), and run_tasks() to customize phase behaviour. + Registered in PhaseRegistry by _discover_and_register_phases() at startup. + """ + + def __init__( + self, + phase_id: str, + dependencies: list[str], + config: PhaseConfig, + agent_config: AgentConfig, + anthropic_client: anthropic.AsyncAnthropic, + checkpoint_manager: CheckpointManager, + runtime_variables: dict[str, str], + flow_variables: dict[str, str], + config_dir: Path, + callback_sets: list[CallbackSet] | None = None, + ) -> None: + super().__init__(name=phase_id) + self._phase_id = phase_id + self._dependencies = set(dependencies) + self._remaining_dependencies = set(dependencies) + self._config = config + self._agent_config = agent_config + self._anthropic_client = anthropic_client + self._checkpoint_manager = checkpoint_manager + self._runtime_variables = runtime_variables + self._flow_variables = flow_variables + self._config_dir = config_dir + self._callback_sets = callback_sets + self._started_at: datetime | None = None + self._resolver: Callable[[str], str] | None = None + self._executed = False + + def should_process_message(self, message: BaseMessage) -> bool: + if isinstance(message, PhaseTrigger): + if message.phase_id is None: + # Initial trigger — only root phases (no declared dependencies) respond + if self._dependencies: + return False + else: + # Phase-completion trigger — check dependency tracking + if message.phase_id not in self._dependencies: + return False + self._remaining_dependencies.discard(message.phase_id) + if self._remaining_dependencies: + return False + if self._executed: + return False + self._executed = True + return True + + def before_react(self) -> None: + """Called once before agent/tools are created. Override for phase-specific setup.""" + + def after_react(self) -> None: + """Called once after all tasks complete. Override for phase-specific teardown.""" + + async def run_tasks( + self, + process: ReActProcess, + context: dict[str, Any], + ) -> tuple[int, int]: + """Run the task loop. Returns (total_input_tokens, total_output_tokens). + + Override to customize task execution -- e.g. add retries, change ordering, etc. + Default implementation iterates through config.tasks sequentially. + """ + total_input = total_output = 0 + last_result = None + for task in self._config.tasks: + if last_result is not None and last_result.context_usage is not None: + if last_result.context_usage.context_pct >= self._config.context_compact_threshold_pct: + compact_in, compact_out = await process.compact() + total_input += compact_in + total_output += compact_out + prompt = render_task_prompt(task, self._config_dir, context, self._resolver) + last_result = await process.start(prompt) + total_input += last_result.total_input_tokens + total_output += last_result.total_output_tokens + return total_input, total_output + + async def process_message(self, message: PhaseTrigger) -> None: + """Full phase pipeline. Not intended to be overridden -- customise via the extension points.""" + # 1. Record start time + self._started_at = datetime.now(UTC) + + # 2. Build template context and memory resolver + context: dict[str, Any] = { + **self._flow_variables, + **self._runtime_variables, + "phase_name": self._phase_id, + "checkpoints": self._checkpoint_manager.read(), + } + self._resolver = _make_memory_resolver(self._checkpoint_manager) + + # 3. Call before_react() + self.before_react() + + # 4. Create system prompt, ToolRegistry, AnthropicAgent + system_prompt = render_prompt( + self._config_dir / "prompts" / f"{self._config.agent}.md", + context, + self._resolver, + ) + tool_registry = ToolRegistry.from_names(self._agent_config.tools) + + agent_kwargs: dict[str, Any] = {} + if self._agent_config.model is not None: + agent_kwargs["model"] = self._agent_config.model + if self._agent_config.max_tokens is not None: + agent_kwargs["max_tokens"] = self._agent_config.max_tokens + + agent = AnthropicAgent( + client=self._anthropic_client, + tools=tool_registry, + system_prompt=system_prompt, + name=self._phase_id, + **agent_kwargs, + ) + + # 5. Build ReActProcess + process = ReActProcess( + agent=agent, + tool_registry=tool_registry, + callback_sets=self._callback_sets, + ) + + # 6. Call run_tasks() + total_input, total_output = await self.run_tasks(process, context) + + # 7. Call after_react() + self.after_react() + + # 8. Write success checkpoint — task work is done + self._checkpoint_manager.write_phase_checkpoint( + self._phase_id, + { + "status": "success", + "started_at": self._started_at.isoformat(), + "finished_at": datetime.now(UTC).isoformat(), + "tokens": {"total_input": total_input, "total_output": total_output}, + }, + ) + + # 9. Best-effort memory step — failure here doesn't invalidate a completed phase + user_additions = None + if self._config.checkpoint is not None: + user_additions = render_memory_prompt(self._config.checkpoint, self._config_dir, context) + + try: + prompt = self._checkpoint_manager.build_memory_prompt(user_additions) + # allowed_tools=[] forces a text-only response + response = await agent.send(prompt, allowed_tools=[]) + total_input += response.usage.input_tokens + total_output += response.usage.output_tokens + self._checkpoint_manager.write_memory(self._phase_id, response.text) + except Exception: + pass + + async def on_success(self, message: PhaseTrigger) -> None: + """Emit PhaseTrigger to unblock dependent phases.""" + self.submit_message( + PhaseTrigger( + id=f"{self._phase_id}_finished_{message.id}", + phase_id=self._phase_id, + ) + ) + + async def on_error(self, message: PhaseTrigger, error: Exception) -> None: + """Write failed checkpoint and emit PhaseFailedMessage.""" + try: + self._checkpoint_manager.write_phase_checkpoint( + self._phase_id, + { + "status": "failed", + "started_at": self._started_at.isoformat() if self._started_at else None, + "finished_at": datetime.now(UTC).isoformat(), + "error": str(error), + }, + ) + except Exception: + pass + self.submit_message( + PhaseFailedMessage( + id=f"{self._phase_id}_failed_{message.id}", + phase_id=self._phase_id, + error=str(error), + ) + ) diff --git a/ddev/src/ddev/ai/phases/checkpoint.py b/ddev/src/ddev/ai/phases/checkpoint.py new file mode 100644 index 0000000000000..df9f093ead8fc --- /dev/null +++ b/ddev/src/ddev/ai/phases/checkpoint.py @@ -0,0 +1,51 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from pathlib import Path +from typing import Any + +import yaml + + +class CheckpointReadError(Exception): + """Raised when checkpoints.yaml exists but cannot be read or parsed.""" + + +class CheckpointManager: + """Manages checkpoints.yaml and per-phase memory files for the full pipeline.""" + + def __init__(self, path: Path) -> None: + self._path = path + + def read(self) -> dict[str, Any]: + """Return full checkpoint data, keyed by phase_id. Empty dict if file absent.""" + if not self._path.exists(): + return {} + try: + return yaml.safe_load(self._path.read_text()) or {} + except (OSError, yaml.YAMLError) as e: + raise CheckpointReadError(f"Failed to load checkpoints from {self._path}: {e}") from e + + def write_phase_checkpoint(self, phase_id: str, data: dict[str, Any]) -> None: + """Write or overwrite one phase's section in checkpoints.yaml.""" + checkpoints = self.read() + checkpoints[phase_id] = data + self._path.parent.mkdir(parents=True, exist_ok=True) + self._path.write_text(yaml.dump(checkpoints, default_flow_style=False)) + + def build_memory_prompt(self, user_additions: str | None) -> str: + """Build the memory prompt to send to the agent at the end of a phase.""" + base_prompt = "Write a brief summary of what you accomplished in this phase." + return f"{user_additions}\n\n{base_prompt}" if user_additions else base_prompt + + def write_memory(self, phase_id: str, text: str) -> None: + """Write agent-authored text to this phase's memory file ({phase_id}_memory.md).""" + memory_path = self._path.parent / f"{phase_id}_memory.md" + self._path.parent.mkdir(parents=True, exist_ok=True) + memory_path.write_text(text) + + def get_memory(self, phase_id: str) -> str: + """Return the contents of a phase's memory file, or a NOT FOUND placeholder.""" + memory_path = self._path.parent / f"{phase_id}_memory.md" + return memory_path.read_text() if memory_path.exists() else f"" diff --git a/ddev/src/ddev/ai/phases/config.py b/ddev/src/ddev/ai/phases/config.py new file mode 100644 index 0000000000000..95879bb7a8e73 --- /dev/null +++ b/ddev/src/ddev/ai/phases/config.py @@ -0,0 +1,145 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from pathlib import Path + +import yaml +from pydantic import BaseModel, ConfigDict, ValidationError, field_validator, model_validator + +from ddev.ai.tools.core.registry import ToolRegistry + + +class FlowConfigError(Exception): + """Wraps Pydantic ValidationError or YAML errors with a user-friendly message.""" + + +class TaskConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + name: str + prompt_path: Path | None = None + prompt: str | None = None + + @model_validator(mode="after") + def exactly_one_source(self) -> "TaskConfig": + if (self.prompt_path is None) == (self.prompt is None): + raise ValueError("Exactly one of 'prompt_path' or 'prompt' must be set") + return self + + +class CheckpointConfig(BaseModel): + """Optional extra instructions for the memory step. If omitted, only a summary is written.""" + + model_config = ConfigDict(extra="forbid") + memory_prompt: str | None = None + memory_prompt_path: Path | None = None + + @model_validator(mode="after") + def exactly_one_source(self) -> "CheckpointConfig": + if (self.memory_prompt is None) == (self.memory_prompt_path is None): + raise ValueError("Exactly one of 'memory_prompt' or 'memory_prompt_path' must be set") + return self + + +class AgentConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + model: str | None = None + max_tokens: int | None = None + tools: list[str] = [] + + @field_validator("tools", mode="after") + @classmethod + def tools_must_be_known(cls, tools: list[str]) -> list[str]: + unknown = set(tools) - set(ToolRegistry.available_tool_names()) + if unknown: + raise ValueError(f"Unknown tool names: {sorted(unknown)}") + return tools + + +class PhaseConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + type: str = "Phase" + agent: str + tasks: list[TaskConfig] + context_compact_threshold_pct: int = 80 + checkpoint: CheckpointConfig | None = None + + @field_validator("tasks", mode="after") + @classmethod + def at_least_one_task(cls, tasks: list[TaskConfig]) -> list[TaskConfig]: + if not tasks: + raise ValueError("A phase must have at least one task") + return tasks + + +class FlowEntry(BaseModel): + model_config = ConfigDict(extra="forbid") + phase: str + dependencies: list[str] = [] + + +class FlowConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + variables: dict[str, str] = {} + agents: dict[str, AgentConfig] + phases: dict[str, PhaseConfig] + flow: list[FlowEntry] + + @model_validator(mode="after") + def cross_references(self) -> "FlowConfig": + """Validate all cross-references between agents, phases, and dependencies.""" + scheduled = {entry.phase for entry in self.flow} + seen: set[str] = set() + for entry in self.flow: + if entry.phase in seen: + raise ValueError(f"Duplicate phase in flow: {entry.phase!r}") + seen.add(entry.phase) + if entry.phase not in self.phases: + raise ValueError(f"Flow references unknown phase: {entry.phase!r}") + for dep in entry.dependencies: + if dep not in self.phases: + raise ValueError(f"Phase {entry.phase!r} depends on unknown phase: {dep!r}") + if dep not in scheduled: + raise ValueError(f"Phase {entry.phase!r} depends on {dep!r} which is not scheduled in flow") + + for phase_id, phase in self.phases.items(): + if phase.agent not in self.agents: + raise ValueError(f"Phase {phase_id!r} references unknown agent: {phase.agent!r}") + return self + + @classmethod + def from_yaml(cls, path: Path, config_dir: Path) -> "FlowConfig": + """Load, parse, and validate flow.yaml. Raises FlowConfigError on any problem.""" + try: + raw = yaml.safe_load(path.read_text()) + except (OSError, yaml.YAMLError) as e: + raise FlowConfigError(f"Failed to load {path}: {e}") from e + + try: + config = cls.model_validate(raw) + except ValidationError as e: + raise FlowConfigError(f"Invalid flow config:\n{e}") from e + + config._validate_files(config_dir) + return config + + def _validate_files(self, config_dir: Path) -> None: + """Check all referenced files exist.""" + for agent_name in self.agents: + system_prompt = config_dir / "prompts" / f"{agent_name}.md" + if not system_prompt.exists(): + raise FlowConfigError(f"System prompt not found for agent {agent_name!r}: {system_prompt}") + + for phase_id, phase in self.phases.items(): + for i, task in enumerate(phase.tasks): + if task.prompt_path is not None: + resolved = config_dir / task.prompt_path + if not resolved.exists(): + raise FlowConfigError( + f"Phase {phase_id!r} task {i} ({task.name!r}): prompt_path not found: {resolved}" + ) + + if phase.checkpoint is not None and phase.checkpoint.memory_prompt_path is not None: + resolved = config_dir / phase.checkpoint.memory_prompt_path + if not resolved.exists(): + raise FlowConfigError(f"Phase {phase_id!r} checkpoint memory_prompt_path not found: {resolved}") diff --git a/ddev/src/ddev/ai/phases/messages.py b/ddev/src/ddev/ai/phases/messages.py new file mode 100644 index 0000000000000..1b67193a9a9d3 --- /dev/null +++ b/ddev/src/ddev/ai/phases/messages.py @@ -0,0 +1,18 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from dataclasses import dataclass + +from ddev.event_bus.orchestrator import BaseMessage + + +@dataclass +class PhaseTrigger(BaseMessage): + phase_id: str | None # None = initial pipeline start; str = the phase that just finished + + +@dataclass +class PhaseFailedMessage(BaseMessage): + phase_id: str + error: str diff --git a/ddev/src/ddev/ai/phases/orchestrator.py b/ddev/src/ddev/ai/phases/orchestrator.py new file mode 100644 index 0000000000000..3086be127dd28 --- /dev/null +++ b/ddev/src/ddev/ai/phases/orchestrator.py @@ -0,0 +1,118 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import importlib +import inspect +import logging +from pathlib import Path + +import anthropic + +from ddev.ai.phases.base import Phase, PhaseRegistry +from ddev.ai.phases.checkpoint import CheckpointManager +from ddev.ai.phases.config import FlowConfig, FlowConfigError +from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger +from ddev.ai.react.callbacks import CallbackSet +from ddev.event_bus.exceptions import FatalProcessingError +from ddev.event_bus.orchestrator import BaseMessage, EventBusOrchestrator + + +def _discover_and_register_phases( + registry: PhaseRegistry, + phases_dir: Path | None = None, + import_prefix: str = "ddev.ai.phases", +) -> None: + """Import all non-private modules in phases_dir and register Phase subclasses.""" + if phases_dir is None: + phases_dir = Path(__file__).parent + for py_file in phases_dir.glob("*.py"): + if py_file.stem.startswith("_"): + continue + try: + module = importlib.import_module(f"{import_prefix}.{py_file.stem}") + except Exception as e: + raise FlowConfigError(f"Failed to import phase module '{py_file.stem}': {e}") from e + for _, obj in inspect.getmembers(module, inspect.isclass): + if issubclass(obj, Phase) and obj.__module__ == module.__name__: + registry.register(obj.__name__, obj) + + +class PhaseOrchestrator(EventBusOrchestrator): + def __init__( + self, + flow_yaml_path: Path, + checkpoint_path: Path, + runtime_variables: dict[str, str], + anthropic_client: anthropic.AsyncAnthropic, + callback_sets: list[CallbackSet] | None = None, + grace_period: float = 10, + ) -> None: + super().__init__(logger=logging.getLogger(__name__), grace_period=grace_period) + self._flow_yaml_path = flow_yaml_path + self._checkpoint_path = checkpoint_path + self._runtime_variables = runtime_variables + self._anthropic_client = anthropic_client + self._callback_sets = callback_sets + self._phase_registry = PhaseRegistry() + self._failed_phase: str | None = None + self._failed_error: str | None = None + + async def on_initialize(self) -> None: + """Discover custom phases, parse flow.yaml, construct phases, submit PhaseTrigger.""" + config_dir = self._flow_yaml_path.parent + + _discover_and_register_phases(self._phase_registry) + + config = FlowConfig.from_yaml(self._flow_yaml_path, config_dir) + + flow_phase_ids = {entry.phase for entry in config.flow} + for phase_id, phase_config in config.phases.items(): + if phase_id not in flow_phase_ids: + self._logger.warning("Phase %r is defined but not referenced in flow — it will not run", phase_id) + continue + try: + self._phase_registry.get(phase_config.type) + except ValueError as e: + raise FlowConfigError(str(e)) from e + + checkpoint_manager = CheckpointManager(self._checkpoint_path) + + dependency_map: dict[str, list[str]] = {entry.phase: entry.dependencies for entry in config.flow} + + for entry in config.flow: + phase_id = entry.phase + phase_config = config.phases[phase_id] + agent_config = config.agents[phase_config.agent] + dependencies = dependency_map[phase_id] + + phase_cls = self._phase_registry.get(phase_config.type) + phase = phase_cls( + phase_id=phase_id, + dependencies=dependencies, + config=phase_config, + agent_config=agent_config, + anthropic_client=self._anthropic_client, + checkpoint_manager=checkpoint_manager, + runtime_variables=self._runtime_variables, + flow_variables=config.variables, + config_dir=config_dir, + callback_sets=self._callback_sets, + ) + + self.register_processor(phase, [PhaseTrigger]) + + self.submit_message(PhaseTrigger(id="start", phase_id=None)) + + async def on_message_received(self, message: BaseMessage) -> None: + """Stop the entire pipeline immediately when any phase fails.""" + if isinstance(message, PhaseFailedMessage): + self._failed_phase = message.phase_id + self._failed_error = message.error + raise FatalProcessingError(f"Phase '{message.phase_id}' failed: {message.error}") + + async def on_finalize(self, exception: Exception | None) -> None: + if self._failed_phase is not None: + raise RuntimeError( + f"Pipeline aborted: phase '{self._failed_phase}' failed: {self._failed_error or ''}" + ) diff --git a/ddev/src/ddev/ai/phases/template.py b/ddev/src/ddev/ai/phases/template.py new file mode 100644 index 0000000000000..b7143d8afca4f --- /dev/null +++ b/ddev/src/ddev/ai/phases/template.py @@ -0,0 +1,39 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from collections.abc import Callable, Iterator, Mapping +from pathlib import Path +from string import Template +from typing import Any + + +class _SafeMapping(Mapping[str, str]): + """Returns the resolver result or an UNDEFINED placeholder for missing keys instead of raising.""" + + def __init__(self, context: dict[str, Any], resolver: Callable[[str], str] | None = None) -> None: + self._context = context + self._resolver = resolver + + def __getitem__(self, key: str) -> str: + if key in self._context: + return str(self._context[key]) + if self._resolver is not None: + return self._resolver(key) + return f"" + + def __iter__(self) -> Iterator[str]: + return iter(self._context) + + def __len__(self) -> int: + return len(self._context) + + +def render_prompt(template_path: Path, context: dict[str, Any], resolver: Callable[[str], str] | None = None) -> str: + """Render a template file with the given context.""" + return Template(template_path.read_text()).substitute(_SafeMapping(context, resolver)) + + +def render_inline(prompt: str, context: dict[str, Any], resolver: Callable[[str], str] | None = None) -> str: + """Render an inline prompt string with the given context.""" + return Template(prompt).substitute(_SafeMapping(context, resolver)) diff --git a/ddev/src/ddev/ai/tools/core/registry.py b/ddev/src/ddev/ai/tools/core/registry.py index 240e969a81843..9dcbe5adf7131 100644 --- a/ddev/src/ddev/ai/tools/core/registry.py +++ b/ddev/src/ddev/ai/tools/core/registry.py @@ -7,6 +7,24 @@ from .protocol import ToolProtocol from .types import ToolResult +_TOOL_NAMES: list[str] = [ + "read_file", + "create_file", + "edit_file", + "append_file", + "grep", + "list_files", + "mkdir", + "http_get", + "ddev_create", + "ddev_test", + "ddev_env_show", + "ddev_env_start", + "ddev_env_stop", + "ddev_env_test", + "ddev_release_changelog", +] + class ToolRegistry: """Registry holding all available tools.""" @@ -14,6 +32,72 @@ class ToolRegistry: def __init__(self, tools: list[ToolProtocol]) -> None: self._tools: dict[str, ToolProtocol] = {tool.name: tool for tool in tools} + @staticmethod + def available_tool_names() -> list[str]: + """Return all tool names that from_names can resolve.""" + return list(_TOOL_NAMES) + + @classmethod + def from_names(cls, tool_names: list[str]) -> "ToolRegistry": + """Build a ToolRegistry from a list of tool name strings. + + All file-system tools in the same registry share a single FileRegistry. + """ + from ddev.ai.tools.fs.append_file import AppendFileTool + from ddev.ai.tools.fs.create_file import CreateFileTool + from ddev.ai.tools.fs.edit_file import EditFileTool + from ddev.ai.tools.fs.file_registry import FileRegistry + from ddev.ai.tools.fs.read_file import ReadFileTool + from ddev.ai.tools.http.http_get import HttpGetTool + from ddev.ai.tools.shell.ddev.create import DdevCreateTool + from ddev.ai.tools.shell.ddev.ddev_test import DdevTestTool + from ddev.ai.tools.shell.ddev.env_show import DdevEnvShowTool + from ddev.ai.tools.shell.ddev.env_start import DdevEnvStartTool + from ddev.ai.tools.shell.ddev.env_stop import DdevEnvStopTool + from ddev.ai.tools.shell.ddev.env_test import DdevEnvTestTool + from ddev.ai.tools.shell.ddev.release_changelog import DdevReleaseChangelogTool + from ddev.ai.tools.shell.grep import GrepTool + from ddev.ai.tools.shell.list_files import ListFilesTool + from ddev.ai.tools.shell.mkdir import MkdirTool + + file_registry = FileRegistry() + tools: list[ToolProtocol] = [] + for name in tool_names: + match name: + case "read_file": + tools.append(ReadFileTool(file_registry)) + case "create_file": + tools.append(CreateFileTool(file_registry)) + case "edit_file": + tools.append(EditFileTool(file_registry)) + case "append_file": + tools.append(AppendFileTool(file_registry)) + case "grep": + tools.append(GrepTool()) + case "list_files": + tools.append(ListFilesTool()) + case "mkdir": + tools.append(MkdirTool()) + case "http_get": + tools.append(HttpGetTool()) + case "ddev_create": + tools.append(DdevCreateTool()) + case "ddev_test": + tools.append(DdevTestTool()) + case "ddev_env_show": + tools.append(DdevEnvShowTool()) + case "ddev_env_start": + tools.append(DdevEnvStartTool()) + case "ddev_env_stop": + tools.append(DdevEnvStopTool()) + case "ddev_env_test": + tools.append(DdevEnvTestTool()) + case "ddev_release_changelog": + tools.append(DdevReleaseChangelogTool()) + case _: + raise ValueError(f"Unknown tool name: {name!r}") + return cls(tools) + @property def definitions(self) -> list[ToolParam]: """Return Anthropic SDK tool definitions for all registered tools.""" diff --git a/ddev/tests/ai/phases/__init__.py b/ddev/tests/ai/phases/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/phases/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/phases/conftest.py b/ddev/tests/ai/phases/conftest.py new file mode 100644 index 0000000000000..16b078d677202 --- /dev/null +++ b/ddev/tests/ai/phases/conftest.py @@ -0,0 +1,114 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import asyncio +from typing import Any + +import pytest + +from ddev.ai.agent.types import AgentResponse, ContextUsage, StopReason, TokenUsage, ToolResultMessage + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def make_response( + text: str = "", + input_tokens: int = 100, + output_tokens: int = 50, + context_pct: float | None = None, + stop_reason: StopReason = StopReason.END_TURN, +) -> AgentResponse: + context_usage = None + if context_pct is not None: + context_usage = ContextUsage(window_size=100_000, used_tokens=int(100_000 * context_pct / 100)) + return AgentResponse( + stop_reason=stop_reason, + text=text, + tool_calls=[], + usage=TokenUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_read_input_tokens=0, + cache_creation_input_tokens=0, + context_usage=context_usage, + ), + ) + + +# --------------------------------------------------------------------------- +# Mock helpers +# --------------------------------------------------------------------------- + + +class MockAgent: + """Agent mock that replays a fixed list of responses. + + Used via monkeypatch to replace AnthropicAgent in Phase tests. + """ + + def __init__(self, responses: list[AgentResponse]) -> None: + self._responses = list(responses) + self._index = 0 + self.send_calls: list[str | list[ToolResultMessage]] = [] + self.compact_call_count: int = 0 + self.name = "mock" + self._history: list[Any] = [] + + async def send( + self, + content: str | list[ToolResultMessage], + allowed_tools: list[str] | None = None, + ) -> AgentResponse: + self.send_calls.append(content) + response = self._responses[self._index] + self._index += 1 + return response + + def reset(self) -> None: + self._history = [] + + async def compact(self) -> AgentResponse | None: + self.compact_call_count += 1 + return None + + async def compact_preserving_last_turn(self) -> AgentResponse | None: + self.compact_call_count += 1 + return None + + +def make_agent_factory(mock_agent: MockAgent): + """Create a callable that replaces AnthropicAgent constructor, returning the given mock.""" + + def factory(**kwargs: Any) -> MockAgent: + mock_agent.name = kwargs.get("name", "mock") + return mock_agent + + return factory + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +def resolve_key(key: str) -> str: + """Resolver that wraps a key in 'resolved(...)' for use in template tests.""" + return f"resolved({key})" + + +@pytest.fixture +def flow_dir(tmp_path): + """Create a minimal flow directory with a system prompt.""" + prompts_dir = tmp_path / "prompts" + prompts_dir.mkdir() + (prompts_dir / "writer.md").write_text("You are a writer for ${phase_name}.") + return tmp_path + + +@pytest.fixture +def message_queue(): + """An asyncio.Queue that can be attached to a Phase for submit_message.""" + return asyncio.Queue() diff --git a/ddev/tests/ai/phases/test_base.py b/ddev/tests/ai/phases/test_base.py new file mode 100644 index 0000000000000..13973152a186b --- /dev/null +++ b/ddev/tests/ai/phases/test_base.py @@ -0,0 +1,588 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from datetime import UTC, datetime +from unittest.mock import MagicMock + +import pytest + +from ddev.ai.phases.base import Phase, _make_memory_resolver, render_memory_prompt, render_task_prompt +from ddev.ai.phases.checkpoint import CheckpointManager +from ddev.ai.phases.config import AgentConfig, CheckpointConfig, FlowConfigError, PhaseConfig, TaskConfig +from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger +from ddev.ai.tools.core.registry import ToolRegistry + +from .conftest import MockAgent, make_agent_factory, make_response, resolve_key + + +def _empty_registry_from_names(cls, names): + return ToolRegistry([]) + + +# --------------------------------------------------------------------------- +# _make_memory_resolver +# --------------------------------------------------------------------------- + + +def test_resolver_memory_suffix(tmp_path): + mgr = CheckpointManager(tmp_path / "checkpoints.yaml") + mgr.write_phase_checkpoint("x", {}) + mgr.write_memory("draft", "Draft memory content.") + resolver = _make_memory_resolver(mgr) + assert resolver("draft_memory") == "Draft memory content." + + +def test_resolver_non_memory_key(): + mgr = MagicMock() + resolver = _make_memory_resolver(mgr) + assert resolver("some_variable") == "" + mgr.get_memory.assert_not_called() + + +def test_resolver_absent_memory(tmp_path): + mgr = CheckpointManager(tmp_path / "checkpoints.yaml") + resolver = _make_memory_resolver(mgr) + assert resolver("nonexistent_memory") == "" + + +# --------------------------------------------------------------------------- +# render_task_prompt +# --------------------------------------------------------------------------- + + +def test_render_task_prompt_from_file(tmp_path): + prompt_file = tmp_path / "task.md" + prompt_file.write_text("Hello ${name}.") + task = TaskConfig(name="t1", prompt_path="task.md") + result = render_task_prompt(task, tmp_path, {"name": "Alice"}) + assert result == "Hello Alice." + + +def test_render_task_prompt_inline(): + task = TaskConfig(name="t1", prompt="Hello ${name}.") + result = render_task_prompt(task, None, {"name": "Bob"}) + assert result == "Hello Bob." + + +def test_render_task_prompt_forwards_resolver(tmp_path): + prompt_file = tmp_path / "task.md" + prompt_file.write_text("Memory: ${draft_memory}") + task = TaskConfig(name="t1", prompt_path="task.md") + result = render_task_prompt(task, tmp_path, {}, resolve_key) + assert result == "Memory: resolved(draft_memory)" + + +# --------------------------------------------------------------------------- +# render_memory_prompt +# --------------------------------------------------------------------------- + + +def test_render_memory_prompt_from_file(tmp_path): + mem_file = tmp_path / "mem.md" + mem_file.write_text("List files for ${phase_name}.") + checkpoint = CheckpointConfig(memory_prompt_path="mem.md") + result = render_memory_prompt(checkpoint, tmp_path, {"phase_name": "draft"}) + assert result == "List files for draft." + + +def test_render_memory_prompt_inline(): + checkpoint = CheckpointConfig(memory_prompt="List files for ${phase_name}.") + result = render_memory_prompt(checkpoint, None, {"phase_name": "draft"}) + assert result == "List files for draft." + + +def test_render_task_prompt_raises_when_both_unset(): + task = TaskConfig.model_construct(name="t1", prompt=None, prompt_path=None) + with pytest.raises(FlowConfigError, match="prompt"): + render_task_prompt(task, None, {}) + + +def test_render_memory_prompt_raises_when_both_unset(): + checkpoint = CheckpointConfig.model_construct(memory_prompt=None, memory_prompt_path=None) + with pytest.raises(FlowConfigError, match="memory_prompt"): + render_memory_prompt(checkpoint, None, {}) + + +# --------------------------------------------------------------------------- +# Phase helpers +# --------------------------------------------------------------------------- + + +def _make_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + *, + phase_id="p1", + dependencies=None, + tasks=None, + checkpoint=None, + agent_tools=None, + flow_variables=None, + runtime_variables=None, + context_compact_threshold_pct=80, +): + monkeypatch.setattr("ddev.ai.phases.base.AnthropicAgent", make_agent_factory(mock_agent)) + monkeypatch.setattr(ToolRegistry, "from_names", classmethod(_empty_registry_from_names)) + + config = PhaseConfig( + agent="writer", + tasks=tasks or [TaskConfig(name="t1", prompt="Do the work.")], + checkpoint=checkpoint, + context_compact_threshold_pct=context_compact_threshold_pct, + ) + agent_config = AgentConfig(tools=agent_tools or []) + checkpoint_manager = CheckpointManager(flow_dir / "checkpoints.yaml") + + phase = Phase( + phase_id=phase_id, + dependencies=dependencies or [], + config=config, + agent_config=agent_config, + anthropic_client=MagicMock(), + checkpoint_manager=checkpoint_manager, + runtime_variables=runtime_variables or {}, + flow_variables=flow_variables or {}, + config_dir=flow_dir, + callback_sets=None, + ) + phase.queue = message_queue + return phase, checkpoint_manager + + +# --------------------------------------------------------------------------- +# Phase.process_message — happy path +# --------------------------------------------------------------------------- + + +async def test_happy_path_single_task(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task done", 100, 50), # task 1 via ReActProcess + make_response("summary", 10, 5), # memory step + ] + mock_agent = MockAgent(responses) + phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + # Memory was written + assert mgr.get_memory("p1") == "summary" + + # Checkpoint was written + checkpoint = mgr.read()["p1"] + assert checkpoint["status"] == "success" + assert checkpoint["tokens"]["total_input"] == 100 + assert checkpoint["tokens"]["total_output"] == 50 + + # on_success is called by _task_wrapper, not process_message directly. + # But we verify it would work by checking the send calls. + assert len(mock_agent.send_calls) == 2 + assert mock_agent.send_calls[0] == "Do the work." + assert "Write a brief summary" in mock_agent.send_calls[1] + + +async def test_happy_path_two_tasks(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task1 done", 100, 50), + make_response("task2 done", 200, 80), + make_response("summary", 10, 5), + ] + mock_agent = MockAgent(responses) + phase, mgr = _make_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + tasks=[ + TaskConfig(name="t1", prompt="First task."), + TaskConfig(name="t2", prompt="Second task."), + ], + ) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + checkpoint = mgr.read()["p1"] + assert checkpoint["tokens"]["total_input"] == 300 + assert checkpoint["tokens"]["total_output"] == 130 + + +# --------------------------------------------------------------------------- +# Phase.process_message — memory step with checkpoint config +# --------------------------------------------------------------------------- + + +async def test_memory_step_with_checkpoint_config(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task done", 100, 50), + make_response("summary with files", 10, 5), + ] + mock_agent = MockAgent(responses) + phase, mgr = _make_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + checkpoint=CheckpointConfig(memory_prompt="Also list the files."), + ) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + # Memory prompt should include user additions + memory_prompt = mock_agent.send_calls[1] + assert "Also list the files." in memory_prompt + assert "Write a brief summary" in memory_prompt + + +async def test_memory_step_without_checkpoint_config(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task done", 100, 50), + make_response("summary", 10, 5), + ] + mock_agent = MockAgent(responses) + phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + memory_prompt = mock_agent.send_calls[1] + assert memory_prompt == "Write a brief summary of what you accomplished in this phase." + + +# --------------------------------------------------------------------------- +# Phase.process_message — context compaction between tasks +# --------------------------------------------------------------------------- + + +async def test_compact_between_tasks_when_above_threshold(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task1 done", 100, 50, context_pct=85), # above 80% threshold + make_response("task2 done", 200, 80), + make_response("summary", 10, 5), + ] + mock_agent = MockAgent(responses) + phase, mgr = _make_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + tasks=[ + TaskConfig(name="t1", prompt="First task."), + TaskConfig(name="t2", prompt="Second task."), + ], + ) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + checkpoint = mgr.read()["p1"] + assert checkpoint["status"] == "success" + assert mock_agent.compact_call_count >= 1 + + +async def test_no_compact_when_below_threshold(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task1 done", 100, 50, context_pct=50), # below 80% threshold + make_response("task2 done", 200, 80), + make_response("summary", 10, 5), + ] + mock_agent = MockAgent(responses) + phase, mgr = _make_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + tasks=[ + TaskConfig(name="t1", prompt="First task."), + TaskConfig(name="t2", prompt="Second task."), + ], + ) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + assert mgr.read()["p1"]["status"] == "success" + assert mock_agent.compact_call_count == 0 + + +# --------------------------------------------------------------------------- +# Phase.process_message — template context +# --------------------------------------------------------------------------- + + +async def test_flow_variables_in_system_prompt(flow_dir, monkeypatch, message_queue): + # System prompt references ${project} + (flow_dir / "prompts" / "writer.md").write_text("Project: ${project}") + responses = [ + make_response("done", 100, 50), + make_response("summary", 10, 5), + ] + mock_agent = MockAgent(responses) + captured_kwargs = {} + original_factory = make_agent_factory(mock_agent) + + def capturing_factory(**kwargs): + captured_kwargs.update(kwargs) + return original_factory(**kwargs) + + monkeypatch.setattr("ddev.ai.phases.base.AnthropicAgent", capturing_factory) + monkeypatch.setattr(ToolRegistry, "from_names", classmethod(_empty_registry_from_names)) + + config = PhaseConfig( + agent="writer", + tasks=[TaskConfig(name="t1", prompt="Do it.")], + ) + phase = Phase( + phase_id="p1", + dependencies=[], + config=config, + agent_config=AgentConfig(), + anthropic_client=MagicMock(), + checkpoint_manager=CheckpointManager(flow_dir / "checkpoints.yaml"), + runtime_variables={}, + flow_variables={"project": "myproj"}, + config_dir=flow_dir, + ) + phase.queue = message_queue + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert "Project: myproj" == captured_kwargs["system_prompt"] + + +async def test_runtime_variables_override_flow_variables(flow_dir, monkeypatch, message_queue): + (flow_dir / "prompts" / "writer.md").write_text("Project: ${project}") + responses = [ + make_response("done", 100, 50), + make_response("summary", 10, 5), + ] + mock_agent = MockAgent(responses) + captured_kwargs = {} + original_factory = make_agent_factory(mock_agent) + + def capturing_factory(**kwargs): + captured_kwargs.update(kwargs) + return original_factory(**kwargs) + + monkeypatch.setattr("ddev.ai.phases.base.AnthropicAgent", capturing_factory) + monkeypatch.setattr(ToolRegistry, "from_names", classmethod(_empty_registry_from_names)) + + config = PhaseConfig( + agent="writer", + tasks=[TaskConfig(name="t1", prompt="Do it.")], + ) + phase = Phase( + phase_id="p1", + dependencies=[], + config=config, + agent_config=AgentConfig(), + anthropic_client=MagicMock(), + checkpoint_manager=CheckpointManager(flow_dir / "checkpoints.yaml"), + runtime_variables={"project": "runtime_override"}, + flow_variables={"project": "flow_default"}, + config_dir=flow_dir, + ) + phase.queue = message_queue + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert captured_kwargs["system_prompt"] == "Project: runtime_override" + + +# --------------------------------------------------------------------------- +# Phase.process_message — before_react / after_react errors +# --------------------------------------------------------------------------- + + +async def test_before_react_raises_propagates(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([]) + phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + def failing_hook(): + raise RuntimeError("setup failed") + + phase.before_react = failing_hook + + with pytest.raises(RuntimeError, match="setup failed"): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + +async def test_after_react_raises_propagates(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("done", 100, 50), + ] + mock_agent = MockAgent(responses) + phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + def failing_hook(): + raise RuntimeError("teardown failed") + + phase.after_react = failing_hook + + with pytest.raises(RuntimeError, match="teardown failed"): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + +# --------------------------------------------------------------------------- +# Phase.on_success +# --------------------------------------------------------------------------- + + +async def test_on_success_emits_finished_message(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([]) + phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + await phase.on_success(PhaseTrigger(id="start", phase_id=None)) + + msg = message_queue.get_nowait() + assert isinstance(msg, PhaseTrigger) + assert msg.phase_id == "p1" + assert msg.id == "p1_finished_start" + + +# --------------------------------------------------------------------------- +# Phase.on_error +# --------------------------------------------------------------------------- + + +async def test_on_error_writes_failed_checkpoint(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([]) + phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + await phase.on_error(PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) + + checkpoint = mgr.read()["p1"] + assert checkpoint["status"] == "failed" + assert checkpoint["error"] == "boom" + assert checkpoint["started_at"] is None # not started yet + + +async def test_on_error_emits_failed_message(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([]) + phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + await phase.on_error(PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) + + msg = message_queue.get_nowait() + assert isinstance(msg, PhaseFailedMessage) + assert msg.phase_id == "p1" + assert msg.error == "boom" + + +async def test_on_error_writes_failed_checkpoint_after_start(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([]) + phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) + phase._started_at = datetime.now(UTC) + + await phase.on_error(PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) + + checkpoint = mgr.read()["p1"] + assert checkpoint["status"] == "failed" + assert checkpoint["started_at"] is not None + + +# --------------------------------------------------------------------------- +# Phase.process_message — resolver integration with memory files +# --------------------------------------------------------------------------- + + +async def test_task_prompt_resolves_memory_variable(flow_dir, monkeypatch, message_queue): + # Create a memory file for "draft" phase + mgr = CheckpointManager(flow_dir / "checkpoints.yaml") + mgr.write_phase_checkpoint("draft", {"status": "success"}) + mgr.write_memory("draft", "Created file.py") + + # Task prompt references ${draft_memory} + responses = [ + make_response("done", 100, 50), + make_response("summary", 10, 5), + ] + mock_agent = MockAgent(responses) + + monkeypatch.setattr("ddev.ai.phases.base.AnthropicAgent", make_agent_factory(mock_agent)) + monkeypatch.setattr(ToolRegistry, "from_names", classmethod(_empty_registry_from_names)) + + config = PhaseConfig( + agent="writer", + tasks=[TaskConfig(name="t1", prompt="Review: ${draft_memory}")], + ) + phase = Phase( + phase_id="review", + dependencies=[], + config=config, + agent_config=AgentConfig(), + anthropic_client=MagicMock(), + checkpoint_manager=mgr, + runtime_variables={}, + flow_variables={}, + config_dir=flow_dir, + ) + phase.queue = message_queue + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mock_agent.send_calls[0] == "Review: Created file.py" + + +# --------------------------------------------------------------------------- +# Phase.should_process_message +# --------------------------------------------------------------------------- + + +def test_should_process_returns_true_for_initial_trigger_on_root_phase(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([]) + phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + result = phase.should_process_message(PhaseTrigger(id="start", phase_id=None)) + + assert result is True + assert phase._executed is True + + +def test_should_process_returns_false_for_initial_trigger_on_dependent_phase(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([]) + phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue, dependencies=["dep1"]) + + result = phase.should_process_message(PhaseTrigger(id="start", phase_id=None)) + + assert result is False + assert phase._executed is False + + +def test_should_process_returns_false_for_unrelated_dep(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([]) + phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue, dependencies=["dep1"]) + + result = phase.should_process_message(PhaseTrigger(id="msg1", phase_id="other")) + + assert result is False + assert phase._executed is False + + +def test_should_process_returns_false_while_deps_pending(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([]) + phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue, dependencies=["dep1", "dep2"]) + + result = phase.should_process_message(PhaseTrigger(id="msg1", phase_id="dep1")) + + assert result is False + assert phase._remaining_dependencies == {"dep2"} + assert phase._executed is False + + +def test_should_process_returns_true_when_last_dep_arrives(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([]) + phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue, dependencies=["dep1", "dep2"]) + + phase.should_process_message(PhaseTrigger(id="msg1", phase_id="dep1")) + result = phase.should_process_message(PhaseTrigger(id="msg2", phase_id="dep2")) + + assert result is True + assert phase._executed is True + + +def test_should_process_returns_false_after_already_executed(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([]) + phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + phase.should_process_message(PhaseTrigger(id="start", phase_id=None)) + result = phase.should_process_message(PhaseTrigger(id="start2", phase_id=None)) + + assert result is False diff --git a/ddev/tests/ai/phases/test_checkpoint.py b/ddev/tests/ai/phases/test_checkpoint.py new file mode 100644 index 0000000000000..8685378cb1da6 --- /dev/null +++ b/ddev/tests/ai/phases/test_checkpoint.py @@ -0,0 +1,117 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import pytest + +from ddev.ai.phases.checkpoint import CheckpointManager, CheckpointReadError + + +@pytest.fixture +def manager(tmp_path) -> CheckpointManager: + return CheckpointManager(tmp_path / "checkpoints.yaml") + + +# --------------------------------------------------------------------------- +# read +# --------------------------------------------------------------------------- + + +def test_read_returns_empty_when_file_absent(manager): + assert manager.read() == {} + + +def test_read_returns_empty_when_file_is_empty(manager): + manager._path.write_text("") + assert manager.read() == {} + + +def test_read_malformed_yaml_raises_checkpoint_read_error(manager): + manager._path.write_text(": :\n -[") + with pytest.raises(CheckpointReadError, match="checkpoints.yaml"): + manager.read() + + +def test_read_unreadable_file_raises_checkpoint_read_error(manager, monkeypatch): + manager._path.write_text("phase1:\n status: success\n") + monkeypatch.setattr("pathlib.Path.read_text", lambda *_: (_ for _ in ()).throw(OSError("permission denied"))) + with pytest.raises(CheckpointReadError, match="checkpoints.yaml"): + manager.read() + + +# --------------------------------------------------------------------------- +# write_phase_checkpoint +# --------------------------------------------------------------------------- + + +def test_write_and_read_back(manager): + manager.write_phase_checkpoint("phase1", {"status": "success", "tokens": 100}) + data = manager.read() + assert data["phase1"]["status"] == "success" + assert data["phase1"]["tokens"] == 100 + + +def test_write_creates_parent_dirs(tmp_path): + manager = CheckpointManager(tmp_path / "nested" / "dir" / "checkpoints.yaml") + manager.write_phase_checkpoint("p", {"status": "success"}) + assert manager.read()["p"]["status"] == "success" + + +def test_write_multiple_phases(manager): + manager.write_phase_checkpoint("phase1", {"status": "success"}) + manager.write_phase_checkpoint("phase2", {"status": "failed"}) + data = manager.read() + assert data["phase1"]["status"] == "success" + assert data["phase2"]["status"] == "failed" + + +def test_write_overwrites_existing_phase(manager): + manager.write_phase_checkpoint("phase1", {"status": "running"}) + manager.write_phase_checkpoint("phase1", {"status": "success"}) + assert manager.read()["phase1"]["status"] == "success" + + +# --------------------------------------------------------------------------- +# build_memory_prompt +# --------------------------------------------------------------------------- + + +def test_build_memory_prompt_no_additions(manager): + result = manager.build_memory_prompt(None) + assert result == "Write a brief summary of what you accomplished in this phase." + + +def test_build_memory_prompt_with_additions(manager): + result = manager.build_memory_prompt("Also list the files you created.") + assert result.startswith("Also list the files you created.") + assert "Write a brief summary" in result + + +# --------------------------------------------------------------------------- +# write_memory / get_memory +# --------------------------------------------------------------------------- + + +def test_write_memory_and_read_back(manager): + manager.write_phase_checkpoint("p", {}) # ensure parent dir exists + manager.write_memory("draft", "Created integration.py and tests.") + assert manager.get_memory("draft") == "Created integration.py and tests." + + +def test_write_memory_overwrites(manager): + manager.write_phase_checkpoint("p", {}) + manager.write_memory("draft", "first version") + manager.write_memory("draft", "second version") + assert manager.get_memory("draft") == "second version" + + +def test_get_memory_absent_returns_placeholder(manager): + assert manager.get_memory("nonexistent") == "" + + +def test_memory_file_location(manager): + manager.write_phase_checkpoint("p", {}) + manager.write_memory("phase1", "content") + expected_path = manager._path.parent / "phase1_memory.md" + assert expected_path.exists() + assert expected_path.read_text() == "content" diff --git a/ddev/tests/ai/phases/test_config.py b/ddev/tests/ai/phases/test_config.py new file mode 100644 index 0000000000000..f0cc845dde704 --- /dev/null +++ b/ddev/tests/ai/phases/test_config.py @@ -0,0 +1,297 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import pytest +from pydantic import ValidationError + +from ddev.ai.phases.config import ( + AgentConfig, + CheckpointConfig, + FlowConfig, + FlowConfigError, + PhaseConfig, + TaskConfig, +) + +# --------------------------------------------------------------------------- +# TaskConfig +# --------------------------------------------------------------------------- + + +def test_task_config_with_prompt(): + tc = TaskConfig(name="t1", prompt="Do it.") + assert tc.prompt == "Do it." + assert tc.prompt_path is None + + +def test_task_config_with_prompt_path(): + tc = TaskConfig(name="t1", prompt_path="prompts/task.md") + assert tc.prompt is None + assert tc.prompt_path is not None + + +def test_task_config_both_set_raises(): + with pytest.raises(ValidationError, match="Exactly one"): + TaskConfig(name="t1", prompt="Do it.", prompt_path="prompts/task.md") + + +def test_task_config_neither_set_raises(): + with pytest.raises(ValidationError, match="Exactly one"): + TaskConfig(name="t1") + + +def test_task_config_extra_field_raises(): + with pytest.raises(ValidationError, match="extra"): + TaskConfig(name="t1", prompt="Do it.", unknown_field="x") + + +# --------------------------------------------------------------------------- +# CheckpointConfig +# --------------------------------------------------------------------------- + + +def test_checkpoint_config_with_memory_prompt(): + cc = CheckpointConfig(memory_prompt="List files.") + assert cc.memory_prompt == "List files." + + +def test_checkpoint_config_with_memory_prompt_path(): + cc = CheckpointConfig(memory_prompt_path="prompts/mem.md") + assert cc.memory_prompt_path is not None + + +def test_checkpoint_config_both_set_raises(): + with pytest.raises(ValidationError, match="Exactly one"): + CheckpointConfig(memory_prompt="List files.", memory_prompt_path="prompts/mem.md") + + +def test_checkpoint_config_neither_set_raises(): + with pytest.raises(ValidationError, match="Exactly one"): + CheckpointConfig() + + +# --------------------------------------------------------------------------- +# AgentConfig +# --------------------------------------------------------------------------- + + +def test_agent_config_valid_tools(): + ac = AgentConfig(tools=["read_file", "grep"]) + assert ac.tools == ["read_file", "grep"] + + +def test_agent_config_unknown_tool_raises(): + with pytest.raises(ValidationError, match="Unknown tool names"): + AgentConfig(tools=["read_file", "teleport"]) + + +def test_agent_config_empty_tools(): + ac = AgentConfig() + assert ac.tools == [] + + +def test_agent_config_optional_fields(): + ac = AgentConfig(model="claude-opus-4-5", max_tokens=4096) + assert ac.model == "claude-opus-4-5" + assert ac.max_tokens == 4096 + + +# --------------------------------------------------------------------------- +# PhaseConfig +# --------------------------------------------------------------------------- + + +def test_phase_config_defaults(): + pc = PhaseConfig(agent="writer", tasks=[TaskConfig(name="t1", prompt="Do it.")]) + assert pc.type == "Phase" + assert pc.context_compact_threshold_pct == 80 + assert pc.checkpoint is None + + +def test_phase_config_empty_tasks_raises(): + with pytest.raises(ValidationError, match="at least one task"): + PhaseConfig(agent="writer", tasks=[]) + + +def test_phase_config_with_checkpoint(): + pc = PhaseConfig( + agent="writer", + tasks=[TaskConfig(name="t1", prompt="Do it.")], + checkpoint=CheckpointConfig(memory_prompt="List files."), + ) + assert pc.checkpoint is not None + + +# --------------------------------------------------------------------------- +# FlowConfig cross-reference validation +# --------------------------------------------------------------------------- + + +def _minimal_config(**overrides) -> dict: + base = { + "agents": {"writer": {"tools": []}}, + "phases": {"p1": {"agent": "writer", "tasks": [{"name": "t1", "prompt": "Do it."}]}}, + "flow": [{"phase": "p1"}], + } + base.update(overrides) + return base + + +def test_flow_config_minimal_valid(): + config = FlowConfig.model_validate(_minimal_config()) + assert "p1" in config.phases + + +def test_flow_config_unknown_phase_in_flow(): + raw = _minimal_config() + raw["flow"] = [{"phase": "nonexistent"}] + with pytest.raises(ValidationError, match="unknown phase"): + FlowConfig.model_validate(raw) + + +def test_flow_config_unknown_dependency(): + raw = _minimal_config() + raw["flow"] = [{"phase": "p1", "dependencies": ["nonexistent"]}] + with pytest.raises(ValidationError, match="unknown phase"): + FlowConfig.model_validate(raw) + + +def test_flow_config_dependency_not_scheduled_in_flow(): + raw = { + "agents": {"writer": {"tools": []}}, + "phases": { + "p1": {"agent": "writer", "tasks": [{"name": "t1", "prompt": "Do it."}]}, + "p2": {"agent": "writer", "tasks": [{"name": "t2", "prompt": "Review it."}]}, + }, + "flow": [{"phase": "p2", "dependencies": ["p1"]}], + } + with pytest.raises(ValidationError, match="not scheduled in flow"): + FlowConfig.model_validate(raw) + + +def test_flow_config_unknown_agent_in_phase(): + raw = _minimal_config() + raw["phases"]["p1"]["agent"] = "nonexistent" + with pytest.raises(ValidationError, match="unknown agent"): + FlowConfig.model_validate(raw) + + +def test_flow_config_with_variables(): + raw = _minimal_config(variables={"project": "myproj"}) + config = FlowConfig.model_validate(raw) + assert config.variables["project"] == "myproj" + + +def test_flow_config_multiple_phases_and_deps(): + raw = { + "agents": {"writer": {"tools": []}}, + "phases": { + "p1": {"agent": "writer", "tasks": [{"name": "t1", "prompt": "Do it."}]}, + "p2": {"agent": "writer", "tasks": [{"name": "t2", "prompt": "Review it."}]}, + }, + "flow": [ + {"phase": "p1"}, + {"phase": "p2", "dependencies": ["p1"]}, + ], + } + config = FlowConfig.model_validate(raw) + assert len(config.flow) == 2 + assert config.flow[1].dependencies == ["p1"] + + +def test_flow_config_extra_field_raises(): + raw = _minimal_config() + raw["extra"] = "boom" + with pytest.raises(ValidationError, match="extra"): + FlowConfig.model_validate(raw) + + +# --------------------------------------------------------------------------- +# FlowConfig.from_yaml +# --------------------------------------------------------------------------- + + +def test_from_yaml_valid(tmp_path): + prompts_dir = tmp_path / "prompts" + prompts_dir.mkdir() + (prompts_dir / "writer.md").write_text("system prompt") + + flow_yaml = tmp_path / "flow.yaml" + flow_yaml.write_text( + """\ +agents: + writer: + tools: [] +phases: + p1: + agent: writer + tasks: + - name: t1 + prompt: "Do it." +flow: + - phase: p1 +""" + ) + config = FlowConfig.from_yaml(flow_yaml, tmp_path) + assert "p1" in config.phases + + +def test_from_yaml_missing_system_prompt(tmp_path): + (tmp_path / "prompts").mkdir() + + flow_yaml = tmp_path / "flow.yaml" + flow_yaml.write_text( + """\ +agents: + writer: + tools: [] +phases: + p1: + agent: writer + tasks: + - name: t1 + prompt: "Do it." +flow: + - phase: p1 +""" + ) + with pytest.raises(FlowConfigError, match="System prompt not found"): + FlowConfig.from_yaml(flow_yaml, tmp_path) + + +def test_from_yaml_missing_task_prompt_path(tmp_path): + prompts_dir = tmp_path / "prompts" + prompts_dir.mkdir() + (prompts_dir / "writer.md").write_text("system prompt") + + flow_yaml = tmp_path / "flow.yaml" + flow_yaml.write_text( + """\ +agents: + writer: + tools: [] +phases: + p1: + agent: writer + tasks: + - name: t1 + prompt_path: prompts/nonexistent.md +flow: + - phase: p1 +""" + ) + with pytest.raises(FlowConfigError, match="prompt_path not found"): + FlowConfig.from_yaml(flow_yaml, tmp_path) + + +def test_from_yaml_invalid_yaml(tmp_path): + flow_yaml = tmp_path / "flow.yaml" + flow_yaml.write_text(": invalid: yaml: [") + with pytest.raises(FlowConfigError, match="Failed to load"): + FlowConfig.from_yaml(flow_yaml, tmp_path) + + +def test_from_yaml_missing_file(tmp_path): + with pytest.raises(FlowConfigError, match="Failed to load"): + FlowConfig.from_yaml(tmp_path / "nonexistent.yaml", tmp_path) diff --git a/ddev/tests/ai/phases/test_orchestrator.py b/ddev/tests/ai/phases/test_orchestrator.py new file mode 100644 index 0000000000000..ad96179dbf2f9 --- /dev/null +++ b/ddev/tests/ai/phases/test_orchestrator.py @@ -0,0 +1,480 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from pathlib import Path +from textwrap import dedent +from unittest.mock import MagicMock + +import pytest + +from ddev.ai.phases.base import Phase, PhaseRegistry +from ddev.ai.phases.config import FlowConfigError +from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger +from ddev.ai.phases.orchestrator import PhaseOrchestrator, _discover_and_register_phases +from ddev.event_bus.exceptions import FatalProcessingError + +# --------------------------------------------------------------------------- +# _discover_and_register_phases +# --------------------------------------------------------------------------- + + +def test_discover_registers_phase_itself(): + registry = PhaseRegistry() + _discover_and_register_phases(registry) + assert "Phase" in registry.known_names() + assert registry.get("Phase") is Phase + + +def test_discover_registers_custom_subclass(tmp_path, monkeypatch): + """Discovery imports a real .py file and registers the Phase subclass it defines.""" + fake_dir = tmp_path / "fake_phases" + fake_dir.mkdir() + (fake_dir / "__init__.py").write_text("") + (fake_dir / "custom.py").write_text("from ddev.ai.phases.base import Phase\nclass CustomPhase(Phase):\n pass\n") + monkeypatch.syspath_prepend(str(tmp_path)) + + registry = PhaseRegistry() + _discover_and_register_phases(registry, phases_dir=fake_dir, import_prefix="fake_phases") + + assert "CustomPhase" in registry.known_names() + assert issubclass(registry.get("CustomPhase"), Phase) + + +def test_discover_ignores_module_without_phase_subclass(tmp_path, monkeypatch): + fake_dir = tmp_path / "no_phase_pkg" + fake_dir.mkdir() + (fake_dir / "__init__.py").write_text("") + (fake_dir / "helpers.py").write_text("CONSTANT = 42\n") + monkeypatch.syspath_prepend(str(tmp_path)) + + registry = PhaseRegistry() + _discover_and_register_phases(registry, phases_dir=fake_dir, import_prefix="no_phase_pkg") + + assert registry.known_names() == [] + + +def test_discover_does_not_register_imported_phase_class(tmp_path, monkeypatch): + """A module that imports Phase but defines no subclass should not register Phase itself.""" + fake_dir = tmp_path / "importer_pkg" + fake_dir.mkdir() + (fake_dir / "__init__.py").write_text("") + (fake_dir / "importer.py").write_text("from ddev.ai.phases.base import Phase\n") + monkeypatch.syspath_prepend(str(tmp_path)) + + registry = PhaseRegistry() + _discover_and_register_phases(registry, phases_dir=fake_dir, import_prefix="importer_pkg") + + assert "Phase" not in registry.known_names() + + +def test_discover_skips_underscore_prefixed_files(): + """After discovery, only non-underscore files are imported. + __init__.py is underscore-prefixed and is skipped.""" + registry = PhaseRegistry() + _discover_and_register_phases(registry) + assert "Phase" in registry.known_names() + + +def test_discover_idempotent(): + registry = PhaseRegistry() + _discover_and_register_phases(registry) + first = registry.known_names() + _discover_and_register_phases(registry) + second = registry.known_names() + assert first == second + + +def test_registry_get_unknown_raises(): + registry = PhaseRegistry() + with pytest.raises(ValueError, match="Unknown phase type"): + registry.get("NonexistentPhase") + + +def test_imported_class_not_registered(): + """A class imported into a phases module but defined elsewhere should not be registered.""" + registry = PhaseRegistry() + _discover_and_register_phases(registry) + # BaseMessage is imported in messages.py but defined in event_bus — it should NOT be registered + assert "BaseMessage" not in registry.known_names() + + +def test_two_orchestrators_have_independent_registries(tmp_path): + """Each PhaseOrchestrator owns its own registry; registering in one does not affect the other.""" + o1 = PhaseOrchestrator( + flow_yaml_path=tmp_path / "flow.yaml", + checkpoint_path=tmp_path / "checkpoints.yaml", + runtime_variables={}, + anthropic_client=MagicMock(), + ) + o2 = PhaseOrchestrator( + flow_yaml_path=tmp_path / "flow.yaml", + checkpoint_path=tmp_path / "checkpoints.yaml", + runtime_variables={}, + anthropic_client=MagicMock(), + ) + + class ExclusivePhase(Phase): + pass + + o1._phase_registry.register("ExclusivePhase", ExclusivePhase) + assert "ExclusivePhase" in o1._phase_registry.known_names() + assert "ExclusivePhase" not in o2._phase_registry.known_names() + + +def test_discover_does_not_mutate_global_state(): + """_discover_and_register_phases only touches the registry passed to it.""" + registry = PhaseRegistry() + _discover_and_register_phases(registry) + # No module-level / class-level container should have been touched. + # Verify by checking there is no class-level _registry attribute on PhaseRegistry. + assert not hasattr(PhaseRegistry, "_registry") + + +# --------------------------------------------------------------------------- +# PhaseOrchestrator.on_message_received +# --------------------------------------------------------------------------- + + +async def test_on_message_received_fatal_on_phase_failed(): + orchestrator = PhaseOrchestrator( + flow_yaml_path=Path("/fake/flow.yaml"), + checkpoint_path=Path("/fake/checkpoints.yaml"), + runtime_variables={}, + anthropic_client=MagicMock(), + ) + msg = PhaseFailedMessage(id="f1", phase_id="p1", error="something broke") + + with pytest.raises(FatalProcessingError, match="Phase 'p1' failed"): + await orchestrator.on_message_received(msg) + + +async def test_on_message_received_ignores_other_messages(): + orchestrator = PhaseOrchestrator( + flow_yaml_path=Path("/fake/flow.yaml"), + checkpoint_path=Path("/fake/checkpoints.yaml"), + runtime_variables={}, + anthropic_client=MagicMock(), + ) + # These should not raise + await orchestrator.on_message_received(PhaseTrigger(id="start", phase_id=None)) + await orchestrator.on_message_received(PhaseTrigger(id="f1", phase_id="p1")) + + +# --------------------------------------------------------------------------- +# PhaseOrchestrator.on_initialize +# --------------------------------------------------------------------------- + + +@pytest.fixture +def minimal_flow(tmp_path): + """Two-phase flow: 'a' is root, 'b' depends on 'a'.""" + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + a: + type: Phase + agent: writer + tasks: + - name: task_a + prompt: task a + b: + type: Phase + agent: writer + tasks: + - name: task_b + prompt: task b + flow: + - phase: a + - phase: b + dependencies: [a] + """) + ) + return tmp_path + + +async def test_on_initialize_registers_all_flow_phases(minimal_flow): + orchestrator = PhaseOrchestrator( + flow_yaml_path=minimal_flow / "flow.yaml", + checkpoint_path=minimal_flow / "checkpoints.yaml", + runtime_variables={}, + anthropic_client=MagicMock(), + ) + await orchestrator.on_initialize() + + processors = orchestrator._subscribers.get(PhaseTrigger, []) + phase_names = {p.name for p in processors} + assert phase_names == {"a", "b"} + + +async def test_on_initialize_wires_dependencies(minimal_flow): + orchestrator = PhaseOrchestrator( + flow_yaml_path=minimal_flow / "flow.yaml", + checkpoint_path=minimal_flow / "checkpoints.yaml", + runtime_variables={}, + anthropic_client=MagicMock(), + ) + await orchestrator.on_initialize() + + processors = orchestrator._subscribers.get(PhaseTrigger, []) + phases_by_name = {p.name: p for p in processors} + assert phases_by_name["a"]._dependencies == set() + assert phases_by_name["b"]._dependencies == {"a"} + + +async def test_on_initialize_submits_initial_phase_trigger(minimal_flow): + orchestrator = PhaseOrchestrator( + flow_yaml_path=minimal_flow / "flow.yaml", + checkpoint_path=minimal_flow / "checkpoints.yaml", + runtime_variables={}, + anthropic_client=MagicMock(), + ) + await orchestrator.on_initialize() + + assert not orchestrator._queue.empty() + msg = orchestrator._queue.get_nowait() + assert isinstance(msg, PhaseTrigger) + assert msg.phase_id is None + + +async def test_on_initialize_unknown_phase_type_raises_flow_config_error(tmp_path): + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + a: + type: NotARealPhase + agent: writer + tasks: + - name: task_a + prompt: task a + flow: + - phase: a + """) + ) + orchestrator = PhaseOrchestrator( + flow_yaml_path=tmp_path / "flow.yaml", + checkpoint_path=tmp_path / "checkpoints.yaml", + runtime_variables={}, + anthropic_client=MagicMock(), + ) + with pytest.raises(FlowConfigError, match="Unknown phase type"): + await orchestrator.on_initialize() + + +async def test_on_initialize_missing_agent_raises(tmp_path): + (tmp_path / "prompts").mkdir() + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + a: + type: Phase + agent: nonexistent_agent + tasks: + - name: task_a + prompt: task a + flow: + - phase: a + """) + ) + orchestrator = PhaseOrchestrator( + flow_yaml_path=tmp_path / "flow.yaml", + checkpoint_path=tmp_path / "checkpoints.yaml", + runtime_variables={}, + anthropic_client=MagicMock(), + ) + with pytest.raises(FlowConfigError): + await orchestrator.on_initialize() + + +# --------------------------------------------------------------------------- +# PhaseOrchestrator.on_initialize — orphan-phase validation +# --------------------------------------------------------------------------- + + +async def test_orphan_phase_with_unknown_type_does_not_block_init(tmp_path): + """A phase defined in phases: but absent from flow: may have an unknown type — no error.""" + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + real: + type: Phase + agent: writer + tasks: + - name: t1 + prompt: do it + orphan: + type: BogusType + agent: writer + tasks: + - name: t2 + prompt: ignored + flow: + - phase: real + """) + ) + orchestrator = PhaseOrchestrator( + flow_yaml_path=tmp_path / "flow.yaml", + checkpoint_path=tmp_path / "checkpoints.yaml", + runtime_variables={}, + anthropic_client=MagicMock(), + ) + await orchestrator.on_initialize() + + processors = orchestrator._subscribers.get(PhaseTrigger, []) + assert {p.name for p in processors} == {"real"} + + +async def test_phase_in_flow_with_unknown_type_raises(tmp_path): + """A phase referenced from flow: with an unknown type must still raise FlowConfigError.""" + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + a: + type: NotARealPhase + agent: writer + tasks: + - name: t1 + prompt: do it + flow: + - phase: a + """) + ) + orchestrator = PhaseOrchestrator( + flow_yaml_path=tmp_path / "flow.yaml", + checkpoint_path=tmp_path / "checkpoints.yaml", + runtime_variables={}, + anthropic_client=MagicMock(), + ) + with pytest.raises(FlowConfigError, match="Unknown phase type"): + await orchestrator.on_initialize() + + +async def test_orphan_phase_logs_warning(tmp_path, caplog): + """An orphan phase must emit a warning containing its phase id.""" + import logging + + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + real: + type: Phase + agent: writer + tasks: + - name: t1 + prompt: do it + orphan: + type: Phase + agent: writer + tasks: + - name: t2 + prompt: ignored + flow: + - phase: real + """) + ) + orchestrator = PhaseOrchestrator( + flow_yaml_path=tmp_path / "flow.yaml", + checkpoint_path=tmp_path / "checkpoints.yaml", + runtime_variables={}, + anthropic_client=MagicMock(), + ) + with caplog.at_level(logging.WARNING): + await orchestrator.on_initialize() + + assert any("orphan" in record.message for record in caplog.records) + + +# --------------------------------------------------------------------------- +# PhaseOrchestrator.on_finalize +# --------------------------------------------------------------------------- + + +async def test_on_finalize_no_failure_is_noop(): + orchestrator = PhaseOrchestrator( + flow_yaml_path=Path("/fake/flow.yaml"), + checkpoint_path=Path("/fake/checkpoints.yaml"), + runtime_variables={}, + anthropic_client=MagicMock(), + ) + await orchestrator.on_finalize(None) # must not raise + + +async def test_on_finalize_after_phase_failed_raises(): + orchestrator = PhaseOrchestrator( + flow_yaml_path=Path("/fake/flow.yaml"), + checkpoint_path=Path("/fake/checkpoints.yaml"), + runtime_variables={}, + anthropic_client=MagicMock(), + ) + msg = PhaseFailedMessage(id="f1", phase_id="p1", error="boom") + with pytest.raises(FatalProcessingError): + await orchestrator.on_message_received(msg) + + with pytest.raises(RuntimeError, match="Pipeline aborted.*p1.*boom"): + await orchestrator.on_finalize(None) + + +def test_run_raises_runtime_error_when_phase_fails(tmp_path): + """Full pipeline: a failing phase must cause run() to raise RuntimeError.""" + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + failing: + type: FailingPhase + agent: writer + tasks: + - name: t1 + prompt: do it + flow: + - phase: failing + """) + ) + + class FailingPhase(Phase): + async def process_message(self, message: PhaseTrigger) -> None: + raise RuntimeError("intentional failure") + + orchestrator = PhaseOrchestrator( + flow_yaml_path=tmp_path / "flow.yaml", + checkpoint_path=tmp_path / "checkpoints.yaml", + runtime_variables={}, + anthropic_client=MagicMock(), + grace_period=0.1, + ) + orchestrator._phase_registry.register("FailingPhase", FailingPhase) + + with pytest.raises(RuntimeError, match="Pipeline aborted"): + orchestrator.run() diff --git a/ddev/tests/ai/phases/test_template.py b/ddev/tests/ai/phases/test_template.py new file mode 100644 index 0000000000000..f7cf0f9ded088 --- /dev/null +++ b/ddev/tests/ai/phases/test_template.py @@ -0,0 +1,103 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from ddev.ai.phases.template import _SafeMapping, render_inline, render_prompt + +from .conftest import resolve_key + +# --------------------------------------------------------------------------- +# _SafeMapping +# --------------------------------------------------------------------------- + + +def test_safe_mapping_key_in_context(): + mapping = _SafeMapping({"name": "Alice"}) + assert mapping["name"] == "Alice" + + +def test_safe_mapping_key_absent_with_resolver(): + mapping = _SafeMapping({}, resolve_key) + assert mapping["missing"] == "resolved(missing)" + + +def test_safe_mapping_key_absent_no_resolver(): + mapping = _SafeMapping({}) + assert mapping["missing"] == "" + + +def test_safe_mapping_context_takes_precedence_over_resolver(): + def resolver(key): + return "from_resolver" + + mapping = _SafeMapping({"key": "from_context"}, resolver) + assert mapping["key"] == "from_context" + + +def test_safe_mapping_non_string_value_converted(): + mapping = _SafeMapping({"count": 42}) + assert mapping["count"] == "42" + + +# --------------------------------------------------------------------------- +# render_prompt +# --------------------------------------------------------------------------- + + +def test_render_prompt_substitutes_variables(tmp_path): + template = tmp_path / "prompt.md" + template.write_text("Hello ${name}, you are ${role}.") + result = render_prompt(template, {"name": "Alice", "role": "writer"}) + assert result == "Hello Alice, you are writer." + + +def test_render_prompt_missing_variable_shows_placeholder(tmp_path): + template = tmp_path / "prompt.md" + template.write_text("Hello ${name}.") + result = render_prompt(template, {}) + assert result == "Hello ." + + +def test_render_prompt_uses_resolver(tmp_path): + template = tmp_path / "prompt.md" + template.write_text("Memory: ${draft_memory}") + result = render_prompt(template, {}, resolve_key) + assert result == "Memory: resolved(draft_memory)" + + +def test_render_prompt_resolver_not_called_when_key_in_context(tmp_path): + called = [] + template = tmp_path / "prompt.md" + template.write_text("Value: ${key}") + + def resolver(k): + called.append(k) + return "nope" + + render_prompt(template, {"key": "from_context"}, resolver) + assert called == [] + + +# --------------------------------------------------------------------------- +# render_inline +# --------------------------------------------------------------------------- + + +def test_render_inline_substitutes_variables(): + result = render_inline("Hello ${name}.", {"name": "Bob"}) + assert result == "Hello Bob." + + +def test_render_inline_missing_variable_shows_placeholder(): + result = render_inline("Hello ${name}.", {}) + assert result == "Hello ." + + +def test_render_inline_uses_resolver(): + result = render_inline("Memory: ${draft_memory}", {}, resolve_key) + assert result == "Memory: resolved(draft_memory)" + + +def test_render_inline_escaped_dollar(): + result = render_inline("Price: $$5", {}) + assert result == "Price: $5" diff --git a/ddev/tests/ai/tools/core/test_registry.py b/ddev/tests/ai/tools/core/test_registry.py index 1366a9d8b5be8..b770867b909f4 100644 --- a/ddev/tests/ai/tools/core/test_registry.py +++ b/ddev/tests/ai/tools/core/test_registry.py @@ -125,3 +125,61 @@ async def test_empty_registry_always_returns_unknown_error(): result = await registry.run("anything", {}) assert result.success is False assert result.error is not None + + +# --------------------------------------------------------------------------- +# ToolRegistry.available_tool_names +# --------------------------------------------------------------------------- + + +def test_available_tool_names_returns_non_empty_list(): + names = ToolRegistry.available_tool_names() + assert isinstance(names, list) + assert len(names) > 0 + + +def test_available_tool_names_returns_fresh_copy(): + a = ToolRegistry.available_tool_names() + b = ToolRegistry.available_tool_names() + assert a == b + assert a is not b + + +# --------------------------------------------------------------------------- +# ToolRegistry.from_names +# --------------------------------------------------------------------------- + + +def test_from_names_empty(): + registry = ToolRegistry.from_names([]) + assert registry.definitions == [] + + +def test_from_names_unknown_raises(): + with pytest.raises(ValueError, match="Unknown tool name: 'teleport'"): + ToolRegistry.from_names(["teleport"]) + + +@pytest.mark.parametrize("name", ToolRegistry.available_tool_names()) +def test_from_names_each_known_tool(name): + registry = ToolRegistry.from_names([name]) + assert len(registry.definitions) == 1 + assert registry.definitions[0]["name"] == name + + +def test_from_names_all_at_once(): + all_names = ToolRegistry.available_tool_names() + registry = ToolRegistry.from_names(all_names) + built_names = {d["name"] for d in registry.definitions} + assert built_names == set(all_names) + + +def test_from_names_fs_tools_share_file_registry(): + """All file-system tools in the same registry share a single FileRegistry.""" + fs_names = [n for n in ToolRegistry.available_tool_names() if n.endswith("_file")] + if len(fs_names) < 2: + pytest.skip("Need at least 2 fs tools to test shared registry") + registry = ToolRegistry.from_names(fs_names) + tools = list(registry._tools.values()) + registries = [t._registry for t in tools] + assert all(r is registries[0] for r in registries) From 73d9d99c479a449eb2562f2180a889f2f6d6b1e4 Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Thu, 30 Apr 2026 18:27:18 +0200 Subject: [PATCH 30/44] FileAccessPolicy, fs/ hardening, memory-before-checkpoint, and owner_id rename (#23453) * wip: Work in Progress * Write memory before checkpoint and owner id rename * Address code review feedback: test coverage, type narrowing and code quality * Add memory_path property to checkpoints * Fix PR comments * Enforce file policy in grep and add canonicalize_path * Fix tests for Windows * Make file_registry required in Phase and ToolRegistry * Move ToolRegistry to tools/ and make logger an optional argument in orchestrator * Change deny_names and deny_roots to deny_patterns; fix grep; fix tests * Few improves in the code --------- Co-authored-by: Juanpe Araque --- ddev/src/ddev/ai/agent/anthropic_client.py | 2 +- ddev/src/ddev/ai/agent/base.py | 2 +- ddev/src/ddev/ai/phases/base.py | 75 ++-- ddev/src/ddev/ai/phases/checkpoint.py | 26 +- ddev/src/ddev/ai/phases/config.py | 2 +- ddev/src/ddev/ai/phases/orchestrator.py | 14 +- ddev/src/ddev/ai/react/callbacks.py | 38 ++ ddev/src/ddev/ai/react/process.py | 8 +- ddev/src/ddev/ai/tools/core/registry.py | 111 ------ ddev/src/ddev/ai/tools/fs/append_file.py | 7 +- ddev/src/ddev/ai/tools/fs/base.py | 25 +- ddev/src/ddev/ai/tools/fs/create_file.py | 15 +- ddev/src/ddev/ai/tools/fs/edit_file.py | 7 +- .../ddev/ai/tools/fs/file_access_policy.py | 142 ++++++++ ddev/src/ddev/ai/tools/fs/file_registry.py | 50 ++- ddev/src/ddev/ai/tools/fs/mkdir.py | 39 ++ ddev/src/ddev/ai/tools/fs/read_file.py | 10 +- ddev/src/ddev/ai/tools/registry.py | 124 +++++++ ddev/src/ddev/ai/tools/shell/base.py | 10 +- ddev/src/ddev/ai/tools/shell/grep.py | 65 +++- ddev/src/ddev/ai/tools/shell/mkdir.py | 28 -- ddev/tests/ai/agent/test_anthropic_client.py | 2 +- ddev/tests/ai/agent/test_base.py | 2 +- ddev/tests/ai/phases/conftest.py | 6 + ddev/tests/ai/phases/test_base.py | 120 ++++++- ddev/tests/ai/phases/test_checkpoint.py | 30 +- ddev/tests/ai/phases/test_orchestrator.py | 57 ++- ddev/tests/ai/react/test_callbacks.py | 130 +++++++ ddev/tests/ai/react/test_process.py | 2 +- ddev/tests/ai/tools/fs/conftest.py | 39 +- ddev/tests/ai/tools/fs/test_append_file.py | 8 +- ddev/tests/ai/tools/fs/test_base.py | 58 ++- ddev/tests/ai/tools/fs/test_create_file.py | 18 +- ddev/tests/ai/tools/fs/test_edit_file.py | 10 +- .../ai/tools/fs/test_file_access_policy.py | 263 ++++++++++++++ ddev/tests/ai/tools/fs/test_file_registry.py | 76 +++- ddev/tests/ai/tools/fs/test_mkdir.py | 47 +++ .../ai/tools/fs/test_policy_enforcement.py | 336 ++++++++++++++++++ ddev/tests/ai/tools/fs/test_read_file.py | 6 +- ddev/tests/ai/tools/fs/test_workflow.py | 6 +- ddev/tests/ai/tools/shell/test_tools.py | 164 +++++++-- .../ai/tools/{core => }/test_registry.py | 61 +++- 42 files changed, 1880 insertions(+), 361 deletions(-) delete mode 100644 ddev/src/ddev/ai/tools/core/registry.py create mode 100644 ddev/src/ddev/ai/tools/fs/file_access_policy.py create mode 100644 ddev/src/ddev/ai/tools/fs/mkdir.py create mode 100644 ddev/src/ddev/ai/tools/registry.py delete mode 100644 ddev/src/ddev/ai/tools/shell/mkdir.py create mode 100644 ddev/tests/ai/tools/fs/test_file_access_policy.py create mode 100644 ddev/tests/ai/tools/fs/test_mkdir.py create mode 100644 ddev/tests/ai/tools/fs/test_policy_enforcement.py rename ddev/tests/ai/tools/{core => }/test_registry.py (70%) diff --git a/ddev/src/ddev/ai/agent/anthropic_client.py b/ddev/src/ddev/ai/agent/anthropic_client.py index ccdf1d10983fb..6d6af0838bbe1 100644 --- a/ddev/src/ddev/ai/agent/anthropic_client.py +++ b/ddev/src/ddev/ai/agent/anthropic_client.py @@ -10,7 +10,7 @@ from ddev.ai.agent.base import BaseAgent from ddev.ai.agent.exceptions import AgentAPIError, AgentConnectionError, AgentError, AgentRateLimitError from ddev.ai.agent.types import AgentResponse, ContextUsage, StopReason, TokenUsage, ToolCall, ToolResultMessage -from ddev.ai.tools.core.registry import ToolRegistry +from ddev.ai.tools.registry import ToolRegistry DEFAULT_MODEL: Final[str] = "claude-sonnet-4-6" DEFAULT_MAX_TOKENS: Final[int] = 8192 # max tokens per response diff --git a/ddev/src/ddev/ai/agent/base.py b/ddev/src/ddev/ai/agent/base.py index 814f6bedf7429..ad7b57eeca84c 100644 --- a/ddev/src/ddev/ai/agent/base.py +++ b/ddev/src/ddev/ai/agent/base.py @@ -7,7 +7,7 @@ from typing import Final from ddev.ai.agent.types import AgentResponse, ToolResultMessage -from ddev.ai.tools.core.registry import ToolRegistry +from ddev.ai.tools.registry import ToolRegistry _COMPACT_SYSTEM_PROMPT: Final[str] = """\ You are summarizing an agentic conversation to free up context space. diff --git a/ddev/src/ddev/ai/phases/base.py b/ddev/src/ddev/ai/phases/base.py index 152bbd3381426..83feaeea12ee8 100644 --- a/ddev/src/ddev/ai/phases/base.py +++ b/ddev/src/ddev/ai/phases/base.py @@ -2,6 +2,7 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import logging from collections.abc import Callable from datetime import UTC, datetime from pathlib import Path @@ -16,7 +17,8 @@ from ddev.ai.phases.template import render_inline, render_prompt from ddev.ai.react.callbacks import CallbackSet from ddev.ai.react.process import ReActProcess -from ddev.ai.tools.core.registry import ToolRegistry +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.registry import ToolRegistry from ddev.event_bus.orchestrator import AsyncProcessor, BaseMessage @@ -41,7 +43,7 @@ def _make_memory_resolver(checkpoint_manager: CheckpointManager) -> Callable[[st def resolve(key: str) -> str: if key.endswith("_memory"): - return checkpoint_manager.get_memory(key.removesuffix("_memory")) + return checkpoint_manager.memory_content(key.removesuffix("_memory")) return f"" return resolve @@ -89,7 +91,9 @@ def __init__( runtime_variables: dict[str, str], flow_variables: dict[str, str], config_dir: Path, + file_registry: FileRegistry, callback_sets: list[CallbackSet] | None = None, + logger: logging.Logger | None = None, ) -> None: super().__init__(name=phase_id) self._phase_id = phase_id @@ -102,7 +106,9 @@ def __init__( self._runtime_variables = runtime_variables self._flow_variables = flow_variables self._config_dir = config_dir - self._callback_sets = callback_sets + self._callback_sets: list[CallbackSet] = callback_sets or [] + self._file_registry = file_registry + self._logger = logger or logging.getLogger(__name__) self._started_at: datetime | None = None self._resolver: Callable[[str], str] | None = None self._executed = False @@ -157,8 +163,10 @@ async def run_tasks( async def process_message(self, message: PhaseTrigger) -> None: """Full phase pipeline. Not intended to be overridden -- customise via the extension points.""" - # 1. Record start time + # 1. Record start time and notify observers self._started_at = datetime.now(UTC) + for cb_set in self._callback_sets: + await cb_set.fire_phase_start(self._phase_id) # 2. Build template context and memory resolver context: dict[str, Any] = { @@ -178,7 +186,11 @@ async def process_message(self, message: PhaseTrigger) -> None: context, self._resolver, ) - tool_registry = ToolRegistry.from_names(self._agent_config.tools) + tool_registry = ToolRegistry.from_names( + self._agent_config.tools, + owner_id=self._phase_id, + file_registry=self._file_registry, + ) agent_kwargs: dict[str, Any] = {} if self._agent_config.model is not None: @@ -207,7 +219,27 @@ async def process_message(self, message: PhaseTrigger) -> None: # 7. Call after_react() self.after_react() - # 8. Write success checkpoint — task work is done + # 8. Build memory prompt (template errors fail the phase) + user_additions = None + if self._config.checkpoint is not None: + user_additions = render_memory_prompt(self._config.checkpoint, self._config_dir, context) + memory_prompt = self._checkpoint_manager.build_memory_prompt(user_additions) + + # 9. Call the agent for the summary — text-only (allowed_tools=[]) + for cb_set in self._callback_sets: + await cb_set.fire_before_agent_send(1) + + response = await agent.send(memory_prompt, allowed_tools=[]) + total_input += response.usage.input_tokens + total_output += response.usage.output_tokens + + for cb_set in self._callback_sets: + await cb_set.fire_agent_response(response, 1) + + # 10. Persist the memory file + self._checkpoint_manager.write_memory(self._phase_id, response.text) + + # 11. Write the success checkpoint (with memory_path and final token totals) self._checkpoint_manager.write_phase_checkpoint( self._phase_id, { @@ -215,24 +247,10 @@ async def process_message(self, message: PhaseTrigger) -> None: "started_at": self._started_at.isoformat(), "finished_at": datetime.now(UTC).isoformat(), "tokens": {"total_input": total_input, "total_output": total_output}, + "memory_path": str(self._checkpoint_manager.memory_path(self._phase_id)), }, ) - # 9. Best-effort memory step — failure here doesn't invalidate a completed phase - user_additions = None - if self._config.checkpoint is not None: - user_additions = render_memory_prompt(self._config.checkpoint, self._config_dir, context) - - try: - prompt = self._checkpoint_manager.build_memory_prompt(user_additions) - # allowed_tools=[] forces a text-only response - response = await agent.send(prompt, allowed_tools=[]) - total_input += response.usage.input_tokens - total_output += response.usage.output_tokens - self._checkpoint_manager.write_memory(self._phase_id, response.text) - except Exception: - pass - async def on_success(self, message: PhaseTrigger) -> None: """Emit PhaseTrigger to unblock dependent phases.""" self.submit_message( @@ -255,11 +273,12 @@ async def on_error(self, message: PhaseTrigger, error: Exception) -> None: }, ) except Exception: - pass - self.submit_message( - PhaseFailedMessage( - id=f"{self._phase_id}_failed_{message.id}", - phase_id=self._phase_id, - error=str(error), + self._logger.exception("Failed to write failure checkpoint for phase %s", self._phase_id) + finally: + self.submit_message( + PhaseFailedMessage( + id=f"{self._phase_id}_failed_{message.id}", + phase_id=self._phase_id, + error=str(error), + ) ) - ) diff --git a/ddev/src/ddev/ai/phases/checkpoint.py b/ddev/src/ddev/ai/phases/checkpoint.py index df9f093ead8fc..6951a7ff2b3ea 100644 --- a/ddev/src/ddev/ai/phases/checkpoint.py +++ b/ddev/src/ddev/ai/phases/checkpoint.py @@ -18,12 +18,15 @@ class CheckpointManager: def __init__(self, path: Path) -> None: self._path = path + def _ensure_dir(self) -> None: + self._path.parent.mkdir(parents=True, exist_ok=True) + def read(self) -> dict[str, Any]: """Return full checkpoint data, keyed by phase_id. Empty dict if file absent.""" if not self._path.exists(): return {} try: - return yaml.safe_load(self._path.read_text()) or {} + return yaml.safe_load(self._path.read_text(encoding="utf-8")) or {} except (OSError, yaml.YAMLError) as e: raise CheckpointReadError(f"Failed to load checkpoints from {self._path}: {e}") from e @@ -31,21 +34,24 @@ def write_phase_checkpoint(self, phase_id: str, data: dict[str, Any]) -> None: """Write or overwrite one phase's section in checkpoints.yaml.""" checkpoints = self.read() checkpoints[phase_id] = data - self._path.parent.mkdir(parents=True, exist_ok=True) - self._path.write_text(yaml.dump(checkpoints, default_flow_style=False)) + self._ensure_dir() + self._path.write_text(yaml.dump(checkpoints, default_flow_style=False), encoding="utf-8") def build_memory_prompt(self, user_additions: str | None) -> str: """Build the memory prompt to send to the agent at the end of a phase.""" base_prompt = "Write a brief summary of what you accomplished in this phase." return f"{user_additions}\n\n{base_prompt}" if user_additions else base_prompt + def memory_path(self, phase_id: str) -> Path: + """Return the resolved path to a phase's memory file.""" + return (self._path.parent / f"{phase_id}_memory.md").resolve() + def write_memory(self, phase_id: str, text: str) -> None: - """Write agent-authored text to this phase's memory file ({phase_id}_memory.md).""" - memory_path = self._path.parent / f"{phase_id}_memory.md" - self._path.parent.mkdir(parents=True, exist_ok=True) - memory_path.write_text(text) + """Write agent-authored text to this phase's memory file.""" + self._ensure_dir() + self.memory_path(phase_id).write_text(text, encoding="utf-8") - def get_memory(self, phase_id: str) -> str: + def memory_content(self, phase_id: str) -> str: """Return the contents of a phase's memory file, or a NOT FOUND placeholder.""" - memory_path = self._path.parent / f"{phase_id}_memory.md" - return memory_path.read_text() if memory_path.exists() else f"" + path = self.memory_path(phase_id) + return path.read_text(encoding="utf-8") if path.exists() else f"" diff --git a/ddev/src/ddev/ai/phases/config.py b/ddev/src/ddev/ai/phases/config.py index 95879bb7a8e73..534c3e9144013 100644 --- a/ddev/src/ddev/ai/phases/config.py +++ b/ddev/src/ddev/ai/phases/config.py @@ -7,7 +7,7 @@ import yaml from pydantic import BaseModel, ConfigDict, ValidationError, field_validator, model_validator -from ddev.ai.tools.core.registry import ToolRegistry +from ddev.ai.tools.registry import ToolRegistry class FlowConfigError(Exception): diff --git a/ddev/src/ddev/ai/phases/orchestrator.py b/ddev/src/ddev/ai/phases/orchestrator.py index 3086be127dd28..bb6a892fcbe4d 100644 --- a/ddev/src/ddev/ai/phases/orchestrator.py +++ b/ddev/src/ddev/ai/phases/orchestrator.py @@ -14,6 +14,8 @@ from ddev.ai.phases.config import FlowConfig, FlowConfigError from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger from ddev.ai.react.callbacks import CallbackSet +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry from ddev.event_bus.exceptions import FatalProcessingError from ddev.event_bus.orchestrator import BaseMessage, EventBusOrchestrator @@ -45,10 +47,17 @@ def __init__( checkpoint_path: Path, runtime_variables: dict[str, str], anthropic_client: anthropic.AsyncAnthropic, + file_access_policy: FileAccessPolicy, callback_sets: list[CallbackSet] | None = None, grace_period: float = 10, + logger: logging.Logger | None = None, ) -> None: - super().__init__(logger=logging.getLogger(__name__), grace_period=grace_period) + """Initialize the orchestrator. + + ``file_access_policy`` must have ``write_root`` set to the integration + output directory so that agent writes are confined to that path. + """ + super().__init__(logger=logger or logging.getLogger(__name__), grace_period=grace_period) self._flow_yaml_path = flow_yaml_path self._checkpoint_path = checkpoint_path self._runtime_variables = runtime_variables @@ -57,6 +66,7 @@ def __init__( self._phase_registry = PhaseRegistry() self._failed_phase: str | None = None self._failed_error: str | None = None + self._file_registry = FileRegistry(policy=file_access_policy) async def on_initialize(self) -> None: """Discover custom phases, parse flow.yaml, construct phases, submit PhaseTrigger.""" @@ -97,7 +107,9 @@ async def on_initialize(self) -> None: runtime_variables=self._runtime_variables, flow_variables=config.variables, config_dir=config_dir, + file_registry=self._file_registry, callback_sets=self._callback_sets, + logger=self._logger, ) self.register_processor(phase, [PhaseTrigger]) diff --git a/ddev/src/ddev/ai/react/callbacks.py b/ddev/src/ddev/ai/react/callbacks.py index 43b25a2d5f52e..2127366b76ded 100644 --- a/ddev/src/ddev/ai/react/callbacks.py +++ b/ddev/src/ddev/ai/react/callbacks.py @@ -45,6 +45,18 @@ class AfterCompactCallback(Protocol): async def __call__(self) -> None: ... +class OnPhaseStartCallback(Protocol): + """Called once when a phase begins executing, before any agent interaction.""" + + async def __call__(self, phase_id: str) -> None: ... + + +class OnBeforeAgentSendCallback(Protocol): + """Called immediately before each agent.send() request is issued.""" + + async def __call__(self, iteration: int) -> None: ... + + class CallbackSet: """Decorator-based registry for ReAct lifecycle event handlers. @@ -64,6 +76,8 @@ def __init__(self) -> None: self._on_error: list[OnErrorCallback] = [] self._before_compact: list[BeforeCompactCallback] = [] self._after_compact: list[AfterCompactCallback] = [] + self._on_phase_start: list[OnPhaseStartCallback] = [] + self._on_before_agent_send: list[OnBeforeAgentSendCallback] = [] def on_agent_response(self, func: OnAgentResponseCallback) -> OnAgentResponseCallback: """Register a handler fired after every agent response.""" @@ -95,6 +109,16 @@ def on_after_compact(self, func: AfterCompactCallback) -> AfterCompactCallback: self._after_compact.append(func) return func + def on_phase_start(self, func: OnPhaseStartCallback) -> OnPhaseStartCallback: + """Register a handler fired at the start of a phase.""" + self._on_phase_start.append(func) + return func + + def on_before_agent_send(self, func: OnBeforeAgentSendCallback) -> OnBeforeAgentSendCallback: + """Register a handler fired right before each agent.send() request.""" + self._on_before_agent_send.append(func) + return func + async def fire_agent_response(self, response: AgentResponse, iteration: int) -> None: for handler in self._on_agent_response: try: @@ -136,3 +160,17 @@ async def fire_after_compact(self) -> None: await handler() except Exception: pass + + async def fire_phase_start(self, phase_id: str) -> None: + for handler in self._on_phase_start: + try: + await handler(phase_id) + except Exception: + pass + + async def fire_before_agent_send(self, iteration: int) -> None: + for handler in self._on_before_agent_send: + try: + await handler(iteration) + except Exception: + pass diff --git a/ddev/src/ddev/ai/react/process.py b/ddev/src/ddev/ai/react/process.py index a4e5476dfffda..2a1adbad90b9b 100644 --- a/ddev/src/ddev/ai/react/process.py +++ b/ddev/src/ddev/ai/react/process.py @@ -10,8 +10,8 @@ from ddev.ai.agent.types import AgentResponse, StopReason, ToolResultMessage from ddev.ai.react.callbacks import CallbackSet from ddev.ai.react.types import ReActResult -from ddev.ai.tools.core.registry import ToolRegistry from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.registry import ToolRegistry class ReActProcess: @@ -93,6 +93,9 @@ async def start(self, prompt: str, allowed_tools: list[str] | None = None) -> Re Every exception is forwarded after notifying callbacks. """ try: + for cb_set in self._callback_sets: + await cb_set.fire_before_agent_send(1) + response = await self._agent.send(prompt, allowed_tools) iterations = 1 total_input = response.usage.input_tokens @@ -123,6 +126,9 @@ async def start(self, prompt: str, allowed_tools: list[str] | None = None) -> Re messages = [ToolResultMessage(tool_call_id=tc.id, result=result) for tc, result in tool_call_results] + for cb_set in self._callback_sets: + await cb_set.fire_before_agent_send(iterations + 1) + response = await self._agent.send(messages, allowed_tools) iterations += 1 total_input += response.usage.input_tokens diff --git a/ddev/src/ddev/ai/tools/core/registry.py b/ddev/src/ddev/ai/tools/core/registry.py deleted file mode 100644 index 9dcbe5adf7131..0000000000000 --- a/ddev/src/ddev/ai/tools/core/registry.py +++ /dev/null @@ -1,111 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -from anthropic.types import ToolParam - -from .protocol import ToolProtocol -from .types import ToolResult - -_TOOL_NAMES: list[str] = [ - "read_file", - "create_file", - "edit_file", - "append_file", - "grep", - "list_files", - "mkdir", - "http_get", - "ddev_create", - "ddev_test", - "ddev_env_show", - "ddev_env_start", - "ddev_env_stop", - "ddev_env_test", - "ddev_release_changelog", -] - - -class ToolRegistry: - """Registry holding all available tools.""" - - def __init__(self, tools: list[ToolProtocol]) -> None: - self._tools: dict[str, ToolProtocol] = {tool.name: tool for tool in tools} - - @staticmethod - def available_tool_names() -> list[str]: - """Return all tool names that from_names can resolve.""" - return list(_TOOL_NAMES) - - @classmethod - def from_names(cls, tool_names: list[str]) -> "ToolRegistry": - """Build a ToolRegistry from a list of tool name strings. - - All file-system tools in the same registry share a single FileRegistry. - """ - from ddev.ai.tools.fs.append_file import AppendFileTool - from ddev.ai.tools.fs.create_file import CreateFileTool - from ddev.ai.tools.fs.edit_file import EditFileTool - from ddev.ai.tools.fs.file_registry import FileRegistry - from ddev.ai.tools.fs.read_file import ReadFileTool - from ddev.ai.tools.http.http_get import HttpGetTool - from ddev.ai.tools.shell.ddev.create import DdevCreateTool - from ddev.ai.tools.shell.ddev.ddev_test import DdevTestTool - from ddev.ai.tools.shell.ddev.env_show import DdevEnvShowTool - from ddev.ai.tools.shell.ddev.env_start import DdevEnvStartTool - from ddev.ai.tools.shell.ddev.env_stop import DdevEnvStopTool - from ddev.ai.tools.shell.ddev.env_test import DdevEnvTestTool - from ddev.ai.tools.shell.ddev.release_changelog import DdevReleaseChangelogTool - from ddev.ai.tools.shell.grep import GrepTool - from ddev.ai.tools.shell.list_files import ListFilesTool - from ddev.ai.tools.shell.mkdir import MkdirTool - - file_registry = FileRegistry() - tools: list[ToolProtocol] = [] - for name in tool_names: - match name: - case "read_file": - tools.append(ReadFileTool(file_registry)) - case "create_file": - tools.append(CreateFileTool(file_registry)) - case "edit_file": - tools.append(EditFileTool(file_registry)) - case "append_file": - tools.append(AppendFileTool(file_registry)) - case "grep": - tools.append(GrepTool()) - case "list_files": - tools.append(ListFilesTool()) - case "mkdir": - tools.append(MkdirTool()) - case "http_get": - tools.append(HttpGetTool()) - case "ddev_create": - tools.append(DdevCreateTool()) - case "ddev_test": - tools.append(DdevTestTool()) - case "ddev_env_show": - tools.append(DdevEnvShowTool()) - case "ddev_env_start": - tools.append(DdevEnvStartTool()) - case "ddev_env_stop": - tools.append(DdevEnvStopTool()) - case "ddev_env_test": - tools.append(DdevEnvTestTool()) - case "ddev_release_changelog": - tools.append(DdevReleaseChangelogTool()) - case _: - raise ValueError(f"Unknown tool name: {name!r}") - return cls(tools) - - @property - def definitions(self) -> list[ToolParam]: - """Return Anthropic SDK tool definitions for all registered tools.""" - return [tool.definition for tool in self._tools.values()] - - async def run(self, name: str, raw: dict[str, object]) -> ToolResult: - """Execute a tool by name, returning an error result if not found.""" - tool = self._tools.get(name) - if tool is None: - return ToolResult(success=False, error=f"Unknown tool: {name!r}") - return await tool.run(raw) diff --git a/ddev/src/ddev/ai/tools/fs/append_file.py b/ddev/src/ddev/ai/tools/fs/append_file.py index c3908059af46f..5b3ba918c938e 100644 --- a/ddev/src/ddev/ai/tools/fs/append_file.py +++ b/ddev/src/ddev/ai/tools/fs/append_file.py @@ -1,7 +1,6 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from pathlib import Path from typing import Annotated from pydantic import Field @@ -10,6 +9,7 @@ from ddev.ai.tools.core.types import ToolResult from .base import FileRegistryTool +from .file_access_policy import FileAccessError class AppendFileInput(BaseToolInput): @@ -26,7 +26,10 @@ def name(self) -> str: return "append_file" async def __call__(self, tool_input: AppendFileInput) -> ToolResult: - path = Path(tool_input.path).resolve() + try: + path = self._assert_writable(tool_input.path) + except FileAccessError as e: + return ToolResult(success=False, error=str(e)) async with self._registry.lock_for(str(path)): current_content, fail = self._read_verified(str(path)) diff --git a/ddev/src/ddev/ai/tools/fs/base.py b/ddev/src/ddev/ai/tools/fs/base.py index b952b03363f63..54c523703d768 100644 --- a/ddev/src/ddev/ai/tools/fs/base.py +++ b/ddev/src/ddev/ai/tools/fs/base.py @@ -10,22 +10,33 @@ class FileRegistryTool[TInput: BaseToolInput](BaseTool[TInput]): - """Abstract base for file system tools with hash-based consistency checks.""" + """Abstract base for file system tools with hash-based consistency checks. - def __init__(self, file_registry: FileRegistry) -> None: + Each tool instance is bound to an owner_id; all registry operations run + under that identity so read-before-write is enforced per owner. + """ + + def __init__(self, file_registry: FileRegistry, owner_id: str) -> None: self._registry = file_registry + self._owner_id = owner_id def _register(self, path: str, content: str) -> None: - self._registry.record(path, content) + self._registry.record(self._owner_id, path, content) + + def _assert_writable(self, path: str) -> Path: + return self._registry.policy.assert_writable(path) + + def _assert_readable(self, path: str) -> Path: + return self._registry.policy.assert_readable(path) def _read_verified(self, path: str) -> tuple[str, ToolResult | None]: - """Read file content and verify it matches the last recorded hash.""" - if not self._registry.is_known(path): + """Read file content and verify it matches this agent's last recorded hash.""" + if not self._registry.is_known(self._owner_id, path): return "", ToolResult(success=False, error=f"Not authorized to modify '{path}'.") try: content = Path(path).read_text(encoding="utf-8") - except OSError as e: + except (OSError, UnicodeDecodeError) as e: return "", ToolResult(success=False, error=str(e)) - if not self._registry.verify(path, content): + if not self._registry.verify(self._owner_id, path, content): return "", ToolResult(success=False, error=f"File '{path}' has changed since last read. Re-read and retry.") return content, None diff --git a/ddev/src/ddev/ai/tools/fs/create_file.py b/ddev/src/ddev/ai/tools/fs/create_file.py index aa3dff51428ea..e73f79e383be0 100644 --- a/ddev/src/ddev/ai/tools/fs/create_file.py +++ b/ddev/src/ddev/ai/tools/fs/create_file.py @@ -1,7 +1,6 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from pathlib import Path from typing import Annotated from pydantic import Field @@ -10,6 +9,7 @@ from ddev.ai.tools.core.types import ToolResult from .base import FileRegistryTool +from .file_access_policy import FileAccessError class CreateFileInput(BaseToolInput): @@ -29,15 +29,18 @@ def name(self) -> str: return "create_file" async def __call__(self, tool_input: CreateFileInput) -> ToolResult: - path = Path(tool_input.path).resolve() + try: + path = self._assert_writable(tool_input.path) + except FileAccessError as e: + return ToolResult(success=False, error=str(e)) async with self._registry.lock_for(str(path)): - if path.exists(): - return ToolResult(success=False, error=f"File already exists: {path}") - try: path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(tool_input.content, encoding="utf-8") + with open(path, "x", encoding="utf-8") as fh: + fh.write(tool_input.content) + except FileExistsError: + return ToolResult(success=False, error=f"File already exists: {path}") except OSError as e: return ToolResult(success=False, error=str(e)) self._register(str(path), tool_input.content) diff --git a/ddev/src/ddev/ai/tools/fs/edit_file.py b/ddev/src/ddev/ai/tools/fs/edit_file.py index 7e4eafde0ce62..f5b90a4b1b179 100644 --- a/ddev/src/ddev/ai/tools/fs/edit_file.py +++ b/ddev/src/ddev/ai/tools/fs/edit_file.py @@ -1,7 +1,6 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from pathlib import Path from typing import Annotated from pydantic import Field @@ -10,6 +9,7 @@ from ddev.ai.tools.core.types import ToolResult from .base import FileRegistryTool +from .file_access_policy import FileAccessError class EditFileInput(BaseToolInput): @@ -37,7 +37,10 @@ def name(self) -> str: return "edit_file" async def __call__(self, tool_input: EditFileInput) -> ToolResult: - path = Path(tool_input.path).resolve() + try: + path = self._assert_writable(tool_input.path) + except FileAccessError as e: + return ToolResult(success=False, error=str(e)) async with self._registry.lock_for(str(path)): content, fail = self._read_verified(str(path)) diff --git a/ddev/src/ddev/ai/tools/fs/file_access_policy.py b/ddev/src/ddev/ai/tools/fs/file_access_policy.py new file mode 100644 index 0000000000000..d25fcb0f99573 --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/file_access_policy.py @@ -0,0 +1,142 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os +from collections.abc import Iterable +from fnmatch import fnmatch +from pathlib import Path + + +def canonicalize_path(path: str | Path) -> Path: + """Single source of truth for path canonicalization across the fs layer. + + Every component that compares, indexes, or operates on filesystem paths + must run them through this function so the policy, the tools, and the + registry agree on what path each input names. + + ``strict=False`` allows resolving paths for files that don't exist yet + (e.g. pre-creation checks). Idempotent: calling on an already-canonical + path returns the same path. + """ + return Path(path).expanduser().resolve(strict=False) + + +WILDCARD_CHARS = "*?[" + + +def _canonicalize_pattern(pat: str) -> str: + """Resolve a path pattern's static prefix while leaving wildcards intact. + + Splits at the first wildcard character (``*``, ``?`` or ``[``), runs + expanduser + resolve on the leading prefix, then re-attaches the + wildcard suffix. Patterns without wildcards are fully resolved. Used so + symlinked deny roots (e.g. ``~/.ssh -> /secrets/ssh``) are matched + against the same target the path side canonicalizes to. + """ + indices = [pat.find(c) for c in WILDCARD_CHARS if c in pat] + idx = min(indices) if indices else -1 + if idx == -1: + return str(canonicalize_path(pat)) + prefix, suffix = pat[:idx], pat[idx:] + if not prefix: + return suffix + resolved = str(canonicalize_path(prefix)) + # canonicalize_path strips trailing separators; restore one if the prefix + # had it so "/" + "*" stays "/*" instead of "*". + if prefix.endswith(("/", os.sep)) and not resolved.endswith(("/", os.sep)): + resolved += os.sep + return f"{resolved}{suffix}" + + +DEFAULT_DENY_PATTERNS: tuple[str, ...] = ( + # Location-independent: secrets identified by name or extension. + ".env", + ".env.*", + ".envrc", + ".netrc", + "*.pem", + "*.key", + # Location-rooted: entire directories of secrets. + "~/.ssh/*", + "~/.aws/*", + "~/.gnupg/*", + "~/.config/gcloud/*", + "~/.kube/*", + "~/.docker/*", +) + + +class FileAccessError(Exception): + """Raised when a file access violates the configured policy.""" + + +class FileAccessPolicy: + """Global file access policy shared across agents and phases in a run. + + Enforces a two-zone model based on ``write_root``: inside it, all reads + and writes are allowed. Outside, writes are always denied; reads are + allowed only if the path does not match a deny pattern. + + Each entry in ``deny_patterns`` is an fnmatch-style glob. Patterns are + classified at construction time: + + - **Basename patterns** (no ``/``) are matched against the resolved + path's basename — they apply globally regardless of location. Use + these for location-independent rules like ``*.pem`` or ``.env``. + - **Path patterns** (contain ``/``) are matched against the resolved + path's full string. Their static prefix is run through + ``expanduser + resolve`` at construction so symlinked roots cannot + bypass the rule. Use these for location-specific rules like + ``~/.ssh/*`` or ``~/.aws/credentials``. + + Paths checked at runtime go through ``canonicalize_path`` before + matching, so symlinks and ``..`` cannot bypass the checks. + """ + + def __init__( + self, + write_root: Path | str, + deny_patterns: Iterable[str] = DEFAULT_DENY_PATTERNS, + ) -> None: + self._write_root = canonicalize_path(write_root) + patterns = tuple(deny_patterns) + self._deny_patterns: tuple[str, ...] = patterns + + basename: list[str] = [] + path: list[str] = [] + for p in patterns: + (path if "/" in p else basename).append(p) + self._basename_patterns: tuple[str, ...] = tuple(basename) + self._path_patterns: tuple[str, ...] = tuple(_canonicalize_pattern(p) for p in path) + + @property + def write_root(self) -> Path: + return self._write_root + + @property + def deny_patterns(self) -> tuple[str, ...]: + return self._deny_patterns + + @property + def basename_patterns(self) -> tuple[str, ...]: + return self._basename_patterns + + def _is_denied(self, resolved: Path) -> bool: + if any(fnmatch(resolved.name, pat) for pat in self._basename_patterns): + return True + full = str(resolved) + return any(fnmatch(full, pat) for pat in self._path_patterns) + + def assert_readable(self, path: str | Path) -> Path: + resolved = canonicalize_path(path) + if resolved.is_relative_to(self._write_root): + return resolved + if self._is_denied(resolved): + raise FileAccessError(f"Read denied by policy: {resolved}") + return resolved + + def assert_writable(self, path: str | Path) -> Path: + resolved = canonicalize_path(path) + if not resolved.is_relative_to(self._write_root): + raise FileAccessError(f"Write denied: {resolved} is outside write root {self._write_root}") + return resolved diff --git a/ddev/src/ddev/ai/tools/fs/file_registry.py b/ddev/src/ddev/ai/tools/fs/file_registry.py index 5bf64221100ed..013956d9401aa 100644 --- a/ddev/src/ddev/ai/tools/fs/file_registry.py +++ b/ddev/src/ddev/ai/tools/fs/file_registry.py @@ -3,34 +3,52 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import asyncio import hashlib -from pathlib import Path + +from .file_access_policy import FileAccessPolicy, canonicalize_path class FileRegistry: - """Tracks files created by the agent and their last-seen content hash.""" + """Tracks the files each owner has seen, along with their last-seen content hash. + + One FileRegistry is intended to be shared across all owners in a run. Hashes + are partitioned by owner_id so that each owner must independently read or + create a file before modifying it; reads by owner A never authorize writes + by owner B. Only SHA-256 digests are stored (not file contents). + + Path-level locks are shared across owners so that concurrent writes to the + same file are serialized regardless of which owner initiated them. - def __init__(self) -> None: - self._hashes: dict[str, str] = {} + _hashes layout: {owner_id: {normalized_path: sha256_hex}}. + _locks and _hashes grow for the registry's lifetime and are never evicted. + """ + + def __init__(self, policy: FileAccessPolicy) -> None: + self._policy = policy + self._hashes: dict[str, dict[str, str]] = {} self._locks: dict[str, asyncio.Lock] = {} + @property + def policy(self) -> FileAccessPolicy: + return self._policy + def _normalize(self, path: str) -> str: - return Path(path).resolve().as_posix() + return canonicalize_path(path).as_posix() def _hash(self, content: str) -> str: return hashlib.sha256(content.encode()).hexdigest() - def record(self, path: str, content: str) -> None: - self._hashes[self._normalize(path)] = self._hash(content) + def record(self, owner_id: str, path: str, content: str) -> None: + self._hashes.setdefault(owner_id, {})[self._normalize(path)] = self._hash(content) - def is_known(self, path: str) -> bool: - return self._normalize(path) in self._hashes + def is_known(self, owner_id: str, path: str) -> bool: + return self._normalize(path) in self._hashes.get(owner_id, {}) + + def verify(self, owner_id: str, path: str, content: str) -> bool: + """Check whether content matches what this agent last recorded for path.""" + stored = self._hashes.get(owner_id, {}).get(self._normalize(path)) + return stored is not None and self._hash(content) == stored def lock_for(self, path: str) -> asyncio.Lock: - # Safe under single-threaded asyncio; asyncio.Lock is not thread-safe + # Safe under single-threaded asyncio; asyncio.Lock is not thread-safe. + # Path-level (not agent-scoped) so concurrent writes from different agents serialize. return self._locks.setdefault(self._normalize(path), asyncio.Lock()) - - def verify(self, path: str, content: str) -> bool: - """Check whether content matches what was last recorded for path.""" - normalized = self._normalize(path) - stored = self._hashes.get(normalized) - return stored is not None and self._hash(content) == stored diff --git a/ddev/src/ddev/ai/tools/fs/mkdir.py b/ddev/src/ddev/ai/tools/fs/mkdir.py new file mode 100644 index 0000000000000..e5c7ea3a83647 --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/mkdir.py @@ -0,0 +1,39 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseTool, BaseToolInput +from ddev.ai.tools.core.types import ToolResult + +from .file_access_policy import FileAccessError, FileAccessPolicy + + +class MkdirInput(BaseToolInput): + path: Annotated[str, Field(description="Path of the directory to create")] + + +class MkdirTool(BaseTool[MkdirInput]): + """Creates a directory at the given path, including any missing parent directories. + Use to create directories for config files, logs, source code. + Writes are restricted to the configured write root.""" + + def __init__(self, policy: FileAccessPolicy) -> None: + self._policy = policy + + @property + def name(self) -> str: + return "mkdir" + + async def __call__(self, tool_input: MkdirInput) -> ToolResult: + try: + path = self._policy.assert_writable(tool_input.path) + except FileAccessError as e: + return ToolResult(success=False, error=str(e)) + try: + path.mkdir(parents=True, exist_ok=True) + except OSError as e: + return ToolResult(success=False, error=str(e)) + return ToolResult(success=True, data=f"Directory created: {path}") diff --git a/ddev/src/ddev/ai/tools/fs/read_file.py b/ddev/src/ddev/ai/tools/fs/read_file.py index 18367e1b0a6ac..c1b6293c3f395 100644 --- a/ddev/src/ddev/ai/tools/fs/read_file.py +++ b/ddev/src/ddev/ai/tools/fs/read_file.py @@ -1,7 +1,6 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from pathlib import Path from typing import Annotated from pydantic import Field @@ -11,6 +10,7 @@ from ddev.ai.tools.core.types import ToolResult from .base import FileRegistryTool +from .file_access_policy import FileAccessError class ReadFileInput(BaseToolInput): @@ -37,11 +37,15 @@ def name(self) -> str: async def __call__(self, tool_input: ReadFileInput) -> ToolResult: try: - content = Path(tool_input.path).resolve().read_text(encoding="utf-8") + path = self._assert_readable(tool_input.path) + except FileAccessError as e: + return ToolResult(success=False, error=str(e)) + try: + content = path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError) as e: return ToolResult(success=False, error=f"{tool_input.path}: {e}") - self._register(tool_input.path, content) + self._register(str(path), content) offset = tool_input.offset limit = tool_input.limit diff --git a/ddev/src/ddev/ai/tools/registry.py b/ddev/src/ddev/ai/tools/registry.py new file mode 100644 index 0000000000000..f8249bfb7fd93 --- /dev/null +++ b/ddev/src/ddev/ai/tools/registry.py @@ -0,0 +1,124 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from importlib import import_module + +from anthropic.types import ToolParam + +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry + +from .core.protocol import ToolProtocol +from .core.types import ToolResult + + +@dataclass +class ToolContext: + """Shared resources passed to every tool factory during construction.""" + + file_registry: FileRegistry + owner_id: str + + @property + def policy(self) -> FileAccessPolicy: + return self.file_registry.policy + + +def _plain_factory(tool_cls: type[ToolProtocol], ctx: ToolContext) -> ToolProtocol: + return tool_cls() + + +def _file_registry_factory(tool_cls: type, ctx: ToolContext) -> ToolProtocol: + return tool_cls(ctx.file_registry, ctx.owner_id) + + +def _file_policy_factory(tool_cls: type, ctx: ToolContext) -> ToolProtocol: + return tool_cls(ctx.policy) + + +@dataclass(frozen=True) +class ToolSpec: + """Lazy pointer to a tool class and how to construct it. + + ``module`` is relative to the registry's package (e.g. ``"fs.read_file"``). + ``factory`` receives the already-imported class and the shared ToolContext + and returns a constructed tool instance. + """ + + module: str + cls: str + factory: Callable[[type, ToolContext], ToolProtocol] = _plain_factory + + +TOOL_MANIFEST: dict[str, ToolSpec] = { + "read_file": ToolSpec("fs.read_file", "ReadFileTool", factory=_file_registry_factory), + "create_file": ToolSpec("fs.create_file", "CreateFileTool", factory=_file_registry_factory), + "edit_file": ToolSpec("fs.edit_file", "EditFileTool", factory=_file_registry_factory), + "append_file": ToolSpec("fs.append_file", "AppendFileTool", factory=_file_registry_factory), + "grep": ToolSpec("shell.grep", "GrepTool", factory=_file_policy_factory), + "list_files": ToolSpec("shell.list_files", "ListFilesTool"), + "mkdir": ToolSpec("fs.mkdir", "MkdirTool", factory=_file_policy_factory), + "http_get": ToolSpec("http.http_get", "HttpGetTool"), + "ddev_create": ToolSpec("shell.ddev.create", "DdevCreateTool"), + "ddev_test": ToolSpec("shell.ddev.ddev_test", "DdevTestTool"), + "ddev_env_show": ToolSpec("shell.ddev.env_show", "DdevEnvShowTool"), + "ddev_env_start": ToolSpec("shell.ddev.env_start", "DdevEnvStartTool"), + "ddev_env_stop": ToolSpec("shell.ddev.env_stop", "DdevEnvStopTool"), + "ddev_env_test": ToolSpec("shell.ddev.env_test", "DdevEnvTestTool"), + "ddev_release_changelog": ToolSpec("shell.ddev.release_changelog", "DdevReleaseChangelogTool"), +} + + +class ToolRegistry: + """Registry holding all available tools.""" + + def __init__(self, tools: list[ToolProtocol]) -> None: + self._tools: dict[str, ToolProtocol] = {tool.name: tool for tool in tools} + + @staticmethod + def available_tool_names() -> list[str]: + """Return all tool names that from_names can resolve.""" + return list(TOOL_MANIFEST) + + @classmethod + def from_names( + cls, + tool_names: list[str], + *, + owner_id: str, + file_registry: FileRegistry, + ) -> ToolRegistry: + """Build a ToolRegistry from a list of tool name strings. + + The file_registry is shared across all owners in a run so that the access + policy applies globally; hashes inside it are partitioned by owner_id so + each owner must still read-before-write on its own. + """ + ctx = ToolContext( + file_registry=file_registry, + owner_id=owner_id, + ) + tools: list[ToolProtocol] = [] + for name in tool_names: + spec = TOOL_MANIFEST.get(name) + if spec is None: + raise ValueError(f"Unknown tool name: {name!r}") + tool_cls = getattr(import_module(f"{__package__}.{spec.module}"), spec.cls) + tools.append(spec.factory(tool_cls, ctx)) + return cls(tools) + + @property + def definitions(self) -> list[ToolParam]: + """Return Anthropic SDK tool definitions for all registered tools.""" + return [tool.definition for tool in self._tools.values()] + + async def run(self, name: str, raw: dict[str, object]) -> ToolResult: + """Execute a tool by name, returning an error result if not found.""" + tool = self._tools.get(name) + if tool is None: + return ToolResult(success=False, error=f"Unknown tool: {name!r}") + return await tool.run(raw) diff --git a/ddev/src/ddev/ai/tools/shell/base.py b/ddev/src/ddev/ai/tools/shell/base.py index 099c4f10993eb..2ff25110e4161 100644 --- a/ddev/src/ddev/ai/tools/shell/base.py +++ b/ddev/src/ddev/ai/tools/shell/base.py @@ -3,6 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import asyncio from abc import abstractmethod +from collections.abc import Callable from typing import ClassVar from ddev.ai.tools.core.base import BaseTool, BaseToolInput @@ -24,7 +25,11 @@ async def __call__(self, tool_input: TInput) -> ToolResult: return await run_command(self.cmd(tool_input), timeout=self.timeout) -async def run_command(cmd: list[str], timeout: int = 10) -> ToolResult: +async def run_command( + cmd: list[str], + timeout: int = 10, + stdout_filter: Callable[[str], str] | None = None, +) -> ToolResult: try: proc = await asyncio.create_subprocess_exec( *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE @@ -43,6 +48,9 @@ async def run_command(cmd: list[str], timeout: int = 10) -> ToolResult: stdout = stdout_bytes.decode("utf-8", errors="replace") stderr = stderr_bytes.decode("utf-8", errors="replace") + if stdout_filter is not None: + stdout = stdout_filter(stdout) + output = stdout if proc.returncode != 0 and stderr: output = (output + "\n" + stderr) if output else stderr diff --git a/ddev/src/ddev/ai/tools/shell/grep.py b/ddev/src/ddev/ai/tools/shell/grep.py index 15cabe87594ee..3d8ba491b7b4c 100644 --- a/ddev/src/ddev/ai/tools/shell/grep.py +++ b/ddev/src/ddev/ai/tools/shell/grep.py @@ -1,12 +1,14 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +from pathlib import Path from typing import Annotated from pydantic import Field from ddev.ai.tools.core.base import BaseToolInput from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.fs.file_access_policy import FileAccessError, FileAccessPolicy, canonicalize_path from .base import CmdTool, run_command @@ -20,24 +22,81 @@ class GrepInput(BaseToolInput): class GrepTool(CmdTool[GrepInput]): """Searches for a regex pattern in files. Returns matching lines with file path and line numbers. Use to find specific config values, ports, hostnames across files. Supports extended - regex syntax. Output might be truncated for large results.""" + regex syntax. Output might be truncated for large results. + """ timeout = 30 + def __init__(self, policy: FileAccessPolicy) -> None: + self._policy = policy + @property def name(self) -> str: return "grep" async def __call__(self, tool_input: GrepInput) -> ToolResult: - result = await run_command(self.cmd(tool_input), timeout=self.timeout) + try: + self._policy.assert_readable(tool_input.path) + except FileAccessError as e: + return ToolResult(success=False, error=str(e)) + result = await run_command( + self.cmd(tool_input), + timeout=self.timeout, + stdout_filter=self._filter_stdout if tool_input.recursive else None, + ) # grep exits 1 when no lines match — not a failure if not result.success and result.error is None: return result.model_copy(update={"success": True}) return result def cmd(self, tool_input: GrepInput) -> list[str]: - cmd = ["grep", "-n", "-E"] + cmd = ["grep", "-n", "-E", "--null", "-I", "--no-messages"] if tool_input.recursive: cmd.append("-r") + cmd.extend(self._exclude_flags(canonicalize_path(tool_input.path))) cmd += ["--", tool_input.pattern, tool_input.path] return cmd + + def _exclude_flags(self, search_path: Path) -> list[str]: + # Skip --exclude= flags when the search overlaps write_root: either the + # search is inside write_root (all files are visible) or write_root is + # inside the search (mixing zones). In both cases the post-filter handles + # per-line decisions correctly. Only apply flags when the entire search + # is outside write_root, where deny patterns are fully in effect. + write_root = self._policy.write_root + if search_path.is_relative_to(write_root) or write_root.is_relative_to(search_path): + return [] + return [f"--exclude={pat}" for pat in self._policy.basename_patterns] + + def _filter_stdout(self, stdout: str) -> str: + """Filter stdout to only include lines whose filename is allowed by the policy. + If the filename is denied, we return 'Read denied by policy' instead of the line. + + ``grep --null`` output: ``\\0:\\n``. Split on the + first NUL and run the filename through ``assert_readable`` (which + canonicalizes through symlinks). + + Only use when recursive is True. + """ + decision: dict[str, bool] = {} + result: list[str] = [] + emitted_denials: set[str] = set() + for line in stdout.splitlines(): + nul = line.find("\0") + if nul == -1: + continue + filename, rest = line[:nul], line[nul + 1 :] + allowed = decision.get(filename) + if allowed is None: + try: + self._policy.assert_readable(filename) + allowed = True + except FileAccessError: + allowed = False + decision[filename] = allowed + if allowed: + result.append(f"{filename}:{rest}") + elif filename not in emitted_denials: + result.append(f"{filename}: Read denied by policy") + emitted_denials.add(filename) + return "\n".join(result) diff --git a/ddev/src/ddev/ai/tools/shell/mkdir.py b/ddev/src/ddev/ai/tools/shell/mkdir.py deleted file mode 100644 index 7bd25733a245d..0000000000000 --- a/ddev/src/ddev/ai/tools/shell/mkdir.py +++ /dev/null @@ -1,28 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) -from typing import Annotated - -from pydantic import Field - -from ddev.ai.tools.core.base import BaseToolInput - -from .base import CmdTool - - -class MkdirInput(BaseToolInput): - path: Annotated[str, Field(description="Path of the directory to create")] - - -class MkdirTool(CmdTool[MkdirInput]): - """Creates a directory at the given path, including any missing parent directories. - Use to create directories for config files, logs, source code.""" - - timeout = 5 - - @property - def name(self) -> str: - return "mkdir" - - def cmd(self, tool_input: MkdirInput) -> list[str]: - return ["mkdir", "-p", tool_input.path] diff --git a/ddev/tests/ai/agent/test_anthropic_client.py b/ddev/tests/ai/agent/test_anthropic_client.py index 5b68c0ff9aca5..739f99d53c74a 100644 --- a/ddev/tests/ai/agent/test_anthropic_client.py +++ b/ddev/tests/ai/agent/test_anthropic_client.py @@ -11,8 +11,8 @@ from ddev.ai.agent.anthropic_client import AnthropicAgent from ddev.ai.agent.exceptions import AgentAPIError, AgentConnectionError, AgentError, AgentRateLimitError from ddev.ai.agent.types import StopReason, ToolResultMessage -from ddev.ai.tools.core.registry import ToolRegistry from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.registry import ToolRegistry # --------------------------------------------------------------------------- # Helpers diff --git a/ddev/tests/ai/agent/test_base.py b/ddev/tests/ai/agent/test_base.py index 757674ac43446..c5fd73a10a4b9 100644 --- a/ddev/tests/ai/agent/test_base.py +++ b/ddev/tests/ai/agent/test_base.py @@ -6,7 +6,7 @@ from ddev.ai.agent.base import _COMPACT_SYSTEM_PROMPT, BaseAgent from ddev.ai.agent.types import AgentResponse, StopReason, TokenUsage, ToolResultMessage -from ddev.ai.tools.core.registry import ToolRegistry +from ddev.ai.tools.registry import ToolRegistry _AGENT_NAME: str = "test" _AGENT_SYSTEM_PROMPT: str = "original" diff --git a/ddev/tests/ai/phases/conftest.py b/ddev/tests/ai/phases/conftest.py index 16b078d677202..0174d2907350f 100644 --- a/ddev/tests/ai/phases/conftest.py +++ b/ddev/tests/ai/phases/conftest.py @@ -8,6 +8,7 @@ import pytest from ddev.ai.agent.types import AgentResponse, ContextUsage, StopReason, TokenUsage, ToolResultMessage +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy # --------------------------------------------------------------------------- # Helpers @@ -112,3 +113,8 @@ def flow_dir(tmp_path): def message_queue(): """An asyncio.Queue that can be attached to a Phase for submit_message.""" return asyncio.Queue() + + +@pytest.fixture +def file_access_policy(tmp_path) -> FileAccessPolicy: + return FileAccessPolicy(write_root=tmp_path) diff --git a/ddev/tests/ai/phases/test_base.py b/ddev/tests/ai/phases/test_base.py index 13973152a186b..4481e21c2e725 100644 --- a/ddev/tests/ai/phases/test_base.py +++ b/ddev/tests/ai/phases/test_base.py @@ -3,6 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from datetime import UTC, datetime +from pathlib import Path from unittest.mock import MagicMock import pytest @@ -11,12 +12,14 @@ from ddev.ai.phases.checkpoint import CheckpointManager from ddev.ai.phases.config import AgentConfig, CheckpointConfig, FlowConfigError, PhaseConfig, TaskConfig from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger -from ddev.ai.tools.core.registry import ToolRegistry +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.registry import ToolRegistry from .conftest import MockAgent, make_agent_factory, make_response, resolve_key -def _empty_registry_from_names(cls, names): +def _empty_registry_from_names(cls, names, *, owner_id, file_registry): return ToolRegistry([]) @@ -37,7 +40,7 @@ def test_resolver_non_memory_key(): mgr = MagicMock() resolver = _make_memory_resolver(mgr) assert resolver("some_variable") == "" - mgr.get_memory.assert_not_called() + mgr.memory_content.assert_not_called() def test_resolver_absent_memory(tmp_path): @@ -146,6 +149,7 @@ def _make_phase( runtime_variables=runtime_variables or {}, flow_variables=flow_variables or {}, config_dir=flow_dir, + file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), callback_sets=None, ) phase.queue = message_queue @@ -168,13 +172,14 @@ async def test_happy_path_single_task(flow_dir, monkeypatch, message_queue): await phase.process_message(PhaseTrigger(id="start", phase_id=None)) # Memory was written - assert mgr.get_memory("p1") == "summary" + assert mgr.memory_content("p1") == "summary" - # Checkpoint was written + # Checkpoint was written with memory_path and final token totals (including memory step) checkpoint = mgr.read()["p1"] assert checkpoint["status"] == "success" - assert checkpoint["tokens"]["total_input"] == 100 - assert checkpoint["tokens"]["total_output"] == 50 + assert checkpoint["tokens"]["total_input"] == 110 + assert checkpoint["tokens"]["total_output"] == 55 + assert checkpoint["memory_path"] # non-empty string # on_success is called by _task_wrapper, not process_message directly. # But we verify it would work by checking the send calls. @@ -204,8 +209,9 @@ async def test_happy_path_two_tasks(flow_dir, monkeypatch, message_queue): await phase.process_message(PhaseTrigger(id="start", phase_id=None)) checkpoint = mgr.read()["p1"] - assert checkpoint["tokens"]["total_input"] == 300 - assert checkpoint["tokens"]["total_output"] == 130 + assert checkpoint["tokens"]["total_input"] == 310 + assert checkpoint["tokens"]["total_output"] == 135 + assert checkpoint["memory_path"] # --------------------------------------------------------------------------- @@ -276,6 +282,7 @@ async def test_compact_between_tasks_when_above_threshold(flow_dir, monkeypatch, checkpoint = mgr.read()["p1"] assert checkpoint["status"] == "success" + assert checkpoint["memory_path"] assert mock_agent.compact_call_count >= 1 @@ -298,7 +305,9 @@ async def test_no_compact_when_below_threshold(flow_dir, monkeypatch, message_qu ) await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - assert mgr.read()["p1"]["status"] == "success" + checkpoint = mgr.read()["p1"] + assert checkpoint["status"] == "success" + assert checkpoint["memory_path"] assert mock_agent.compact_call_count == 0 @@ -339,6 +348,7 @@ def capturing_factory(**kwargs): runtime_variables={}, flow_variables={"project": "myproj"}, config_dir=flow_dir, + file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), ) phase.queue = message_queue @@ -378,6 +388,7 @@ def capturing_factory(**kwargs): runtime_variables={"project": "runtime_override"}, flow_variables={"project": "flow_default"}, config_dir=flow_dir, + file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), ) phase.queue = message_queue @@ -513,6 +524,7 @@ async def test_task_prompt_resolves_memory_variable(flow_dir, monkeypatch, messa runtime_variables={}, flow_variables={}, config_dir=flow_dir, + file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), ) phase.queue = message_queue @@ -586,3 +598,91 @@ def test_should_process_returns_false_after_already_executed(flow_dir, monkeypat result = phase.should_process_message(PhaseTrigger(id="start2", phase_id=None)) assert result is False + + +# --------------------------------------------------------------------------- +# Phase.process_message — memory step failure behaviour +# --------------------------------------------------------------------------- + + +async def test_memory_api_failure_fails_phase(flow_dir, monkeypatch, message_queue): + # Only 1 response provided; second send (memory step) raises IndexError. + responses = [make_response("task done", 100, 50)] + mock_agent = MockAgent(responses) + phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + with pytest.raises(IndexError): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + # Checkpoint must not have been written (exception before checkpoint write) + assert mgr.read() == {} + + +async def test_memory_template_error_fails_phase(flow_dir, monkeypatch, message_queue): + responses = [make_response("task done", 100, 50)] + mock_agent = MockAgent(responses) + phase, mgr = _make_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + checkpoint=CheckpointConfig(memory_prompt="Summarize."), + ) + + def raise_render_error(*args, **kwargs): + raise ValueError("template error") + + monkeypatch.setattr("ddev.ai.phases.base.render_memory_prompt", raise_render_error) + + with pytest.raises(ValueError, match="template error"): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.read() == {} + + +async def test_successful_phase_writes_memory_path_into_checkpoint(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task done", 100, 50), + make_response("summary text", 10, 5), + ] + mock_agent = MockAgent(responses) + phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + checkpoint = mgr.read()["p1"] + assert "memory_path" in checkpoint + memory_path = Path(checkpoint["memory_path"]) + assert memory_path.is_absolute() + assert memory_path.exists() + assert memory_path.name == "p1_memory.md" + assert memory_path.read_text() == "summary text" + + +async def test_failed_phase_omits_memory_path(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([]) + phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + await phase.on_error(PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) + + checkpoint = mgr.read()["p1"] + assert "memory_path" not in checkpoint + + +async def test_write_memory_disk_failure_fails_phase(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task done", 100, 50), + make_response("summary text", 10, 5), + ] + mock_agent = MockAgent(responses) + phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + def raise_permission_error(*args, **kwargs): + raise PermissionError("disk is read-only") + + monkeypatch.setattr("ddev.ai.phases.checkpoint.CheckpointManager.write_memory", raise_permission_error) + + with pytest.raises(PermissionError, match="disk is read-only"): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.read() == {} diff --git a/ddev/tests/ai/phases/test_checkpoint.py b/ddev/tests/ai/phases/test_checkpoint.py index 8685378cb1da6..571ea642065da 100644 --- a/ddev/tests/ai/phases/test_checkpoint.py +++ b/ddev/tests/ai/phases/test_checkpoint.py @@ -2,6 +2,8 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +from pathlib import Path + import pytest from ddev.ai.phases.checkpoint import CheckpointManager, CheckpointReadError @@ -34,7 +36,7 @@ def test_read_malformed_yaml_raises_checkpoint_read_error(manager): def test_read_unreadable_file_raises_checkpoint_read_error(manager, monkeypatch): manager._path.write_text("phase1:\n status: success\n") - monkeypatch.setattr("pathlib.Path.read_text", lambda *_: (_ for _ in ()).throw(OSError("permission denied"))) + monkeypatch.setattr("pathlib.Path.read_text", lambda *_, **__: (_ for _ in ()).throw(OSError("permission denied"))) with pytest.raises(CheckpointReadError, match="checkpoints.yaml"): manager.read() @@ -88,30 +90,40 @@ def test_build_memory_prompt_with_additions(manager): # --------------------------------------------------------------------------- -# write_memory / get_memory +# write_memory / memory_content / memory_path # --------------------------------------------------------------------------- def test_write_memory_and_read_back(manager): - manager.write_phase_checkpoint("p", {}) # ensure parent dir exists manager.write_memory("draft", "Created integration.py and tests.") - assert manager.get_memory("draft") == "Created integration.py and tests." + assert manager.memory_content("draft") == "Created integration.py and tests." def test_write_memory_overwrites(manager): - manager.write_phase_checkpoint("p", {}) manager.write_memory("draft", "first version") manager.write_memory("draft", "second version") - assert manager.get_memory("draft") == "second version" + assert manager.memory_content("draft") == "second version" + + +def test_memory_content_absent_returns_placeholder(manager): + assert manager.memory_content("nonexistent") == "" + + +def test_memory_path_returns_absolute_path(manager): + path = manager.memory_path("phase1") + assert isinstance(path, Path) + assert path.is_absolute() + assert path.name == "phase1_memory.md" -def test_get_memory_absent_returns_placeholder(manager): - assert manager.get_memory("nonexistent") == "" +def test_memory_path_before_write(manager): + path = manager.memory_path("phase1") + assert not path.exists() def test_memory_file_location(manager): - manager.write_phase_checkpoint("p", {}) manager.write_memory("phase1", "content") expected_path = manager._path.parent / "phase1_memory.md" assert expected_path.exists() assert expected_path.read_text() == "content" + assert manager.memory_path("phase1") == expected_path.resolve() diff --git a/ddev/tests/ai/phases/test_orchestrator.py b/ddev/tests/ai/phases/test_orchestrator.py index ad96179dbf2f9..c83b6fff23407 100644 --- a/ddev/tests/ai/phases/test_orchestrator.py +++ b/ddev/tests/ai/phases/test_orchestrator.py @@ -99,19 +99,21 @@ def test_imported_class_not_registered(): assert "BaseMessage" not in registry.known_names() -def test_two_orchestrators_have_independent_registries(tmp_path): +def test_two_orchestrators_have_independent_registries(tmp_path, file_access_policy): """Each PhaseOrchestrator owns its own registry; registering in one does not affect the other.""" o1 = PhaseOrchestrator( flow_yaml_path=tmp_path / "flow.yaml", checkpoint_path=tmp_path / "checkpoints.yaml", runtime_variables={}, anthropic_client=MagicMock(), + file_access_policy=file_access_policy, ) o2 = PhaseOrchestrator( flow_yaml_path=tmp_path / "flow.yaml", checkpoint_path=tmp_path / "checkpoints.yaml", runtime_variables={}, anthropic_client=MagicMock(), + file_access_policy=file_access_policy, ) class ExclusivePhase(Phase): @@ -136,12 +138,13 @@ def test_discover_does_not_mutate_global_state(): # --------------------------------------------------------------------------- -async def test_on_message_received_fatal_on_phase_failed(): +async def test_on_message_received_fatal_on_phase_failed(file_access_policy): orchestrator = PhaseOrchestrator( flow_yaml_path=Path("/fake/flow.yaml"), checkpoint_path=Path("/fake/checkpoints.yaml"), runtime_variables={}, anthropic_client=MagicMock(), + file_access_policy=file_access_policy, ) msg = PhaseFailedMessage(id="f1", phase_id="p1", error="something broke") @@ -149,12 +152,13 @@ async def test_on_message_received_fatal_on_phase_failed(): await orchestrator.on_message_received(msg) -async def test_on_message_received_ignores_other_messages(): +async def test_on_message_received_ignores_other_messages(file_access_policy): orchestrator = PhaseOrchestrator( flow_yaml_path=Path("/fake/flow.yaml"), checkpoint_path=Path("/fake/checkpoints.yaml"), runtime_variables={}, anthropic_client=MagicMock(), + file_access_policy=file_access_policy, ) # These should not raise await orchestrator.on_message_received(PhaseTrigger(id="start", phase_id=None)) @@ -198,12 +202,13 @@ def minimal_flow(tmp_path): return tmp_path -async def test_on_initialize_registers_all_flow_phases(minimal_flow): +async def test_on_initialize_registers_all_flow_phases(minimal_flow, file_access_policy): orchestrator = PhaseOrchestrator( flow_yaml_path=minimal_flow / "flow.yaml", checkpoint_path=minimal_flow / "checkpoints.yaml", runtime_variables={}, anthropic_client=MagicMock(), + file_access_policy=file_access_policy, ) await orchestrator.on_initialize() @@ -212,12 +217,13 @@ async def test_on_initialize_registers_all_flow_phases(minimal_flow): assert phase_names == {"a", "b"} -async def test_on_initialize_wires_dependencies(minimal_flow): +async def test_on_initialize_wires_dependencies(minimal_flow, file_access_policy): orchestrator = PhaseOrchestrator( flow_yaml_path=minimal_flow / "flow.yaml", checkpoint_path=minimal_flow / "checkpoints.yaml", runtime_variables={}, anthropic_client=MagicMock(), + file_access_policy=file_access_policy, ) await orchestrator.on_initialize() @@ -227,12 +233,13 @@ async def test_on_initialize_wires_dependencies(minimal_flow): assert phases_by_name["b"]._dependencies == {"a"} -async def test_on_initialize_submits_initial_phase_trigger(minimal_flow): +async def test_on_initialize_submits_initial_phase_trigger(minimal_flow, file_access_policy): orchestrator = PhaseOrchestrator( flow_yaml_path=minimal_flow / "flow.yaml", checkpoint_path=minimal_flow / "checkpoints.yaml", runtime_variables={}, anthropic_client=MagicMock(), + file_access_policy=file_access_policy, ) await orchestrator.on_initialize() @@ -242,7 +249,7 @@ async def test_on_initialize_submits_initial_phase_trigger(minimal_flow): assert msg.phase_id is None -async def test_on_initialize_unknown_phase_type_raises_flow_config_error(tmp_path): +async def test_on_initialize_unknown_phase_type_raises_flow_config_error(tmp_path, file_access_policy): (tmp_path / "prompts").mkdir() (tmp_path / "prompts" / "writer.md").write_text("system prompt") (tmp_path / "flow.yaml").write_text( @@ -266,12 +273,13 @@ async def test_on_initialize_unknown_phase_type_raises_flow_config_error(tmp_pat checkpoint_path=tmp_path / "checkpoints.yaml", runtime_variables={}, anthropic_client=MagicMock(), + file_access_policy=file_access_policy, ) with pytest.raises(FlowConfigError, match="Unknown phase type"): await orchestrator.on_initialize() -async def test_on_initialize_missing_agent_raises(tmp_path): +async def test_on_initialize_missing_agent_raises(tmp_path, file_access_policy): (tmp_path / "prompts").mkdir() (tmp_path / "flow.yaml").write_text( dedent("""\ @@ -294,17 +302,32 @@ async def test_on_initialize_missing_agent_raises(tmp_path): checkpoint_path=tmp_path / "checkpoints.yaml", runtime_variables={}, anthropic_client=MagicMock(), + file_access_policy=file_access_policy, ) with pytest.raises(FlowConfigError): await orchestrator.on_initialize() +async def test_on_initialize_phases_share_file_registry(minimal_flow, file_access_policy): + orchestrator = PhaseOrchestrator( + flow_yaml_path=minimal_flow / "flow.yaml", + checkpoint_path=minimal_flow / "checkpoints.yaml", + runtime_variables={}, + anthropic_client=MagicMock(), + file_access_policy=file_access_policy, + ) + await orchestrator.on_initialize() + phases = orchestrator._subscribers.get(PhaseTrigger, []) + assert len(phases) >= 2 + assert all(p._file_registry is phases[0]._file_registry for p in phases[1:]) + + # --------------------------------------------------------------------------- # PhaseOrchestrator.on_initialize — orphan-phase validation # --------------------------------------------------------------------------- -async def test_orphan_phase_with_unknown_type_does_not_block_init(tmp_path): +async def test_orphan_phase_with_unknown_type_does_not_block_init(tmp_path, file_access_policy): """A phase defined in phases: but absent from flow: may have an unknown type — no error.""" (tmp_path / "prompts").mkdir() (tmp_path / "prompts" / "writer.md").write_text("system prompt") @@ -335,6 +358,7 @@ async def test_orphan_phase_with_unknown_type_does_not_block_init(tmp_path): checkpoint_path=tmp_path / "checkpoints.yaml", runtime_variables={}, anthropic_client=MagicMock(), + file_access_policy=file_access_policy, ) await orchestrator.on_initialize() @@ -342,7 +366,7 @@ async def test_orphan_phase_with_unknown_type_does_not_block_init(tmp_path): assert {p.name for p in processors} == {"real"} -async def test_phase_in_flow_with_unknown_type_raises(tmp_path): +async def test_phase_in_flow_with_unknown_type_raises(tmp_path, file_access_policy): """A phase referenced from flow: with an unknown type must still raise FlowConfigError.""" (tmp_path / "prompts").mkdir() (tmp_path / "prompts" / "writer.md").write_text("system prompt") @@ -367,12 +391,13 @@ async def test_phase_in_flow_with_unknown_type_raises(tmp_path): checkpoint_path=tmp_path / "checkpoints.yaml", runtime_variables={}, anthropic_client=MagicMock(), + file_access_policy=file_access_policy, ) with pytest.raises(FlowConfigError, match="Unknown phase type"): await orchestrator.on_initialize() -async def test_orphan_phase_logs_warning(tmp_path, caplog): +async def test_orphan_phase_logs_warning(tmp_path, file_access_policy, caplog): """An orphan phase must emit a warning containing its phase id.""" import logging @@ -405,6 +430,7 @@ async def test_orphan_phase_logs_warning(tmp_path, caplog): checkpoint_path=tmp_path / "checkpoints.yaml", runtime_variables={}, anthropic_client=MagicMock(), + file_access_policy=file_access_policy, ) with caplog.at_level(logging.WARNING): await orchestrator.on_initialize() @@ -417,22 +443,24 @@ async def test_orphan_phase_logs_warning(tmp_path, caplog): # --------------------------------------------------------------------------- -async def test_on_finalize_no_failure_is_noop(): +async def test_on_finalize_no_failure_is_noop(tmp_path, file_access_policy): orchestrator = PhaseOrchestrator( flow_yaml_path=Path("/fake/flow.yaml"), checkpoint_path=Path("/fake/checkpoints.yaml"), runtime_variables={}, anthropic_client=MagicMock(), + file_access_policy=file_access_policy, ) await orchestrator.on_finalize(None) # must not raise -async def test_on_finalize_after_phase_failed_raises(): +async def test_on_finalize_after_phase_failed_raises(tmp_path, file_access_policy): orchestrator = PhaseOrchestrator( flow_yaml_path=Path("/fake/flow.yaml"), checkpoint_path=Path("/fake/checkpoints.yaml"), runtime_variables={}, anthropic_client=MagicMock(), + file_access_policy=file_access_policy, ) msg = PhaseFailedMessage(id="f1", phase_id="p1", error="boom") with pytest.raises(FatalProcessingError): @@ -442,7 +470,7 @@ async def test_on_finalize_after_phase_failed_raises(): await orchestrator.on_finalize(None) -def test_run_raises_runtime_error_when_phase_fails(tmp_path): +def test_run_raises_runtime_error_when_phase_fails(tmp_path, file_access_policy): """Full pipeline: a failing phase must cause run() to raise RuntimeError.""" (tmp_path / "prompts").mkdir() (tmp_path / "prompts" / "writer.md").write_text("system prompt") @@ -472,6 +500,7 @@ async def process_message(self, message: PhaseTrigger) -> None: checkpoint_path=tmp_path / "checkpoints.yaml", runtime_variables={}, anthropic_client=MagicMock(), + file_access_policy=file_access_policy, grace_period=0.1, ) orchestrator._phase_registry.register("FailingPhase", FailingPhase) diff --git a/ddev/tests/ai/react/test_callbacks.py b/ddev/tests/ai/react/test_callbacks.py index 4b6f3618411df..a8780b1cd006f 100644 --- a/ddev/tests/ai/react/test_callbacks.py +++ b/ddev/tests/ai/react/test_callbacks.py @@ -252,3 +252,133 @@ async def a2() -> None: await cb.fire_before_compact() await cb.fire_after_compact() assert fired == ["before-1", "before-2", "after-1", "after-2"] + + +# --------------------------------------------------------------------------- +# on_phase_start +# --------------------------------------------------------------------------- + + +async def test_phase_start_registered_and_fired() -> None: + cb = CallbackSet() + fired: list[str] = [] + + @cb.on_phase_start + async def h(phase_id: str) -> None: + fired.append(phase_id) + + await cb.fire_phase_start("my-phase") + assert fired == ["my-phase"] + + +async def test_phase_start_receives_correct_phase_id() -> None: + cb = CallbackSet() + received: list[str] = [] + + @cb.on_phase_start + async def h(phase_id: str) -> None: + received.append(phase_id) + + await cb.fire_phase_start("draft") + assert received == ["draft"] + + +async def test_phase_start_multiple_handlers_all_fire_in_order() -> None: + cb = CallbackSet() + fired: list[int] = [] + + @cb.on_phase_start + async def first(phase_id: str) -> None: + fired.append(1) + + @cb.on_phase_start + async def second(phase_id: str) -> None: + fired.append(2) + + @cb.on_phase_start + async def third(phase_id: str) -> None: + fired.append(3) + + await cb.fire_phase_start("p") + assert fired == [1, 2, 3] + + +async def test_phase_start_exception_is_swallowed() -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_phase_start + async def bad(phase_id: str) -> None: + raise RuntimeError("boom") + + @cb.on_phase_start + async def good(phase_id: str) -> None: + fired.append(True) + + await cb.fire_phase_start("p") + assert fired == [True] + + +# --------------------------------------------------------------------------- +# on_before_agent_send +# --------------------------------------------------------------------------- + + +async def test_before_agent_send_registered_and_fired() -> None: + cb = CallbackSet() + fired: list[int] = [] + + @cb.on_before_agent_send + async def h(iteration: int) -> None: + fired.append(iteration) + + await cb.fire_before_agent_send(3) + assert fired == [3] + + +async def test_before_agent_send_receives_correct_iteration() -> None: + cb = CallbackSet() + received: list[int] = [] + + @cb.on_before_agent_send + async def h(iteration: int) -> None: + received.append(iteration) + + await cb.fire_before_agent_send(7) + assert received == [7] + + +async def test_before_agent_send_multiple_handlers_all_fire_in_order() -> None: + cb = CallbackSet() + fired: list[int] = [] + + @cb.on_before_agent_send + async def first(iteration: int) -> None: + fired.append(1) + + @cb.on_before_agent_send + async def second(iteration: int) -> None: + fired.append(2) + + @cb.on_before_agent_send + async def third(iteration: int) -> None: + fired.append(3) + + await cb.fire_before_agent_send(1) + assert fired == [1, 2, 3] + + +async def test_before_agent_send_exception_is_swallowed() -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_before_agent_send + async def bad(iteration: int) -> None: + raise RuntimeError("boom") + + @cb.on_before_agent_send + async def good(iteration: int) -> None: + fired.append(True) + + await cb.fire_before_agent_send(1) + assert fired == [True] diff --git a/ddev/tests/ai/react/test_process.py b/ddev/tests/ai/react/test_process.py index 07dc4fce7b3bc..23a8aefe308a5 100644 --- a/ddev/tests/ai/react/test_process.py +++ b/ddev/tests/ai/react/test_process.py @@ -13,8 +13,8 @@ from ddev.ai.react.callbacks import CallbackSet from ddev.ai.react.process import ReActProcess from ddev.ai.react.types import ReActResult -from ddev.ai.tools.core.registry import ToolRegistry from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.registry import ToolRegistry _TOOL_RESULT_DATA: str = "ok" diff --git a/ddev/tests/ai/tools/fs/conftest.py b/ddev/tests/ai/tools/fs/conftest.py index 12ae9e34eb1d5..defedae7d9269 100644 --- a/ddev/tests/ai/tools/fs/conftest.py +++ b/ddev/tests/ai/tools/fs/conftest.py @@ -7,33 +7,52 @@ from ddev.ai.tools.fs.append_file import AppendFileTool from ddev.ai.tools.fs.create_file import CreateFileTool from ddev.ai.tools.fs.edit_file import EditFileTool +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.fs.mkdir import MkdirTool from ddev.ai.tools.fs.read_file import ReadFileTool +OWNER_ID = "test-agent" + + +@pytest.fixture +def owner_id() -> str: + return OWNER_ID + + +@pytest.fixture +def permissive_policy(tmp_path) -> FileAccessPolicy: + return FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + + +@pytest.fixture +def registry(permissive_policy: FileAccessPolicy) -> FileRegistry: + return FileRegistry(policy=permissive_policy) + @pytest.fixture -def registry() -> FileRegistry: - return FileRegistry() +def read_tool(registry: FileRegistry, owner_id: str) -> ReadFileTool: + return ReadFileTool(registry, owner_id) @pytest.fixture -def read_tool(registry: FileRegistry) -> ReadFileTool: - return ReadFileTool(registry) +def create_tool(registry: FileRegistry, owner_id: str) -> CreateFileTool: + return CreateFileTool(registry, owner_id) @pytest.fixture -def create_tool(registry: FileRegistry) -> CreateFileTool: - return CreateFileTool(registry) +def edit_tool(registry: FileRegistry, owner_id: str) -> EditFileTool: + return EditFileTool(registry, owner_id) @pytest.fixture -def edit_tool(registry: FileRegistry) -> EditFileTool: - return EditFileTool(registry) +def append_tool(registry: FileRegistry, owner_id: str) -> AppendFileTool: + return AppendFileTool(registry, owner_id) @pytest.fixture -def append_tool(registry: FileRegistry) -> AppendFileTool: - return AppendFileTool(registry) +def mkdir_tool(permissive_policy: FileAccessPolicy) -> MkdirTool: + return MkdirTool(permissive_policy) @pytest.fixture diff --git a/ddev/tests/ai/tools/fs/test_append_file.py b/ddev/tests/ai/tools/fs/test_append_file.py index 289142e378191..7b0b6cfb33d91 100644 --- a/ddev/tests/ai/tools/fs/test_append_file.py +++ b/ddev/tests/ai/tools/fs/test_append_file.py @@ -9,9 +9,11 @@ from ddev.ai.tools.fs.create_file import CreateFileTool from ddev.ai.tools.fs.file_registry import FileRegistry +from .conftest import OWNER_ID + def test_tool_name(registry: FileRegistry) -> None: - assert AppendFileTool(registry).name == "append_file" + assert AppendFileTool(registry, OWNER_ID).name == "append_file" @pytest.mark.parametrize( @@ -76,7 +78,7 @@ async def test_append_file_updates_registry(append_tool: AppendFileTool, registr await append_tool.run({"path": str(known_file), "content": "extra\n"}) new_content = known_file.read_text(encoding="utf-8") - assert registry.verify(str(known_file), new_content) is True + assert registry.verify(OWNER_ID, str(known_file), new_content) is True async def test_append_file_oserror_on_write(append_tool: AppendFileTool, registry: FileRegistry, known_file) -> None: @@ -89,4 +91,4 @@ async def test_append_file_oserror_on_write(append_tool: AppendFileTool, registr assert result.error is not None # File must be untouched and registry must still reflect the original content assert known_file.read_text(encoding="utf-8") == original_content - assert registry.verify(str(known_file), original_content) is True + assert registry.verify(OWNER_ID, str(known_file), original_content) is True diff --git a/ddev/tests/ai/tools/fs/test_base.py b/ddev/tests/ai/tools/fs/test_base.py index d19d71092322d..d7183726c0779 100644 --- a/ddev/tests/ai/tools/fs/test_base.py +++ b/ddev/tests/ai/tools/fs/test_base.py @@ -9,8 +9,11 @@ from ddev.ai.tools.core.base import BaseToolInput from ddev.ai.tools.core.types import ToolResult from ddev.ai.tools.fs.base import FileRegistryTool +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy from ddev.ai.tools.fs.file_registry import FileRegistry +OWNER_ID = "test-agent" + # --------------------------------------------------------------------------- # Minimal concrete subclass for testing # --------------------------------------------------------------------------- @@ -37,13 +40,13 @@ async def __call__(self, tool_input: DummyInput) -> ToolResult: @pytest.fixture -def registry() -> FileRegistry: - return FileRegistry() +def registry(tmp_path) -> FileRegistry: + return FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) @pytest.fixture def tool(registry: FileRegistry) -> DummyTool: - return DummyTool(registry) + return DummyTool(registry, OWNER_ID) # --------------------------------------------------------------------------- @@ -64,7 +67,7 @@ def test_read_verified_fails_if_not_known(tool: DummyTool, tmp_path) -> None: def test_read_verified_fails_if_file_changed_externally(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: f = tmp_path / "file.txt" f.write_text("original", encoding="utf-8") - registry.record(str(f), "original") + registry.record(OWNER_ID, str(f), "original") f.write_text("modified", encoding="utf-8") @@ -79,7 +82,7 @@ def test_read_verified_fails_if_file_changed_externally(tool: DummyTool, registr def test_read_verified_succeeds_if_content_matches(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: f = tmp_path / "file.txt" f.write_text("hello", encoding="utf-8") - registry.record(str(f), "hello") + registry.record(OWNER_ID, str(f), "hello") content, error = tool._read_verified(str(f)) @@ -89,8 +92,7 @@ def test_read_verified_succeeds_if_content_matches(tool: DummyTool, registry: Fi def test_read_verified_handles_oserror(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: path = str(tmp_path / "ghost.txt") - # Record the path so it passes the is_known check, but never create the file - registry.record(path, "anything") + registry.record(OWNER_ID, path, "anything") content, error = tool._read_verified(path) @@ -99,6 +101,32 @@ def test_read_verified_handles_oserror(tool: DummyTool, registry: FileRegistry, assert error.success is False +def test_read_verified_handles_unicode_decode_error(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: + f = tmp_path / "binary.bin" + f.write_bytes(b"\xff\xfe invalid utf-8") + registry.record(OWNER_ID, str(f), "anything") + + content, error = tool._read_verified(str(f)) + + assert content == "" + assert error is not None + assert error.success is False + + +def test_read_verified_is_isolated_between_agents(registry: FileRegistry, tmp_path) -> None: + """A file registered by agent A cannot be read-verified by agent B.""" + f = tmp_path / "file.txt" + f.write_text("hello", encoding="utf-8") + registry.record("agent-a", str(f), "hello") + + tool_b = DummyTool(registry, "agent-b") + content, error = tool_b._read_verified(str(f)) + + assert content == "" + assert error is not None + assert "Not authorized" in error.error + + # --------------------------------------------------------------------------- # _register # --------------------------------------------------------------------------- @@ -108,8 +136,8 @@ def test_register_registers_path(tool: DummyTool, registry: FileRegistry, tmp_pa path = str(tmp_path / "file.txt") tool._register(path, "written") - assert registry.is_known(path) is True - assert registry.verify(path, "written") is True + assert registry.is_known(OWNER_ID, path) is True + assert registry.verify(OWNER_ID, path, "written") is True def test_register_updates_hash_after_register(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: @@ -117,5 +145,13 @@ def test_register_updates_hash_after_register(tool: DummyTool, registry: FileReg tool._register(path, "old") tool._register(path, "new") - assert registry.verify(path, "new") is True - assert registry.verify(path, "old") is False + assert registry.verify(OWNER_ID, path, "new") is True + assert registry.verify(OWNER_ID, path, "old") is False + + +def test_register_scopes_to_the_tools_agent(registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + DummyTool(registry, "agent-a")._register(path, "x") + + assert registry.is_known("agent-a", path) is True + assert registry.is_known("agent-b", path) is False diff --git a/ddev/tests/ai/tools/fs/test_create_file.py b/ddev/tests/ai/tools/fs/test_create_file.py index 8b0c0296fa38a..226a45a9d732d 100644 --- a/ddev/tests/ai/tools/fs/test_create_file.py +++ b/ddev/tests/ai/tools/fs/test_create_file.py @@ -6,9 +6,11 @@ from ddev.ai.tools.fs.create_file import CreateFileTool from ddev.ai.tools.fs.file_registry import FileRegistry +from .conftest import OWNER_ID + def test_tool_name(registry: FileRegistry) -> None: - assert CreateFileTool(registry).name == "create_file" + assert CreateFileTool(registry, OWNER_ID).name == "create_file" async def test_create_file_success(create_tool: CreateFileTool, tmp_path) -> None: @@ -48,17 +50,17 @@ async def test_create_file_fails_if_file_already_exists( result = await create_tool.run({"path": str(f), "content": "new"}) assert result.success is False - assert result.error is not None + assert "File already exists" in result.error assert f.read_text(encoding="utf-8") == "original" - assert not registry.is_known(str(f)) + assert not registry.is_known(OWNER_ID, str(f)) async def test_create_tool_registers_in_registry(create_tool: CreateFileTool, registry: FileRegistry, tmp_path) -> None: f = tmp_path / "file.txt" await create_tool.run({"path": str(f), "content": "hi"}) - assert registry.is_known(str(f)) is True - assert registry.verify(str(f), "hi") is True + assert registry.is_known(OWNER_ID, str(f)) is True + assert registry.verify(OWNER_ID, str(f), "hi") is True async def test_create_file_oserror_on_mkdir(create_tool: CreateFileTool, registry: FileRegistry, tmp_path) -> None: @@ -70,15 +72,15 @@ async def test_create_file_oserror_on_mkdir(create_tool: CreateFileTool, registr assert result.success is False assert result.error is not None assert not f.exists() - assert not registry.is_known(str(f)) + assert not registry.is_known(OWNER_ID, str(f)) async def test_create_file_oserror_on_write(create_tool: CreateFileTool, registry: FileRegistry, tmp_path) -> None: f = tmp_path / "new.txt" - with patch("pathlib.Path.write_text", side_effect=PermissionError("permission denied")): + with patch("builtins.open", side_effect=PermissionError("permission denied")): result = await create_tool.run({"path": str(f), "content": "hi"}) assert result.success is False assert result.error is not None - assert not registry.is_known(str(f)) + assert not registry.is_known(OWNER_ID, str(f)) diff --git a/ddev/tests/ai/tools/fs/test_edit_file.py b/ddev/tests/ai/tools/fs/test_edit_file.py index 27c8b87cedce2..078b46b8bb3f3 100644 --- a/ddev/tests/ai/tools/fs/test_edit_file.py +++ b/ddev/tests/ai/tools/fs/test_edit_file.py @@ -9,9 +9,11 @@ from ddev.ai.tools.fs.edit_file import EditFileTool from ddev.ai.tools.fs.file_registry import FileRegistry +from .conftest import OWNER_ID + def test_tool_name(registry: FileRegistry) -> None: - assert EditFileTool(registry).name == "edit_file" + assert EditFileTool(registry, OWNER_ID).name == "edit_file" async def test_edit_file_replaces_string(edit_tool: EditFileTool, known_file) -> None: @@ -75,8 +77,8 @@ async def test_edit_file_updates_registry(edit_tool: EditFileTool, registry: Fil await edit_tool.run({"path": str(known_file), "old_string": "line one", "new_string": "LINE ONE"}) new_content = known_file.read_text(encoding="utf-8") - assert registry.verify(str(known_file), new_content) is True - assert registry.verify(str(known_file), "line one\nline two\nline three\n") is False + assert registry.verify(OWNER_ID, str(known_file), new_content) is True + assert registry.verify(OWNER_ID, str(known_file), "line one\nline two\nline three\n") is False @pytest.mark.parametrize( @@ -108,4 +110,4 @@ async def test_edit_file_oserror_on_write(edit_tool: EditFileTool, registry: Fil assert result.error is not None # File must be untouched and registry must still reflect the original content assert known_file.read_text(encoding="utf-8") == original_content - assert registry.verify(str(known_file), original_content) is True + assert registry.verify(OWNER_ID, str(known_file), original_content) is True diff --git a/ddev/tests/ai/tools/fs/test_file_access_policy.py b/ddev/tests/ai/tools/fs/test_file_access_policy.py new file mode 100644 index 0000000000000..01371aad4a307 --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_file_access_policy.py @@ -0,0 +1,263 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ddev.ai.tools.fs.file_access_policy import FileAccessError, FileAccessPolicy, canonicalize_path + +# --------------------------------------------------------------------------- +# canonicalize_path +# --------------------------------------------------------------------------- + + +def test_canonicalize_path_expands_tilde(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) # Windows uses USERPROFILE, not HOME + assert canonicalize_path("~/foo") == tmp_path / "foo" + + +def test_canonicalize_path_is_idempotent(tmp_path) -> None: + p = tmp_path / "sub" / "file.txt" + assert canonicalize_path(str(p)) == canonicalize_path(canonicalize_path(str(p))) + + +def test_canonicalize_path_accepts_path_object(tmp_path) -> None: + assert canonicalize_path(tmp_path / "x.txt") == tmp_path / "x.txt" + + +# --------------------------------------------------------------------------- +# assert_* return canonical path +# --------------------------------------------------------------------------- + + +def test_assert_readable_returns_canonical_path(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + returned = policy.assert_readable(str(tmp_path / "file.txt")) + assert returned == tmp_path / "file.txt" + + +def test_assert_writable_returns_canonical_path(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + returned = policy.assert_writable(str(tmp_path / "file.txt")) + assert returned == tmp_path / "file.txt" + + +def test_assert_readable_expands_tilde(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) # Windows uses USERPROFILE, not HOME + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + returned = policy.assert_readable("~/file.txt") + assert returned == tmp_path / "file.txt" + + +# --------------------------------------------------------------------------- +# write_root enforcement +# --------------------------------------------------------------------------- + + +def test_write_inside_root_allowed(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + policy.assert_writable(str(tmp_path / "sub" / "file.txt")) + + +def test_write_outside_root_denied(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + with pytest.raises(FileAccessError, match="outside write root"): + policy.assert_writable(str(tmp_path.parent / "outside.txt")) + + +def test_write_traversal_denied(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + with pytest.raises(FileAccessError, match="outside write root"): + policy.assert_writable(str(tmp_path / ".." / "escape.txt")) + + +def test_write_symlink_escaping_root_denied(tmp_path) -> None: + outside = tmp_path.parent / "outside_target" + outside.mkdir(exist_ok=True) + link = tmp_path / "link_to_outside" + link.symlink_to(outside) + + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + with pytest.raises(FileAccessError, match="outside write root"): + policy.assert_writable(str(link / "file.txt")) + + +# --------------------------------------------------------------------------- +# Inside write_root: deny patterns are bypassed for both reads and writes +# --------------------------------------------------------------------------- + + +def test_read_denied_basename_inside_write_root_is_allowed(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=(".env",)) + policy.assert_readable(str(tmp_path / ".env")) + + +def test_read_denied_path_pattern_inside_write_root_is_allowed(tmp_path) -> None: + secrets = tmp_path / "secrets" + secrets.mkdir() + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=(f"{secrets}/*",)) + policy.assert_readable(str(secrets / "key.txt")) + + +def test_write_denied_basename_inside_write_root_is_allowed(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=(".env",)) + policy.assert_writable(str(tmp_path / ".env")) + + +def test_write_denied_path_pattern_inside_write_root_is_allowed(tmp_path) -> None: + secrets = tmp_path / "secrets" + secrets.mkdir() + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=(f"{secrets}/*",)) + policy.assert_writable(str(secrets / "x.txt")) + + +# --------------------------------------------------------------------------- +# Outside write_root: deny patterns still apply to reads +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "filename", + [".env", ".env.local", ".envrc", ".netrc", "secret.pem", "private.key"], +) +def test_basename_pattern_denies_read_outside_write_root(tmp_path, filename) -> None: + write_root = tmp_path / "sandbox" + policy = FileAccessPolicy(write_root=write_root) # default patterns + with pytest.raises(FileAccessError, match="Read denied"): + policy.assert_readable(str(tmp_path / filename)) + + +@pytest.mark.parametrize("filename", ["app.py", "README.md", "config.yaml", "env.txt"]) +def test_basename_pattern_allows_unrelated_outside_write_root(tmp_path, filename) -> None: + write_root = tmp_path / "sandbox" + policy = FileAccessPolicy(write_root=write_root) + policy.assert_readable(str(tmp_path / filename)) + + +def test_custom_basename_pattern_denies_outside_write_root(tmp_path) -> None: + write_root = tmp_path / "sandbox" + policy = FileAccessPolicy(write_root=write_root, deny_patterns=("*.secret",)) + with pytest.raises(FileAccessError): + policy.assert_readable(str(tmp_path / "api.secret")) + policy.assert_readable(str(tmp_path / "api.public")) + + +# --------------------------------------------------------------------------- +# Path pattern semantics — match against full canonical path string +# --------------------------------------------------------------------------- + + +def test_path_pattern_denies_outside_write_root(tmp_path) -> None: + write_root = tmp_path / "sandbox" + denied = tmp_path / "secrets" + denied.mkdir() + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{denied}/*",)) + with pytest.raises(FileAccessError): + policy.assert_readable(str(denied / "x.txt")) + # fnmatch's '*' is greedy across '/', so subpaths are also denied + with pytest.raises(FileAccessError): + policy.assert_readable(str(denied / "sub" / "deep.txt")) + + +def test_path_pattern_allows_siblings_outside_write_root(tmp_path) -> None: + write_root = tmp_path / "sandbox" + denied = tmp_path / "secrets" + denied.mkdir() + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{denied}/*",)) + policy.assert_readable(str(tmp_path / "public.txt")) + + +def test_specific_path_pattern_denies_only_that_file(tmp_path) -> None: + write_root = tmp_path / "sandbox" + (tmp_path / "secrets").mkdir() + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{tmp_path}/secrets/credentials",)) + with pytest.raises(FileAccessError): + policy.assert_readable(str(tmp_path / "secrets" / "credentials")) + # same name elsewhere is fine + policy.assert_readable(str(tmp_path / "credentials")) + + +def test_path_pattern_with_glob_in_middle(tmp_path) -> None: + write_root = tmp_path / "sandbox" + base = tmp_path / "dir" + base.mkdir() + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{base}/*credentials*",)) + with pytest.raises(FileAccessError): + policy.assert_readable(str(base / "my_credentials_file")) + # '*' spans '/', so a deeper file with 'credentials' in the name is still denied + with pytest.raises(FileAccessError): + policy.assert_readable(str(base / "sub" / "credentials.txt")) + + +def test_path_pattern_resolves_symlinked_root(tmp_path) -> None: + """Pattern's static prefix is resolved at __init__ so symlinks can't bypass.""" + write_root = tmp_path / "sandbox" + real = tmp_path / "real_secrets" + real.mkdir() + (real / "key").write_text("x") + link = tmp_path / "link_secrets" + link.symlink_to(real) + + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{link}/*",)) + # accessing via the real path is denied + with pytest.raises(FileAccessError): + policy.assert_readable(str(real / "key")) + # accessing via the symlinked path is also denied (same resolved target) + with pytest.raises(FileAccessError): + policy.assert_readable(str(link / "key")) + + +def test_symlink_to_denied_target_is_blocked(tmp_path) -> None: + """A symlink in an allowed dir pointing into a denied tree is still denied.""" + write_root = tmp_path / "sandbox" + denied = tmp_path / "secrets" + denied.mkdir() + target = denied / "key" + target.write_text("x") + public = tmp_path / "innocent_link" + public.symlink_to(target) + + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{denied}/*",)) + with pytest.raises(FileAccessError): + policy.assert_readable(str(public)) + + +def test_traversal_does_not_bypass(tmp_path) -> None: + write_root = tmp_path / "sandbox" + denied = tmp_path / "secrets" + denied.mkdir() + (denied / "key").write_text("x") + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{denied}/*",)) + with pytest.raises(FileAccessError): + policy.assert_readable(str(tmp_path / "public" / ".." / "secrets" / "key")) + + +# --------------------------------------------------------------------------- +# Properties +# --------------------------------------------------------------------------- + + +def test_deny_patterns_property_preserves_input(tmp_path) -> None: + patterns = ("*.pem", "~/.ssh/*", ".env") + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=patterns) + assert policy.deny_patterns == patterns + + +def test_basename_patterns_filters_to_basename_only(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=("*.pem", "~/.ssh/*", ".env")) + assert set(policy.basename_patterns) == {"*.pem", ".env"} + + +# --------------------------------------------------------------------------- +# DEFAULT_DENY_PATTERNS — coverage for the rooted secret directories +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("root", ["~/.aws", "~/.kube", "~/.gnupg", "~/.docker", "~/.config/gcloud", "~/.ssh"]) +def test_read_denied_by_default_path_pattern(tmp_path, root) -> None: + write_root = tmp_path / "sandbox" + policy = FileAccessPolicy(write_root=write_root) + resolved_root = canonicalize_path(root) + with pytest.raises(FileAccessError, match="Read denied"): + policy.assert_readable(str(resolved_root / "config")) diff --git a/ddev/tests/ai/tools/fs/test_file_registry.py b/ddev/tests/ai/tools/fs/test_file_registry.py index 17ab79dc73909..8be0779b5951b 100644 --- a/ddev/tests/ai/tools/fs/test_file_registry.py +++ b/ddev/tests/ai/tools/fs/test_file_registry.py @@ -3,12 +3,16 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import pytest +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy from ddev.ai.tools.fs.file_registry import FileRegistry +OWNER_A = "agent-a" +OWNER_B = "agent-b" + @pytest.fixture -def registry() -> FileRegistry: - return FileRegistry() +def registry(tmp_path) -> FileRegistry: + return FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) # --------------------------------------------------------------------------- @@ -26,13 +30,20 @@ def registry() -> FileRegistry: def test_is_known(registry: FileRegistry, tmp_path, record, expected) -> None: path = str(tmp_path / "file.txt") if record: - registry.record(path, "hello") - assert registry.is_known(path) is expected + registry.record(OWNER_A, path, "hello") + assert registry.is_known(OWNER_A, path) is expected def test_is_known_different_path(registry: FileRegistry, tmp_path) -> None: - registry.record(str(tmp_path / "other.txt"), "hello") - assert registry.is_known(str(tmp_path / "file.txt")) is False + registry.record(OWNER_A, str(tmp_path / "other.txt"), "hello") + assert registry.is_known(OWNER_A, str(tmp_path / "file.txt")) is False + + +def test_is_known_is_scoped_to_owner(registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + registry.record(OWNER_A, path, "hello") + assert registry.is_known(OWNER_A, path) is True + assert registry.is_known(OWNER_B, path) is False # --------------------------------------------------------------------------- @@ -51,8 +62,14 @@ def test_is_known_different_path(registry: FileRegistry, tmp_path) -> None: def test_verify(registry: FileRegistry, tmp_path, recorded_content, verify_content, expected) -> None: path = str(tmp_path / "file.txt") if recorded_content is not None: - registry.record(path, recorded_content) - assert registry.verify(path, verify_content) is expected + registry.record(OWNER_A, path, recorded_content) + assert registry.verify(OWNER_A, path, verify_content) is expected + + +def test_verify_fails_for_different_agent(registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + registry.record(OWNER_A, path, "hello") + assert registry.verify(OWNER_B, path, "hello") is False # --------------------------------------------------------------------------- @@ -60,13 +77,24 @@ def test_verify(registry: FileRegistry, tmp_path, recorded_content, verify_conte # --------------------------------------------------------------------------- -def test_record_overwrites_previous_hash(registry: FileRegistry, tmp_path) -> None: +def test_record_overwrites_previous_hash_within_agent(registry: FileRegistry, tmp_path) -> None: path = str(tmp_path / "file.txt") - registry.record(path, "old") - registry.record(path, "new") + registry.record(OWNER_A, path, "old") + registry.record(OWNER_A, path, "new") - assert registry.verify(path, "new") is True - assert registry.verify(path, "old") is False + assert registry.verify(OWNER_A, path, "new") is True + assert registry.verify(OWNER_A, path, "old") is False + + +def test_record_does_not_cross_agents(registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + registry.record(OWNER_A, path, "from-a") + registry.record(OWNER_B, path, "from-b") + + assert registry.verify(OWNER_A, path, "from-a") is True + assert registry.verify(OWNER_A, path, "from-b") is False + assert registry.verify(OWNER_B, path, "from-b") is True + assert registry.verify(OWNER_B, path, "from-a") is False # --------------------------------------------------------------------------- @@ -75,19 +103,31 @@ def test_record_overwrites_previous_hash(registry: FileRegistry, tmp_path) -> No def test_normalize_relative_and_absolute_are_same_key(registry: FileRegistry, tmp_path, monkeypatch) -> None: - # Make tmp_path the cwd so that a relative path resolves to the same absolute path monkeypatch.chdir(tmp_path) abs_path = str(tmp_path / "file.txt") rel_path = "file.txt" - registry.record(abs_path, "hello") - assert registry.is_known(rel_path) is True - assert registry.verify(rel_path, "hello") is True + registry.record(OWNER_A, abs_path, "hello") + assert registry.is_known(OWNER_A, rel_path) is True + assert registry.verify(OWNER_A, rel_path, "hello") is True + + +def test_normalize_tilde_and_absolute_are_same_key(registry: FileRegistry, tmp_path, monkeypatch) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) # Windows uses USERPROFILE, not HOME + + tilde_path = "~/foo.txt" + abs_path = str(tmp_path / "foo.txt") + + registry.record(OWNER_A, tilde_path, "hello") + assert registry.is_known(OWNER_A, abs_path) is True + assert registry.is_known(OWNER_A, tilde_path) is True + assert registry.verify(OWNER_A, abs_path, "hello") is True # --------------------------------------------------------------------------- -# lock_for +# lock_for — shared across agents so concurrent writes serialize on the path # --------------------------------------------------------------------------- diff --git a/ddev/tests/ai/tools/fs/test_mkdir.py b/ddev/tests/ai/tools/fs/test_mkdir.py new file mode 100644 index 0000000000000..830a555ee207d --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_mkdir.py @@ -0,0 +1,47 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from unittest.mock import patch + +from ddev.ai.tools.fs.mkdir import MkdirTool + + +def test_tool_name(mkdir_tool: MkdirTool) -> None: + assert mkdir_tool.name == "mkdir" + + +async def test_mkdir_creates_directory(mkdir_tool: MkdirTool, tmp_path) -> None: + d = tmp_path / "new_dir" + + result = await mkdir_tool.run({"path": str(d)}) + + assert result.success is True + assert d.is_dir() + + +async def test_mkdir_creates_nested_directories(mkdir_tool: MkdirTool, tmp_path) -> None: + d = tmp_path / "a" / "b" / "c" + + result = await mkdir_tool.run({"path": str(d)}) + + assert result.success is True + assert d.is_dir() + + +async def test_mkdir_is_idempotent(mkdir_tool: MkdirTool, tmp_path) -> None: + d = tmp_path / "existing" + d.mkdir() + + result = await mkdir_tool.run({"path": str(d)}) + + assert result.success is True + + +async def test_mkdir_oserror_returns_failure(mkdir_tool: MkdirTool, tmp_path) -> None: + d = tmp_path / "denied" + + with patch("pathlib.Path.mkdir", side_effect=PermissionError("permission denied")): + result = await mkdir_tool.run({"path": str(d)}) + + assert result.success is False + assert result.error is not None diff --git a/ddev/tests/ai/tools/fs/test_policy_enforcement.py b/ddev/tests/ai/tools/fs/test_policy_enforcement.py new file mode 100644 index 0000000000000..57189debd05f9 --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_policy_enforcement.py @@ -0,0 +1,336 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +"""End-to-end policy enforcement: tools must respect the two-zone read/write model.""" + +from unittest.mock import AsyncMock, patch + +import pytest + +from ddev.ai.tools.fs.append_file import AppendFileTool +from ddev.ai.tools.fs.create_file import CreateFileTool +from ddev.ai.tools.fs.edit_file import EditFileTool +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.fs.mkdir import MkdirTool +from ddev.ai.tools.fs.read_file import ReadFileTool +from ddev.ai.tools.shell.grep import GrepTool + +OWNER_ID = "test-agent" + + +@pytest.fixture +def sandbox(tmp_path): + """Write root — a subdirectory of tmp_path so files at the tmp_path level are outside it.""" + s = tmp_path / "sandbox" + s.mkdir() + return s + + +@pytest.fixture +def sandboxed_registry(sandbox) -> FileRegistry: + return FileRegistry(policy=FileAccessPolicy(write_root=sandbox)) + + +# --------------------------------------------------------------------------- +# Write tools refuse paths outside write_root +# --------------------------------------------------------------------------- + + +async def test_create_file_refuses_outside_write_root(tmp_path, sandboxed_registry) -> None: + tool = CreateFileTool(sandboxed_registry, OWNER_ID) + outside = tmp_path / "outside.txt" + result = await tool.run({"path": str(outside), "content": "x"}) + assert result.success is False + assert "outside write root" in result.error + assert not outside.exists() + + +async def test_create_file_allows_inside_write_root(sandbox, sandboxed_registry) -> None: + tool = CreateFileTool(sandboxed_registry, OWNER_ID) + target = sandbox / "nested" / "file.txt" + result = await tool.run({"path": str(target), "content": "x"}) + assert result.success is True + assert target.read_text() == "x" + + +async def test_edit_file_refuses_outside_write_root(tmp_path, sandboxed_registry) -> None: + outside = tmp_path / "outside.txt" + outside.write_text("old") + sandboxed_registry.record(OWNER_ID, str(outside), "old") + + tool = EditFileTool(sandboxed_registry, OWNER_ID) + result = await tool.run({"path": str(outside), "old_string": "old", "new_string": "new"}) + assert result.success is False + assert "outside write root" in result.error + assert outside.read_text() == "old" + + +async def test_append_file_refuses_outside_write_root(tmp_path, sandboxed_registry) -> None: + outside = tmp_path / "outside.txt" + outside.write_text("hello") + sandboxed_registry.record(OWNER_ID, str(outside), "hello") + + tool = AppendFileTool(sandboxed_registry, OWNER_ID) + result = await tool.run({"path": str(outside), "content": " world"}) + assert result.success is False + assert "outside write root" in result.error + + +async def test_mkdir_refuses_outside_write_root(tmp_path, sandboxed_registry) -> None: + tool = MkdirTool(sandboxed_registry.policy) + outside = tmp_path / "outside_dir" + result = await tool.run({"path": str(outside)}) + assert result.success is False + assert "outside write root" in result.error + assert not outside.exists() + + +async def test_mkdir_allows_inside_write_root(sandbox, sandboxed_registry) -> None: + tool = MkdirTool(sandboxed_registry.policy) + target = sandbox / "a" / "b" / "c" + result = await tool.run({"path": str(target)}) + assert result.success is True + assert target.is_dir() + + +# --------------------------------------------------------------------------- +# Inside write_root: deny patterns are bypassed for reads and writes +# --------------------------------------------------------------------------- + + +async def test_write_denied_name_inside_write_root_is_allowed(sandbox, sandboxed_registry) -> None: + """Agents must be able to write .env and similar files inside their sandbox.""" + tool = CreateFileTool(sandboxed_registry, OWNER_ID) + target = sandbox / ".env" + result = await tool.run({"path": str(target), "content": "SECRET=1"}) + assert result.success is True + assert target.exists() + assert target.read_text() == "SECRET=1" + + +async def test_read_denied_name_inside_write_root_is_allowed(sandbox, sandboxed_registry) -> None: + """Agents must be able to read back files they created inside their sandbox.""" + target = sandbox / ".env" + target.write_text("SECRET=1") + sandboxed_registry.record(OWNER_ID, str(target), "SECRET=1") + + tool = ReadFileTool(sandboxed_registry, OWNER_ID) + result = await tool.run({"path": str(target)}) + assert result.success is True + + +# --------------------------------------------------------------------------- +# Outside write_root: read denylist still applies +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("filename", [".env", ".envrc", ".netrc", "api.pem", "private.key"]) +async def test_read_file_refuses_denied_names_outside_write_root(tmp_path, sandboxed_registry, filename) -> None: + # Files sit at tmp_path level, which is outside sandbox (the write_root). + target = tmp_path / filename + target.write_text("secret") + + tool = ReadFileTool(sandboxed_registry, OWNER_ID) + result = await tool.run({"path": str(target)}) + assert result.success is False + assert "Read denied" in result.error + + +async def test_read_file_allows_normal_files(tmp_path) -> None: + registry = FileRegistry(policy=FileAccessPolicy(write_root=tmp_path, deny_patterns=())) + target = tmp_path / "data.txt" + target.write_text("ok") + + tool = ReadFileTool(registry, OWNER_ID) + result = await tool.run({"path": str(target)}) + assert result.success is True + + +# --------------------------------------------------------------------------- +# Per-agent isolation: one agent's read does not authorize another agent's write +# --------------------------------------------------------------------------- + + +async def test_read_by_one_agent_does_not_authorize_another_to_edit(sandbox) -> None: + """Agent A reads a file; agent B tries to edit it without reading — must fail.""" + policy = FileAccessPolicy(write_root=sandbox, deny_patterns=()) + registry = FileRegistry(policy=policy) + target = sandbox / "shared.txt" + target.write_text("hello") + + reader_a = ReadFileTool(registry, "agent-a") + result = await reader_a.run({"path": str(target)}) + assert result.success is True + + editor_b = EditFileTool(registry, "agent-b") + result = await editor_b.run({"path": str(target), "old_string": "hello", "new_string": "world"}) + assert result.success is False + assert "Not authorized" in result.error + assert target.read_text() == "hello" + + +async def test_each_agent_can_edit_after_its_own_read(sandbox) -> None: + policy = FileAccessPolicy(write_root=sandbox, deny_patterns=()) + registry = FileRegistry(policy=policy) + target = sandbox / "shared.txt" + target.write_text("one") + + # Agent A reads, then edits — ok. + await ReadFileTool(registry, "agent-a").run({"path": str(target)}) + result = await EditFileTool(registry, "agent-a").run( + {"path": str(target), "old_string": "one", "new_string": "two"} + ) + assert result.success is True + + # Agent B must read first to refresh its own view, then may edit. + await ReadFileTool(registry, "agent-b").run({"path": str(target)}) + result = await EditFileTool(registry, "agent-b").run( + {"path": str(target), "old_string": "two", "new_string": "three"} + ) + assert result.success is True + assert target.read_text() == "three" + + +# --------------------------------------------------------------------------- +# GrepTool policy enforcement +# --------------------------------------------------------------------------- + + +async def test_grep_refuses_denied_root(tmp_path) -> None: + # write_root is a subdirectory; the search path at tmp_path level is outside it and denied. + write_root = tmp_path / "sandbox" + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{tmp_path}/*",)) + tool = GrepTool(policy) + with patch("ddev.ai.tools.shell.grep.run_command", new=AsyncMock()) as mock_run: + result = await tool.run({"pattern": "secret", "path": str(tmp_path / "foo")}) + assert result.success is False + assert "Read denied" in result.error + mock_run.assert_not_called() + + +async def test_grep_refuses_denied_name(tmp_path) -> None: + write_root = tmp_path / "sandbox" + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(".env",)) + tool = GrepTool(policy) + with patch("ddev.ai.tools.shell.grep.run_command", new=AsyncMock()) as mock_run: + result = await tool.run({"pattern": "SECRET", "path": str(tmp_path / ".env")}) + assert result.success is False + assert "Read denied" in result.error + mock_run.assert_not_called() + + +async def test_grep_allows_normal_path(tmp_path) -> None: + target = tmp_path / "data.txt" + target.write_text("hello world") + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + tool = GrepTool(policy) + result = await tool.run({"pattern": "hello", "path": str(target)}) + assert result.success is True + + +async def test_grep_non_recursive_returns_file_matches(tmp_path) -> None: + """Non-recursive grep on a single file returns actual matches (no post-filter applied).""" + target = tmp_path / "data.txt" + target.write_text("hello world\n") + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + tool = GrepTool(policy) + result = await tool.run({"pattern": "hello", "path": str(target), "recursive": False}) + assert result.success is True + assert "hello" in (result.data or "") + + +async def test_grep_inside_write_root_returns_denied_name_files(tmp_path) -> None: + """Recursive grep inside write_root returns .env and other denied-name files.""" + sandbox = tmp_path / "sandbox" + sandbox.mkdir() + (sandbox / ".env").write_text("SECRET=hello\n") + policy = FileAccessPolicy(write_root=sandbox, deny_patterns=(".env",)) + tool = GrepTool(policy) + result = await tool.run({"pattern": "hello", "path": str(sandbox), "recursive": True}) + assert result.success is True + assert ".env" in (result.data or "") + + +async def test_grep_post_filter_strips_denied_path_pattern_matches(tmp_path) -> None: + """Denied path-pattern files are stripped from grep output even when grep walks them.""" + write_root = tmp_path / "sandbox" + project = tmp_path / "project" + secrets = tmp_path / "secrets" + project.mkdir() + secrets.mkdir() + (project / "ok.txt").write_text("hello world\n") + (secrets / "leak.txt").write_text("hello world\n") + + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{secrets}/*",)) + tool = GrepTool(policy) + result = await tool.run({"pattern": "hello", "path": str(tmp_path), "recursive": True}) + assert result.success is True + assert "ok.txt" in result.data + assert "leak.txt: Read denied by policy" in result.data + + +async def test_grep_post_filter_strips_symlink_to_denied(tmp_path) -> None: + """A symlink in the search root resolving into a denied tree is filtered out.""" + write_root = tmp_path / "sandbox" + project = tmp_path / "project" + secrets = tmp_path / "secrets" + project.mkdir() + secrets.mkdir() + (secrets / "key.txt").write_text("hello world\n") + (project / "link.txt").symlink_to(secrets / "key.txt") + + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{secrets}/*",)) + tool = GrepTool(policy) + result = await tool.run({"pattern": "hello", "path": str(project), "recursive": True}) + assert result.success is True + # link.txt may appear as a denial notice but must not appear as a match line. + assert "link.txt" not in result.data + assert result.data + + +async def test_grep_excludes_basename_pattern_matches(tmp_path) -> None: + """Basename patterns ride on grep's --exclude flag; verify denied files are absent.""" + write_root = tmp_path / "sandbox" + project = tmp_path / "project" + project.mkdir() + (project / "config.py").write_text("token=abc\n") + (project / ".env").write_text("token=abc\n") + + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(".env",)) + tool = GrepTool(policy) + result = await tool.run({"pattern": "token", "path": str(project), "recursive": True}) + assert result.success is True + assert "config.py" in result.data + assert ".env" not in result.data + + +# --------------------------------------------------------------------------- +# Tilde-path canonicalization for write tools +# --------------------------------------------------------------------------- + + +async def test_create_file_with_tilde_path_writes_to_home_when_authorized(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) # Windows uses USERPROFILE, not HOME + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + registry = FileRegistry(policy=policy) + tool = CreateFileTool(registry, OWNER_ID) + + result = await tool.run({"path": "~/x.txt", "content": "hello"}) + + assert result.success is True + assert (tmp_path / "x.txt").read_text() == "hello" + + +async def test_create_file_with_tilde_path_refused_when_outside_write_root(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + policy = FileAccessPolicy(write_root=tmp_path / "sub", deny_patterns=()) + registry = FileRegistry(policy=policy) + tool = CreateFileTool(registry, OWNER_ID) + + result = await tool.run({"path": "~/x.txt", "content": "hello"}) + + assert result.success is False + assert "outside write root" in result.error + assert not (tmp_path / "x.txt").exists() diff --git a/ddev/tests/ai/tools/fs/test_read_file.py b/ddev/tests/ai/tools/fs/test_read_file.py index f2497e6c09a18..2d37eba14c987 100644 --- a/ddev/tests/ai/tools/fs/test_read_file.py +++ b/ddev/tests/ai/tools/fs/test_read_file.py @@ -8,9 +8,11 @@ from ddev.ai.tools.fs.file_registry import FileRegistry from ddev.ai.tools.fs.read_file import ReadFileTool +from .conftest import OWNER_ID + def test_tool_name(registry: FileRegistry) -> None: - assert ReadFileTool(registry).name == "read_file" + assert ReadFileTool(registry, OWNER_ID).name == "read_file" async def test_read_file_success(read_tool: ReadFileTool, tmp_path) -> None: @@ -28,7 +30,7 @@ async def test_read_registers_unknown_file(read_tool: ReadFileTool, registry: Fi f.write_text("content", encoding="utf-8") await read_tool.run({"path": str(f)}) - assert registry.is_known(str(f)) is True + assert registry.is_known(OWNER_ID, str(f)) is True async def test_read_file_missing_file(read_tool: ReadFileTool, tmp_path) -> None: diff --git a/ddev/tests/ai/tools/fs/test_workflow.py b/ddev/tests/ai/tools/fs/test_workflow.py index a45ad9d937e26..aef70a306747d 100644 --- a/ddev/tests/ai/tools/fs/test_workflow.py +++ b/ddev/tests/ai/tools/fs/test_workflow.py @@ -37,8 +37,10 @@ async def test_workflow_create_read_edit_append( assert r.success is True assert f.read_text(encoding="utf-8").endswith("# updated\n") - # Registry must reflect the final state - assert registry.verify(str(f), f.read_text(encoding="utf-8")) is True + # Registry must reflect the final state for this agent + from .conftest import OWNER_ID + + assert registry.verify(OWNER_ID, str(f), f.read_text(encoding="utf-8")) is True async def test_workflow_stale_file( diff --git a/ddev/tests/ai/tools/shell/test_tools.py b/ddev/tests/ai/tools/shell/test_tools.py index 05084acc97e9e..f58ab68fe1a81 100644 --- a/ddev/tests/ai/tools/shell/test_tools.py +++ b/ddev/tests/ai/tools/shell/test_tools.py @@ -5,26 +5,25 @@ import pytest +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy from ddev.ai.tools.shell.grep import GrepInput, GrepTool from ddev.ai.tools.shell.list_files import ListFilesInput, ListFilesTool -from ddev.ai.tools.shell.mkdir import MkdirInput, MkdirTool # --------------------------------------------------------------------------- # Tool metadata # --------------------------------------------------------------------------- -@pytest.mark.parametrize( - "tool_cls,expected_name,expected_timeout", - [ - (GrepTool, "grep", 30), - (ListFilesTool, "list_files", 30), - (MkdirTool, "mkdir", 5), - ], -) -def test_tool_meta(tool_cls, expected_name, expected_timeout): - assert tool_cls().name == expected_name - assert tool_cls.timeout == expected_timeout +def test_grep_tool_meta(tmp_path) -> None: + tool = GrepTool(FileAccessPolicy(write_root=tmp_path)) + assert tool.name == "grep" + assert GrepTool.timeout == 30 + + +def test_list_files_tool_meta() -> None: + tool = ListFilesTool() + assert tool.name == "list_files" + assert ListFilesTool.timeout == 30 # --------------------------------------------------------------------------- @@ -33,15 +32,19 @@ def test_tool_meta(tool_cls, expected_name, expected_timeout): @pytest.fixture -def grep_tool() -> GrepTool: - return GrepTool() +def grep_tool(tmp_path) -> GrepTool: + return GrepTool(FileAccessPolicy(write_root=tmp_path, deny_patterns=())) def test_grep_cmd_full_command(grep_tool: GrepTool): + # deny_patterns=() so no --exclude= flags; paths outside write_root still produce no flags. assert grep_tool.cmd(GrepInput(pattern="ERROR", path="/var/log", recursive=True)) == [ "grep", "-n", "-E", + "--null", + "-I", + "--no-messages", "-r", "--", "ERROR", @@ -51,6 +54,9 @@ def test_grep_cmd_full_command(grep_tool: GrepTool): "grep", "-n", "-E", + "--null", + "-I", + "--no-messages", "--", "ERROR", "/var/log", @@ -65,6 +71,121 @@ def test_grep_cmd_pattern_and_path_placement(grep_tool: GrepTool): assert cmd[-1] == "/my dir/sub dir" +def test_grep_cmd_recursive_outside_write_root_adds_basename_excludes(tmp_path) -> None: + """--exclude= flags are added only for basename patterns when search is outside write_root.""" + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=(".env", "*.pem", f"{tmp_path}/secrets/*")) + tool = GrepTool(policy) + # /project is outside tmp_path (write_root) + cmd = tool.cmd(GrepInput(pattern="SECRET", path="/project", recursive=True)) + flags_before_sep = cmd[: cmd.index("--")] + assert "--exclude=.env" in flags_before_sep + assert "--exclude=*.pem" in flags_before_sep + # Path patterns must NOT become flags — they ride on the post-filter. + assert not any(f.startswith("--exclude-dir") for f in flags_before_sep) + assert not any("secrets" in f for f in flags_before_sep) + assert cmd[-2] == "SECRET" + assert cmd[-1] == "/project" + + +def test_grep_cmd_recursive_inside_write_root_no_excludes(tmp_path) -> None: + """No --exclude= flags when search path is inside write_root.""" + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=(".env", "*.pem")) + tool = GrepTool(policy) + # Search inside write_root — deny patterns are bypassed, so no excludes. + cmd = tool.cmd(GrepInput(pattern="SECRET", path=str(tmp_path / "project"), recursive=True)) + assert not any(arg.startswith("--exclude") for arg in cmd) + + +def test_grep_cmd_recursive_spanning_write_root_no_excludes(tmp_path) -> None: + """No --exclude= flags when write_root is inside the search path (mixed zone).""" + write_root = tmp_path / "sandbox" + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(".env", "*.pem")) + tool = GrepTool(policy) + # Search starts at tmp_path which is a parent of write_root — spanning case. + cmd = tool.cmd(GrepInput(pattern="SECRET", path=str(tmp_path), recursive=True)) + assert not any(arg.startswith("--exclude") for arg in cmd) + + +def test_grep_cmd_non_recursive_no_exclude_flags(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=(".env", "*.pem", f"{tmp_path}/secrets/*")) + tool = GrepTool(policy) + cmd = tool.cmd(GrepInput(pattern="SECRET", path="/project/file.txt", recursive=False)) + assert not any(arg.startswith("--exclude") for arg in cmd) + + +# --------------------------------------------------------------------------- +# GrepTool post-filter — unit tests on the parsing/decision logic +# --------------------------------------------------------------------------- + + +def test_filter_stdout_keeps_allowed_lines(tmp_path) -> None: + f = tmp_path / "ok.txt" + f.write_text("x") + tool = GrepTool(FileAccessPolicy(write_root=tmp_path, deny_patterns=())) + raw = f"{f}\x0042:hello\n{f}\x0043:world\n" + out = tool._filter_stdout(raw) + assert out == f"{f}:42:hello\n{f}:43:world" + + +def test_filter_stdout_filters_denied_path_lines(tmp_path) -> None: + write_root = tmp_path / "sandbox" + secrets = tmp_path / "secrets" + secrets.mkdir() + leak = secrets / "leak.txt" + leak.write_text("x") + public = tmp_path / "ok.txt" + public.write_text("x") + + # write_root is a subdirectory; secrets/ and ok.txt are outside it, so deny patterns apply. + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{secrets}/*",)) + tool = GrepTool(policy) + raw = f"{leak}\x001:hit\n{public}\x002:hit\n" + out = tool._filter_stdout(raw) + assert f"{leak}: Read denied by policy" in out + assert f"{public}:2:hit" in out + + +def test_filter_stdout_drops_lines_without_nul(tmp_path) -> None: + """Defensive: stderr noise / malformed output is dropped, not passed through.""" + tool = GrepTool(FileAccessPolicy(write_root=tmp_path, deny_patterns=())) + assert tool._filter_stdout("grep: something: Permission denied\n") == "" + + +def test_filter_stdout_caches_per_filename(tmp_path, monkeypatch) -> None: + f = tmp_path / "ok.txt" + f.write_text("x") + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + calls = {"n": 0} + real = policy.assert_readable + + def counting(p): + calls["n"] += 1 + return real(p) + + monkeypatch.setattr(policy, "assert_readable", counting) + tool = GrepTool(policy) + raw = "".join(f"{f}\x00{i}:line\n" for i in range(10)) + tool._filter_stdout(raw) + assert calls["n"] == 1 + + +def test_filter_stdout_resolves_symlink_to_denied(tmp_path) -> None: + write_root = tmp_path / "sandbox" + secrets = tmp_path / "secrets" + secrets.mkdir() + target = secrets / "real.txt" + target.write_text("x") + link = tmp_path / "link.txt" + link.symlink_to(target) + + # secrets/ is outside write_root, so its deny pattern applies. + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{secrets}/*",)) + tool = GrepTool(policy) + raw = f"{link}\x001:hit\n" + out = tool._filter_stdout(raw) + assert out == f"{link}: Read denied by policy" + + async def test_grep_no_matches_returns_success(grep_tool: GrepTool): from ddev.ai.tools.core.types import ToolResult @@ -97,18 +218,3 @@ def test_list_files_cmd_non_recursive(list_files_tool: ListFilesTool): def test_list_files_cmd_recursive(list_files_tool: ListFilesTool): cmd = list_files_tool.cmd(ListFilesInput(path="/var", recursive=True)) assert cmd == ["find", "/var", "-mindepth", "1"] - - -# --------------------------------------------------------------------------- -# MkdirTool -# --------------------------------------------------------------------------- - - -@pytest.fixture -def mkdir_tool() -> MkdirTool: - return MkdirTool() - - -def test_mkdir_cmd(mkdir_tool: MkdirTool): - assert mkdir_tool.cmd(MkdirInput(path="/a/b/c")) == ["mkdir", "-p", "/a/b/c"] - assert mkdir_tool.cmd(MkdirInput(path="/my dir/sub dir")) == ["mkdir", "-p", "/my dir/sub dir"] diff --git a/ddev/tests/ai/tools/core/test_registry.py b/ddev/tests/ai/tools/test_registry.py similarity index 70% rename from ddev/tests/ai/tools/core/test_registry.py rename to ddev/tests/ai/tools/test_registry.py index b770867b909f4..e307773f5b7b6 100644 --- a/ddev/tests/ai/tools/core/test_registry.py +++ b/ddev/tests/ai/tools/test_registry.py @@ -4,8 +4,10 @@ import pytest -from ddev.ai.tools.core.registry import ToolRegistry from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.registry import ToolRegistry # --------------------------------------------------------------------------- # Fake tools — implement ToolProtocol without depending on BaseTool @@ -150,36 +152,63 @@ def test_available_tool_names_returns_fresh_copy(): # --------------------------------------------------------------------------- -def test_from_names_empty(): - registry = ToolRegistry.from_names([]) +OWNER_ID = "test-agent" + + +def test_from_names_empty(tmp_path): + registry = ToolRegistry.from_names( + [], owner_id=OWNER_ID, file_registry=FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) + ) assert registry.definitions == [] -def test_from_names_unknown_raises(): +def test_from_names_unknown_raises(tmp_path): with pytest.raises(ValueError, match="Unknown tool name: 'teleport'"): - ToolRegistry.from_names(["teleport"]) + ToolRegistry.from_names( + ["teleport"], owner_id=OWNER_ID, file_registry=FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) + ) @pytest.mark.parametrize("name", ToolRegistry.available_tool_names()) -def test_from_names_each_known_tool(name): - registry = ToolRegistry.from_names([name]) +def test_from_names_each_known_tool(name, tmp_path): + registry = ToolRegistry.from_names( + [name], owner_id=OWNER_ID, file_registry=FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) + ) assert len(registry.definitions) == 1 assert registry.definitions[0]["name"] == name -def test_from_names_all_at_once(): +def test_from_names_all_at_once(tmp_path): all_names = ToolRegistry.available_tool_names() - registry = ToolRegistry.from_names(all_names) + registry = ToolRegistry.from_names( + all_names, owner_id=OWNER_ID, file_registry=FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) + ) built_names = {d["name"] for d in registry.definitions} assert built_names == set(all_names) -def test_from_names_fs_tools_share_file_registry(): - """All file-system tools in the same registry share a single FileRegistry.""" - fs_names = [n for n in ToolRegistry.available_tool_names() if n.endswith("_file")] - if len(fs_names) < 2: +def test_from_names_fs_tools_share_file_registry(tmp_path): + """All tools that use the file registry in the same ToolRegistry share a single instance.""" + all_names = ToolRegistry.available_tool_names() + registry = ToolRegistry.from_names( + all_names, owner_id=OWNER_ID, file_registry=FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) + ) + fs_tools = [t for t in registry._tools.values() if hasattr(t, "_registry")] + if len(fs_tools) < 2: pytest.skip("Need at least 2 fs tools to test shared registry") - registry = ToolRegistry.from_names(fs_names) - tools = list(registry._tools.values()) - registries = [t._registry for t in tools] + registries = [t._registry for t in fs_tools] assert all(r is registries[0] for r in registries) + + +def test_from_names_reuses_supplied_file_registry(tmp_path): + """Multiple ToolRegistries can share one FileRegistry; tools carry their own owner_id.""" + shared = FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) + reg_a = ToolRegistry.from_names(["read_file", "create_file"], owner_id="a", file_registry=shared) + reg_b = ToolRegistry.from_names(["read_file", "create_file"], owner_id="b", file_registry=shared) + + for tool in reg_a._tools.values(): + assert tool._registry is shared + assert tool._owner_id == "a" + for tool in reg_b._tools.values(): + assert tool._registry is shared + assert tool._owner_id == "b" From 26dda0dd873aabcfdb394934ca9bbf86dfcd10c2 Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Mon, 4 May 2026 15:48:55 +0200 Subject: [PATCH 31/44] Fix Phase.on_error signature (#23574) * Fix Phase.on_error signature and test_orchestrator * Tighten on_error's error type * Log instead of raise in on_finalize --- ddev/src/ddev/ai/phases/base.py | 11 ++++---- ddev/src/ddev/ai/phases/orchestrator.py | 8 +++--- ddev/tests/ai/phases/test_base.py | 17 ++++++++---- ddev/tests/ai/phases/test_orchestrator.py | 33 ++++++++++++++++++++--- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/ddev/src/ddev/ai/phases/base.py b/ddev/src/ddev/ai/phases/base.py index 83feaeea12ee8..37d0519649c69 100644 --- a/ddev/src/ddev/ai/phases/base.py +++ b/ddev/src/ddev/ai/phases/base.py @@ -19,6 +19,7 @@ from ddev.ai.react.process import ReActProcess from ddev.ai.tools.fs.file_registry import FileRegistry from ddev.ai.tools.registry import ToolRegistry +from ddev.event_bus.exceptions import MessageProcessingError, ProcessorHookError from ddev.event_bus.orchestrator import AsyncProcessor, BaseMessage @@ -255,12 +256,12 @@ async def on_success(self, message: PhaseTrigger) -> None: """Emit PhaseTrigger to unblock dependent phases.""" self.submit_message( PhaseTrigger( - id=f"{self._phase_id}_finished_{message.id}", + id=f"{self._phase_id}_finished", phase_id=self._phase_id, ) ) - async def on_error(self, message: PhaseTrigger, error: Exception) -> None: + async def on_error(self, error: MessageProcessingError | ProcessorHookError) -> None: """Write failed checkpoint and emit PhaseFailedMessage.""" try: self._checkpoint_manager.write_phase_checkpoint( @@ -269,7 +270,7 @@ async def on_error(self, message: PhaseTrigger, error: Exception) -> None: "status": "failed", "started_at": self._started_at.isoformat() if self._started_at else None, "finished_at": datetime.now(UTC).isoformat(), - "error": str(error), + "error": str(error.original_exception), }, ) except Exception: @@ -277,8 +278,8 @@ async def on_error(self, message: PhaseTrigger, error: Exception) -> None: finally: self.submit_message( PhaseFailedMessage( - id=f"{self._phase_id}_failed_{message.id}", + id=f"{self._phase_id}_failed", phase_id=self._phase_id, - error=str(error), + error=str(error.original_exception), ) ) diff --git a/ddev/src/ddev/ai/phases/orchestrator.py b/ddev/src/ddev/ai/phases/orchestrator.py index bb6a892fcbe4d..1abc6834ef836 100644 --- a/ddev/src/ddev/ai/phases/orchestrator.py +++ b/ddev/src/ddev/ai/phases/orchestrator.py @@ -124,7 +124,9 @@ async def on_message_received(self, message: BaseMessage) -> None: raise FatalProcessingError(f"Phase '{message.phase_id}' failed: {message.error}") async def on_finalize(self, exception: Exception | None) -> None: - if self._failed_phase is not None: - raise RuntimeError( - f"Pipeline aborted: phase '{self._failed_phase}' failed: {self._failed_error or ''}" + if exception is not None and self._failed_phase is not None: + self._logger.error( + "Pipeline aborted: phase '%s' failed: %s", + self._failed_phase, + self._failed_error or "", ) diff --git a/ddev/tests/ai/phases/test_base.py b/ddev/tests/ai/phases/test_base.py index 4481e21c2e725..73e536ea3ff10 100644 --- a/ddev/tests/ai/phases/test_base.py +++ b/ddev/tests/ai/phases/test_base.py @@ -15,6 +15,7 @@ from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy from ddev.ai.tools.fs.file_registry import FileRegistry from ddev.ai.tools.registry import ToolRegistry +from ddev.event_bus.exceptions import HookName, MessageProcessingError, ProcessorHookError from .conftest import MockAgent, make_agent_factory, make_response, resolve_key @@ -445,7 +446,7 @@ async def test_on_success_emits_finished_message(flow_dir, monkeypatch, message_ msg = message_queue.get_nowait() assert isinstance(msg, PhaseTrigger) assert msg.phase_id == "p1" - assert msg.id == "p1_finished_start" + assert msg.id == "p1_finished" # --------------------------------------------------------------------------- @@ -457,7 +458,8 @@ async def test_on_error_writes_failed_checkpoint(flow_dir, monkeypatch, message_ mock_agent = MockAgent([]) phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) - await phase.on_error(PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) + wrapped = MessageProcessingError("p1", PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) + await phase.on_error(wrapped) checkpoint = mgr.read()["p1"] assert checkpoint["status"] == "failed" @@ -469,7 +471,10 @@ async def test_on_error_emits_failed_message(flow_dir, monkeypatch, message_queu mock_agent = MockAgent([]) phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) - await phase.on_error(PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) + wrapped = ProcessorHookError( + HookName.ON_SUCCESS, "p1", PhaseTrigger(id="start", phase_id=None), RuntimeError("boom") + ) + await phase.on_error(wrapped) msg = message_queue.get_nowait() assert isinstance(msg, PhaseFailedMessage) @@ -482,7 +487,8 @@ async def test_on_error_writes_failed_checkpoint_after_start(flow_dir, monkeypat phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) phase._started_at = datetime.now(UTC) - await phase.on_error(PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) + wrapped = MessageProcessingError("p1", PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) + await phase.on_error(wrapped) checkpoint = mgr.read()["p1"] assert checkpoint["status"] == "failed" @@ -663,7 +669,8 @@ async def test_failed_phase_omits_memory_path(flow_dir, monkeypatch, message_que mock_agent = MockAgent([]) phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) - await phase.on_error(PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) + wrapped = MessageProcessingError("p1", PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) + await phase.on_error(wrapped) checkpoint = mgr.read()["p1"] assert "memory_path" not in checkpoint diff --git a/ddev/tests/ai/phases/test_orchestrator.py b/ddev/tests/ai/phases/test_orchestrator.py index c83b6fff23407..b36700a7cd121 100644 --- a/ddev/tests/ai/phases/test_orchestrator.py +++ b/ddev/tests/ai/phases/test_orchestrator.py @@ -454,7 +454,9 @@ async def test_on_finalize_no_failure_is_noop(tmp_path, file_access_policy): await orchestrator.on_finalize(None) # must not raise -async def test_on_finalize_after_phase_failed_raises(tmp_path, file_access_policy): +async def test_on_finalize_after_phase_failed_logs(tmp_path, file_access_policy, caplog): + import logging + orchestrator = PhaseOrchestrator( flow_yaml_path=Path("/fake/flow.yaml"), checkpoint_path=Path("/fake/checkpoints.yaml"), @@ -463,11 +465,34 @@ async def test_on_finalize_after_phase_failed_raises(tmp_path, file_access_polic file_access_policy=file_access_policy, ) msg = PhaseFailedMessage(id="f1", phase_id="p1", error="boom") + exc = FatalProcessingError("Phase 'p1' failed: boom") with pytest.raises(FatalProcessingError): await orchestrator.on_message_received(msg) - with pytest.raises(RuntimeError, match="Pipeline aborted.*p1.*boom"): - await orchestrator.on_finalize(None) + with caplog.at_level(logging.ERROR): + await orchestrator.on_finalize(exc) # must not raise + + assert any("Pipeline aborted" in r.message and "p1" in r.message and "boom" in r.message for r in caplog.records) + + +async def test_on_finalize_no_exception_no_log(tmp_path, file_access_policy, caplog): + import logging + + orchestrator = PhaseOrchestrator( + flow_yaml_path=Path("/fake/flow.yaml"), + checkpoint_path=Path("/fake/checkpoints.yaml"), + runtime_variables={}, + anthropic_client=MagicMock(), + file_access_policy=file_access_policy, + ) + msg = PhaseFailedMessage(id="f1", phase_id="p1", error="boom") + with pytest.raises(FatalProcessingError): + await orchestrator.on_message_received(msg) + + with caplog.at_level(logging.ERROR): + await orchestrator.on_finalize(None) # exception=None means clean exit — no log + + assert not any("Pipeline aborted" in r.message for r in caplog.records) def test_run_raises_runtime_error_when_phase_fails(tmp_path, file_access_policy): @@ -505,5 +530,5 @@ async def process_message(self, message: PhaseTrigger) -> None: ) orchestrator._phase_registry.register("FailingPhase", FailingPhase) - with pytest.raises(RuntimeError, match="Pipeline aborted"): + with pytest.raises(FatalProcessingError, match="Phase 'failing' failed"): orchestrator.run() From 9cfe4048fe36dd9e06075c14abe3bd07c4ee63a4 Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Tue, 5 May 2026 11:49:08 +0200 Subject: [PATCH 32/44] Move callbacks out of react/ into a dedicated callbacks/ package (#23577) * Fix Phase.on_error signature and test_orchestrator * Move callbacks from react/ to its own layer * Tighten on_error's error type * Update Callbacks docstring * Fix typo * Improve CallbackSet docsting --- ddev/src/ddev/ai/callbacks/__init__.py | 3 + .../ddev/ai/{react => callbacks}/callbacks.py | 166 +++++++++++------- ddev/src/ddev/ai/phases/base.py | 18 +- ddev/src/ddev/ai/phases/orchestrator.py | 8 +- ddev/src/ddev/ai/react/process.py | 35 ++-- ddev/tests/ai/callbacks/__init__.py | 3 + .../ai/{react => callbacks}/test_callbacks.py | 112 +++++++++++- ddev/tests/ai/phases/test_base.py | 2 +- ddev/tests/ai/react/test_process.py | 24 +-- 9 files changed, 260 insertions(+), 111 deletions(-) create mode 100644 ddev/src/ddev/ai/callbacks/__init__.py rename ddev/src/ddev/ai/{react => callbacks}/callbacks.py (52%) create mode 100644 ddev/tests/ai/callbacks/__init__.py rename ddev/tests/ai/{react => callbacks}/test_callbacks.py (75%) diff --git a/ddev/src/ddev/ai/callbacks/__init__.py b/ddev/src/ddev/ai/callbacks/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/callbacks/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/react/callbacks.py b/ddev/src/ddev/ai/callbacks/callbacks.py similarity index 52% rename from ddev/src/ddev/ai/react/callbacks.py rename to ddev/src/ddev/ai/callbacks/callbacks.py index 2127366b76ded..d0be2229a27c7 100644 --- a/ddev/src/ddev/ai/react/callbacks.py +++ b/ddev/src/ddev/ai/callbacks/callbacks.py @@ -2,12 +2,16 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from typing import Protocol +from typing import Any, Protocol from ddev.ai.agent.types import AgentResponse, ToolCall from ddev.ai.react.types import ReActResult from ddev.ai.tools.core.types import ToolResult +# --------------------------------------------------------------------------- +# ReAct-layer protocols +# --------------------------------------------------------------------------- + class OnAgentResponseCallback(Protocol): """Called after every agent.send() returns, including the first.""" @@ -22,13 +26,13 @@ async def __call__(self, tool_call: ToolCall, result: ToolResult, iteration: int class OnCompleteCallback(Protocol): - """Called when the loop exits cleanly.""" + """Called when the ReAct loop exits cleanly.""" async def __call__(self, result: ReActResult) -> None: ... class OnErrorCallback(Protocol): - """Called when the loop aborts. The exception is always re-raised after this returns.""" + """Called when the ReAct loop aborts. The exception is always re-raised after this returns.""" async def __call__(self, error: BaseException) -> None: ... @@ -45,28 +49,48 @@ class AfterCompactCallback(Protocol): async def __call__(self) -> None: ... +class OnBeforeAgentSendCallback(Protocol): + """Called immediately before each agent.send() request is issued.""" + + async def __call__(self, iteration: int) -> None: ... + + +# --------------------------------------------------------------------------- +# Phase-layer protocols +# --------------------------------------------------------------------------- + + class OnPhaseStartCallback(Protocol): """Called once when a phase begins executing, before any agent interaction.""" async def __call__(self, phase_id: str) -> None: ... -class OnBeforeAgentSendCallback(Protocol): - """Called immediately before each agent.send() request is issued.""" +class OnPhaseFinishCallback(Protocol): + """Called once when a phase completes successfully.""" - async def __call__(self, iteration: int) -> None: ... + async def __call__(self, phase_id: str) -> None: ... + + +# --------------------------------------------------------------------------- +# CallbackSet and Callbacks +# --------------------------------------------------------------------------- class CallbackSet: - """Decorator-based registry for ReAct lifecycle event handlers. + """Decorator-based registry for framework lifecycle event handlers. - Usage:: + Group related handlers in a single instance for semantic cohesion, then + compose multiple instances via Callbacks(): - cb = CallbackSet() + Usage:: + logger = CallbackSet() - @cb.on_complete + @logger.on_complete async def log_done(result: ReActResult) -> None: print(f"Done in {result.iterations} iterations") + + callbacks = Callbacks([logger]) """ def __init__(self) -> None: @@ -76,101 +100,119 @@ def __init__(self) -> None: self._on_error: list[OnErrorCallback] = [] self._before_compact: list[BeforeCompactCallback] = [] self._after_compact: list[AfterCompactCallback] = [] - self._on_phase_start: list[OnPhaseStartCallback] = [] self._on_before_agent_send: list[OnBeforeAgentSendCallback] = [] + self._on_phase_start: list[OnPhaseStartCallback] = [] + self._on_phase_finish: list[OnPhaseFinishCallback] = [] + + async def _fire(self, handlers: list[Any], *args: Any) -> None: + for handler in handlers: + try: + await handler(*args) + except Exception: + pass def on_agent_response(self, func: OnAgentResponseCallback) -> OnAgentResponseCallback: - """Register a handler fired after every agent response.""" self._on_agent_response.append(func) return func + async def fire_agent_response(self, response: AgentResponse, iteration: int) -> None: + await self._fire(self._on_agent_response, response, iteration) + def on_tool_call(self, func: OnToolCallCallback) -> OnToolCallCallback: - """Register a handler fired after each tool in a batch executes.""" self._on_tool_call.append(func) return func + async def fire_tool_call(self, tool_call: ToolCall, result: ToolResult, iteration: int) -> None: + await self._fire(self._on_tool_call, tool_call, result, iteration) + def on_complete(self, func: OnCompleteCallback) -> OnCompleteCallback: - """Register a handler fired when the loop exits cleanly.""" self._on_complete.append(func) return func + async def fire_complete(self, result: ReActResult) -> None: + await self._fire(self._on_complete, result) + def on_error(self, func: OnErrorCallback) -> OnErrorCallback: - """Register a handler fired when the loop aborts.""" self._on_error.append(func) return func + async def fire_error(self, error: BaseException) -> None: + await self._fire(self._on_error, error) + def on_before_compact(self, func: BeforeCompactCallback) -> BeforeCompactCallback: - """Register a handler fired just before compaction runs.""" self._before_compact.append(func) return func + async def fire_before_compact(self) -> None: + await self._fire(self._before_compact) + def on_after_compact(self, func: AfterCompactCallback) -> AfterCompactCallback: - """Register a handler fired just after compaction completes.""" self._after_compact.append(func) return func + async def fire_after_compact(self) -> None: + await self._fire(self._after_compact) + + def on_before_agent_send(self, func: OnBeforeAgentSendCallback) -> OnBeforeAgentSendCallback: + self._on_before_agent_send.append(func) + return func + + async def fire_before_agent_send(self, iteration: int) -> None: + await self._fire(self._on_before_agent_send, iteration) + def on_phase_start(self, func: OnPhaseStartCallback) -> OnPhaseStartCallback: - """Register a handler fired at the start of a phase.""" self._on_phase_start.append(func) return func - def on_before_agent_send(self, func: OnBeforeAgentSendCallback) -> OnBeforeAgentSendCallback: - """Register a handler fired right before each agent.send() request.""" - self._on_before_agent_send.append(func) + async def fire_phase_start(self, phase_id: str) -> None: + await self._fire(self._on_phase_start, phase_id) + + def on_phase_finish(self, func: OnPhaseFinishCallback) -> OnPhaseFinishCallback: + self._on_phase_finish.append(func) return func + async def fire_phase_finish(self, phase_id: str) -> None: + await self._fire(self._on_phase_finish, phase_id) + + +class Callbacks: + """Container of CallbackSet instances. Dispatches each fire_* to all contained sets.""" + + def __init__(self, sets: list[CallbackSet] | None = None) -> None: + self._sets: list[CallbackSet] = sets or [] + async def fire_agent_response(self, response: AgentResponse, iteration: int) -> None: - for handler in self._on_agent_response: - try: - await handler(response, iteration) - except Exception: - pass # we will see in the future what to do with this + for s in self._sets: + await s.fire_agent_response(response, iteration) async def fire_tool_call(self, tool_call: ToolCall, result: ToolResult, iteration: int) -> None: - for handler in self._on_tool_call: - try: - await handler(tool_call, result, iteration) - except Exception: - pass + for s in self._sets: + await s.fire_tool_call(tool_call, result, iteration) async def fire_complete(self, result: ReActResult) -> None: - for handler in self._on_complete: - try: - await handler(result) - except Exception: - pass + for s in self._sets: + await s.fire_complete(result) async def fire_error(self, error: BaseException) -> None: - for handler in self._on_error: - try: - await handler(error) - except Exception: - pass + for s in self._sets: + await s.fire_error(error) async def fire_before_compact(self) -> None: - for handler in self._before_compact: - try: - await handler() - except Exception: - pass + for s in self._sets: + await s.fire_before_compact() async def fire_after_compact(self) -> None: - for handler in self._after_compact: - try: - await handler() - except Exception: - pass + for s in self._sets: + await s.fire_after_compact() + + async def fire_before_agent_send(self, iteration: int) -> None: + for s in self._sets: + await s.fire_before_agent_send(iteration) async def fire_phase_start(self, phase_id: str) -> None: - for handler in self._on_phase_start: - try: - await handler(phase_id) - except Exception: - pass + for s in self._sets: + await s.fire_phase_start(phase_id) - async def fire_before_agent_send(self, iteration: int) -> None: - for handler in self._on_before_agent_send: - try: - await handler(iteration) - except Exception: - pass + async def fire_phase_finish(self, phase_id: str) -> None: + for s in self._sets: + await s.fire_phase_finish(phase_id) diff --git a/ddev/src/ddev/ai/phases/base.py b/ddev/src/ddev/ai/phases/base.py index 37d0519649c69..c0f1c9fe226c9 100644 --- a/ddev/src/ddev/ai/phases/base.py +++ b/ddev/src/ddev/ai/phases/base.py @@ -11,11 +11,11 @@ import anthropic from ddev.ai.agent.anthropic_client import AnthropicAgent +from ddev.ai.callbacks.callbacks import Callbacks from ddev.ai.phases.checkpoint import CheckpointManager from ddev.ai.phases.config import AgentConfig, CheckpointConfig, FlowConfigError, PhaseConfig, TaskConfig from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger from ddev.ai.phases.template import render_inline, render_prompt -from ddev.ai.react.callbacks import CallbackSet from ddev.ai.react.process import ReActProcess from ddev.ai.tools.fs.file_registry import FileRegistry from ddev.ai.tools.registry import ToolRegistry @@ -93,7 +93,7 @@ def __init__( flow_variables: dict[str, str], config_dir: Path, file_registry: FileRegistry, - callback_sets: list[CallbackSet] | None = None, + callbacks: Callbacks | None = None, logger: logging.Logger | None = None, ) -> None: super().__init__(name=phase_id) @@ -107,7 +107,7 @@ def __init__( self._runtime_variables = runtime_variables self._flow_variables = flow_variables self._config_dir = config_dir - self._callback_sets: list[CallbackSet] = callback_sets or [] + self._callbacks: Callbacks = callbacks or Callbacks() self._file_registry = file_registry self._logger = logger or logging.getLogger(__name__) self._started_at: datetime | None = None @@ -166,8 +166,7 @@ async def process_message(self, message: PhaseTrigger) -> None: """Full phase pipeline. Not intended to be overridden -- customise via the extension points.""" # 1. Record start time and notify observers self._started_at = datetime.now(UTC) - for cb_set in self._callback_sets: - await cb_set.fire_phase_start(self._phase_id) + await self._callbacks.fire_phase_start(self._phase_id) # 2. Build template context and memory resolver context: dict[str, Any] = { @@ -211,7 +210,7 @@ async def process_message(self, message: PhaseTrigger) -> None: process = ReActProcess( agent=agent, tool_registry=tool_registry, - callback_sets=self._callback_sets, + callbacks=self._callbacks, ) # 6. Call run_tasks() @@ -227,15 +226,13 @@ async def process_message(self, message: PhaseTrigger) -> None: memory_prompt = self._checkpoint_manager.build_memory_prompt(user_additions) # 9. Call the agent for the summary — text-only (allowed_tools=[]) - for cb_set in self._callback_sets: - await cb_set.fire_before_agent_send(1) + await self._callbacks.fire_before_agent_send(1) response = await agent.send(memory_prompt, allowed_tools=[]) total_input += response.usage.input_tokens total_output += response.usage.output_tokens - for cb_set in self._callback_sets: - await cb_set.fire_agent_response(response, 1) + await self._callbacks.fire_agent_response(response, 1) # 10. Persist the memory file self._checkpoint_manager.write_memory(self._phase_id, response.text) @@ -251,6 +248,7 @@ async def process_message(self, message: PhaseTrigger) -> None: "memory_path": str(self._checkpoint_manager.memory_path(self._phase_id)), }, ) + await self._callbacks.fire_phase_finish(self._phase_id) async def on_success(self, message: PhaseTrigger) -> None: """Emit PhaseTrigger to unblock dependent phases.""" diff --git a/ddev/src/ddev/ai/phases/orchestrator.py b/ddev/src/ddev/ai/phases/orchestrator.py index 1abc6834ef836..cfdc8851f3062 100644 --- a/ddev/src/ddev/ai/phases/orchestrator.py +++ b/ddev/src/ddev/ai/phases/orchestrator.py @@ -9,11 +9,11 @@ import anthropic +from ddev.ai.callbacks.callbacks import Callbacks from ddev.ai.phases.base import Phase, PhaseRegistry from ddev.ai.phases.checkpoint import CheckpointManager from ddev.ai.phases.config import FlowConfig, FlowConfigError from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger -from ddev.ai.react.callbacks import CallbackSet from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy from ddev.ai.tools.fs.file_registry import FileRegistry from ddev.event_bus.exceptions import FatalProcessingError @@ -48,7 +48,7 @@ def __init__( runtime_variables: dict[str, str], anthropic_client: anthropic.AsyncAnthropic, file_access_policy: FileAccessPolicy, - callback_sets: list[CallbackSet] | None = None, + callbacks: Callbacks | None = None, grace_period: float = 10, logger: logging.Logger | None = None, ) -> None: @@ -62,7 +62,7 @@ def __init__( self._checkpoint_path = checkpoint_path self._runtime_variables = runtime_variables self._anthropic_client = anthropic_client - self._callback_sets = callback_sets + self._callbacks: Callbacks = callbacks or Callbacks() self._phase_registry = PhaseRegistry() self._failed_phase: str | None = None self._failed_error: str | None = None @@ -108,7 +108,7 @@ async def on_initialize(self) -> None: flow_variables=config.variables, config_dir=config_dir, file_registry=self._file_registry, - callback_sets=self._callback_sets, + callbacks=self._callbacks, logger=self._logger, ) diff --git a/ddev/src/ddev/ai/react/process.py b/ddev/src/ddev/ai/react/process.py index 2a1adbad90b9b..4ed620947fc63 100644 --- a/ddev/src/ddev/ai/react/process.py +++ b/ddev/src/ddev/ai/react/process.py @@ -8,7 +8,7 @@ from ddev.ai.agent.base import BaseAgent from ddev.ai.agent.exceptions import AgentError from ddev.ai.agent.types import AgentResponse, StopReason, ToolResultMessage -from ddev.ai.react.callbacks import CallbackSet +from ddev.ai.callbacks.callbacks import Callbacks from ddev.ai.react.types import ReActResult from ddev.ai.tools.core.types import ToolResult from ddev.ai.tools.registry import ToolRegistry @@ -26,20 +26,20 @@ def __init__( self, agent: BaseAgent[Any], tool_registry: ToolRegistry, - callback_sets: list[CallbackSet] | None = None, + callbacks: Callbacks | None = None, compact_threshold_pct: float | None = 75.0, ) -> None: """ Args: agent: A BaseAgent subclass (e.g. AnthropicAgent). tool_registry: Registry of tools available in this loop. - callback_sets: Optional CallbackSet instances to observe loop events. + callbacks: Optional Callbacks instance to observe loop events. compact_threshold_pct: Context usage percentage at which the loop auto-compacts. None disables auto-compaction entirely. """ self._agent = agent self._tool_registry = tool_registry - self._callback_sets: list[CallbackSet] = callback_sets or [] + self._callbacks: Callbacks = callbacks or Callbacks() self._compact_threshold_pct = compact_threshold_pct def reset(self) -> None: @@ -55,8 +55,7 @@ async def compact(self, response: AgentResponse | None = None) -> tuple[int, int Returns (input_tokens, output_tokens) from the compaction API call. Returns (0, 0) if history was already compact and no API call was made. """ - for cb_set in self._callback_sets: - await cb_set.fire_before_compact() + await self._callbacks.fire_before_compact() compact_response = None if response is None or response.stop_reason != StopReason.TOOL_USE: @@ -64,8 +63,7 @@ async def compact(self, response: AgentResponse | None = None) -> tuple[int, int else: compact_response = await self._agent.compact_preserving_last_turn() - for cb_set in self._callback_sets: - await cb_set.fire_after_compact() + await self._callbacks.fire_after_compact() if compact_response is None: return 0, 0 return compact_response.usage.input_tokens, compact_response.usage.output_tokens @@ -93,16 +91,14 @@ async def start(self, prompt: str, allowed_tools: list[str] | None = None) -> Re Every exception is forwarded after notifying callbacks. """ try: - for cb_set in self._callback_sets: - await cb_set.fire_before_agent_send(1) + await self._callbacks.fire_before_agent_send(1) response = await self._agent.send(prompt, allowed_tools) iterations = 1 total_input = response.usage.input_tokens total_output = response.usage.output_tokens - for cb_set in self._callback_sets: - await cb_set.fire_agent_response(response, iterations) + await self._callbacks.fire_agent_response(response, iterations) # No iteration cap — this is an interactive CLI tool; the user can Ctrl+C to stop. while response.stop_reason == StopReason.TOOL_USE: @@ -121,21 +117,18 @@ async def start(self, prompt: str, allowed_tools: list[str] | None = None) -> Re tool_call_results = list(zip(response.tool_calls, tool_results, strict=True)) for tc, result in tool_call_results: - for cb_set in self._callback_sets: - await cb_set.fire_tool_call(tc, result, iterations) + await self._callbacks.fire_tool_call(tc, result, iterations) messages = [ToolResultMessage(tool_call_id=tc.id, result=result) for tc, result in tool_call_results] - for cb_set in self._callback_sets: - await cb_set.fire_before_agent_send(iterations + 1) + await self._callbacks.fire_before_agent_send(iterations + 1) response = await self._agent.send(messages, allowed_tools) iterations += 1 total_input += response.usage.input_tokens total_output += response.usage.output_tokens - for cb_set in self._callback_sets: - await cb_set.fire_agent_response(response, iterations) + await self._callbacks.fire_agent_response(response, iterations) if self._is_compact_needed(response): compact_in, compact_out = await self.compact(response) @@ -150,12 +143,10 @@ async def start(self, prompt: str, allowed_tools: list[str] | None = None) -> Re context_usage=response.usage.context_usage, ) - for cb_set in self._callback_sets: - await cb_set.fire_complete(react_result) + await self._callbacks.fire_complete(react_result) return react_result except BaseException as e: - for cb_set in self._callback_sets: - await cb_set.fire_error(e) + await self._callbacks.fire_error(e) raise diff --git a/ddev/tests/ai/callbacks/__init__.py b/ddev/tests/ai/callbacks/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/callbacks/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/react/test_callbacks.py b/ddev/tests/ai/callbacks/test_callbacks.py similarity index 75% rename from ddev/tests/ai/react/test_callbacks.py rename to ddev/tests/ai/callbacks/test_callbacks.py index a8780b1cd006f..79a4ae151fb8e 100644 --- a/ddev/tests/ai/react/test_callbacks.py +++ b/ddev/tests/ai/callbacks/test_callbacks.py @@ -5,7 +5,7 @@ import pytest from ddev.ai.agent.types import AgentResponse, StopReason, TokenUsage, ToolCall -from ddev.ai.react.callbacks import CallbackSet +from ddev.ai.callbacks.callbacks import Callbacks, CallbackSet from ddev.ai.react.types import ReActResult from ddev.ai.tools.core.types import ToolResult @@ -319,6 +319,71 @@ async def good(phase_id: str) -> None: assert fired == [True] +# --------------------------------------------------------------------------- +# on_phase_finish +# --------------------------------------------------------------------------- + + +async def test_phase_finish_registered_and_fired() -> None: + cb = CallbackSet() + fired: list[str] = [] + + @cb.on_phase_finish + async def h(phase_id: str) -> None: + fired.append(phase_id) + + await cb.fire_phase_finish("my-phase") + assert fired == ["my-phase"] + + +async def test_phase_finish_receives_correct_phase_id() -> None: + cb = CallbackSet() + received: list[str] = [] + + @cb.on_phase_finish + async def h(phase_id: str) -> None: + received.append(phase_id) + + await cb.fire_phase_finish("write-code") + assert received == ["write-code"] + + +async def test_phase_finish_multiple_handlers_all_fire_in_order() -> None: + cb = CallbackSet() + fired: list[int] = [] + + @cb.on_phase_finish + async def first(phase_id: str) -> None: + fired.append(1) + + @cb.on_phase_finish + async def second(phase_id: str) -> None: + fired.append(2) + + @cb.on_phase_finish + async def third(phase_id: str) -> None: + fired.append(3) + + await cb.fire_phase_finish("p") + assert fired == [1, 2, 3] + + +async def test_phase_finish_exception_is_swallowed() -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_phase_finish + async def bad(phase_id: str) -> None: + raise RuntimeError("boom") + + @cb.on_phase_finish + async def good(phase_id: str) -> None: + fired.append(True) + + await cb.fire_phase_finish("p") + assert fired == [True] + + # --------------------------------------------------------------------------- # on_before_agent_send # --------------------------------------------------------------------------- @@ -382,3 +447,48 @@ async def good(iteration: int) -> None: await cb.fire_before_agent_send(1) assert fired == [True] + + +# --------------------------------------------------------------------------- +# Callbacks container +# --------------------------------------------------------------------------- + + +async def test_callbacks_empty_is_noop(response: AgentResponse, tool_call: ToolCall, react_result: ReActResult) -> None: + callbacks = Callbacks() + await callbacks.fire_agent_response(response, 1) + await callbacks.fire_tool_call(tool_call, ToolResult(success=True, data="ok"), 1) + await callbacks.fire_complete(react_result) + await callbacks.fire_error(RuntimeError("boom")) + await callbacks.fire_before_compact() + await callbacks.fire_after_compact() + await callbacks.fire_before_agent_send(1) + await callbacks.fire_phase_start("p") + await callbacks.fire_phase_finish("p") + + +async def test_callbacks_dispatches_to_all_sets(response: AgentResponse) -> None: + fired_a: list[int] = [] + fired_b: list[int] = [] + + set_a = CallbackSet() + set_b = CallbackSet() + + @set_a.on_agent_response + async def a(response: AgentResponse, iteration: int) -> None: + fired_a.append(iteration) + + @set_b.on_agent_response + async def b(response: AgentResponse, iteration: int) -> None: + fired_b.append(iteration) + + callbacks = Callbacks([set_a, set_b]) + await callbacks.fire_agent_response(response, 3) + + assert fired_a == [3] + assert fired_b == [3] + + +async def test_callbacks_set_with_no_registered_handlers_is_noop(response: AgentResponse) -> None: + callbacks = Callbacks([CallbackSet()]) + await callbacks.fire_agent_response(response, 1) diff --git a/ddev/tests/ai/phases/test_base.py b/ddev/tests/ai/phases/test_base.py index 73e536ea3ff10..683a1f4ba9e61 100644 --- a/ddev/tests/ai/phases/test_base.py +++ b/ddev/tests/ai/phases/test_base.py @@ -151,7 +151,7 @@ def _make_phase( flow_variables=flow_variables or {}, config_dir=flow_dir, file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), - callback_sets=None, + callbacks=None, ) phase.queue = message_queue return phase, checkpoint_manager diff --git a/ddev/tests/ai/react/test_process.py b/ddev/tests/ai/react/test_process.py index 23a8aefe308a5..898a983ab4074 100644 --- a/ddev/tests/ai/react/test_process.py +++ b/ddev/tests/ai/react/test_process.py @@ -10,7 +10,7 @@ from ddev.ai.agent.base import BaseAgent from ddev.ai.agent.exceptions import AgentConnectionError from ddev.ai.agent.types import AgentResponse, ContextUsage, StopReason, TokenUsage, ToolCall, ToolResultMessage -from ddev.ai.react.callbacks import CallbackSet +from ddev.ai.callbacks.callbacks import Callbacks, CallbackSet from ddev.ai.react.process import ReActProcess from ddev.ai.react.types import ReActResult from ddev.ai.tools.core.types import ToolResult @@ -164,13 +164,13 @@ def make_tool_call( def make_process( agent: MockAgent, registry: MockToolRegistry | None = None, - callback_sets: list[CallbackSet] | None = None, + callbacks: Callbacks | None = None, compact_threshold_pct: float | None = None, ) -> ReActProcess: return ReActProcess( agent=agent, tool_registry=registry or MockToolRegistry(), - callback_sets=callback_sets, + callbacks=callbacks, compact_threshold_pct=compact_threshold_pct, ) @@ -285,7 +285,7 @@ async def test_tool_exception_on_tool_call_callback_fires_with_error_result() -> await ReActProcess( agent=agent, tool_registry=RaisingToolRegistry(ValueError("oops")), - callback_sets=[recorder.callback_set], + callbacks=Callbacks([recorder.callback_set]), ).start("x") assert len(recorder.tool_calls_seen) == 1 @@ -365,7 +365,9 @@ async def test_callbacks_invoked_correct_counts() -> None: recorder = CallbackRecorder() agent = MockAgent(responses) - result = await make_process(agent, registry=registry, callback_sets=[recorder.callback_set]).start("Run tools") + result = await make_process(agent, registry=registry, callbacks=Callbacks([recorder.callback_set])).start( + "Run tools" + ) assert len(recorder.agent_responses) == 2 assert recorder.agent_responses[0][1] == 1 @@ -384,7 +386,7 @@ async def test_callbacks_invoked_correct_counts() -> None: async def test_two_callback_sets_both_notified() -> None: agent = MockAgent([make_response(StopReason.END_TURN)]) rec_a, rec_b = CallbackRecorder(), CallbackRecorder() - await make_process(agent, callback_sets=[rec_a.callback_set, rec_b.callback_set]).start("x") + await make_process(agent, callbacks=Callbacks([rec_a.callback_set, rec_b.callback_set])).start("x") assert len(rec_a.complete_results) == 1 assert len(rec_b.complete_results) == 1 @@ -409,7 +411,7 @@ async def test_agent_error_notifies_and_reraises() -> None: process = ReActProcess( agent=ErrorAgent(), tool_registry=MockToolRegistry(), - callback_sets=[recorder.callback_set], + callbacks=Callbacks([recorder.callback_set]), ) with pytest.raises(AgentConnectionError): @@ -436,7 +438,7 @@ async def test_keyboard_interrupt_notifies_and_reraises() -> None: process = ReActProcess( agent=InterruptAgent(), tool_registry=MockToolRegistry(), - callback_sets=[recorder.callback_set], + callbacks=Callbacks([recorder.callback_set]), ) with pytest.raises(KeyboardInterrupt): @@ -462,7 +464,7 @@ async def test_cancelled_error_notifies_and_reraises() -> None: process = ReActProcess( agent=CancelledAgent(), tool_registry=MockToolRegistry(), - callback_sets=[recorder.callback_set], + callbacks=Callbacks([recorder.callback_set]), ) with pytest.raises(asyncio.CancelledError): @@ -544,7 +546,7 @@ async def test_compact_returns_tokens_when_compaction_occurs() -> None: async def test_compact_fires_before_and_after_callbacks() -> None: agent = MockAgent([]) recorder = CallbackRecorder() - await make_process(agent, callback_sets=[recorder.callback_set]).compact() + await make_process(agent, callbacks=Callbacks([recorder.callback_set])).compact() assert recorder.before_compacts == 1 assert recorder.after_compacts == 1 @@ -577,7 +579,7 @@ async def test_auto_compact_fires_callbacks() -> None: ] agent = MockAgent(responses) recorder = CallbackRecorder() - await make_process(agent, callback_sets=[recorder.callback_set], compact_threshold_pct=75.0).start("task") + await make_process(agent, callbacks=Callbacks([recorder.callback_set]), compact_threshold_pct=75.0).start("task") assert recorder.before_compacts == 1 assert recorder.after_compacts == 1 From 4da1fea536313fb3023def2dbbba5ce10757cd18 Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Thu, 7 May 2026 14:47:07 +0200 Subject: [PATCH 33/44] Add cycle detection to AI flow config (#23589) * Add cycles detection in flow config * Add two edge-cases tests * Remove _detect_cycle tests and add one for disjoined graphs * Find all possible cycles in _detect_cycles * Changes in _detect_cycles: change comment, add cycle limit and tighten test --- ddev/src/ddev/ai/phases/config.py | 53 ++++++++++++- ddev/tests/ai/phases/test_config.py | 113 ++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 4 deletions(-) diff --git a/ddev/src/ddev/ai/phases/config.py b/ddev/src/ddev/ai/phases/config.py index 534c3e9144013..1f44cf7452634 100644 --- a/ddev/src/ddev/ai/phases/config.py +++ b/ddev/src/ddev/ai/phases/config.py @@ -2,6 +2,8 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + from pathlib import Path import yaml @@ -14,6 +16,41 @@ class FlowConfigError(Exception): """Wraps Pydantic ValidationError or YAML errors with a user-friendly message.""" +def _detect_cycles( + dependency_map: dict[str, list[str]], + limit: int = 50, +) -> tuple[list[list[str]], bool]: + """Return every simple cycle in the dependency graph, each as an ordered list of phase IDs.""" + # Enumerate every simple cycle exactly once: from each node, DFS only through + # higher-ranked nodes, so each cycle is reported only when started from its + # lowest-ranked member. (Tiernan-style enumeration with rank canonicalization.) + rank = {n: i for i, n in enumerate(dependency_map)} + cycles: list[list[str]] = [] + + class _LimitReached(Exception): + """Raised when the cycle limit is reached.""" + + pass + + def dfs(start: str, current: str, path: list[str], on_path: set[str]): + for dep in dependency_map.get(current, []): + if dep == start: + cycles.append(path + [start]) + if len(cycles) >= limit: + raise _LimitReached + elif dep in rank and rank[dep] > rank[start] and dep not in on_path: + on_path.add(dep) + dfs(start, dep, path + [dep], on_path) + on_path.discard(dep) + + try: + for start in dependency_map: + dfs(start, start, [start], {start}) + except _LimitReached: + return cycles, True + return cycles, False + + class TaskConfig(BaseModel): model_config = ConfigDict(extra="forbid") name: str @@ -21,7 +58,7 @@ class TaskConfig(BaseModel): prompt: str | None = None @model_validator(mode="after") - def exactly_one_source(self) -> "TaskConfig": + def exactly_one_source(self) -> TaskConfig: if (self.prompt_path is None) == (self.prompt is None): raise ValueError("Exactly one of 'prompt_path' or 'prompt' must be set") return self @@ -35,7 +72,7 @@ class CheckpointConfig(BaseModel): memory_prompt_path: Path | None = None @model_validator(mode="after") - def exactly_one_source(self) -> "CheckpointConfig": + def exactly_one_source(self) -> CheckpointConfig: if (self.memory_prompt is None) == (self.memory_prompt_path is None): raise ValueError("Exactly one of 'memory_prompt' or 'memory_prompt_path' must be set") return self @@ -86,7 +123,7 @@ class FlowConfig(BaseModel): flow: list[FlowEntry] @model_validator(mode="after") - def cross_references(self) -> "FlowConfig": + def cross_references(self) -> FlowConfig: """Validate all cross-references between agents, phases, and dependencies.""" scheduled = {entry.phase for entry in self.flow} seen: set[str] = set() @@ -105,10 +142,18 @@ def cross_references(self) -> "FlowConfig": for phase_id, phase in self.phases.items(): if phase.agent not in self.agents: raise ValueError(f"Phase {phase_id!r} references unknown agent: {phase.agent!r}") + + dependency_map = {entry.phase: entry.dependencies for entry in self.flow} + cycles, truncated = _detect_cycles(dependency_map) + if cycles: + formatted = "\n ".join(" → ".join(c) for c in cycles) + suffix = f"\n (showing first {len(cycles)}; more cycles exist)" if truncated else "" + raise ValueError(f"Cycle(s) detected in flow:\n {formatted}{suffix}") + return self @classmethod - def from_yaml(cls, path: Path, config_dir: Path) -> "FlowConfig": + def from_yaml(cls, path: Path, config_dir: Path) -> FlowConfig: """Load, parse, and validate flow.yaml. Raises FlowConfigError on any problem.""" try: raw = yaml.safe_load(path.read_text()) diff --git a/ddev/tests/ai/phases/test_config.py b/ddev/tests/ai/phases/test_config.py index f0cc845dde704..82770b723b892 100644 --- a/ddev/tests/ai/phases/test_config.py +++ b/ddev/tests/ai/phases/test_config.py @@ -170,6 +170,13 @@ def test_flow_config_dependency_not_scheduled_in_flow(): FlowConfig.model_validate(raw) +def test_flow_config_duplicate_phase_raises(): + raw = _minimal_config() + raw["flow"] = [{"phase": "p1"}, {"phase": "p1"}] + with pytest.raises(ValidationError, match="Duplicate phase"): + FlowConfig.model_validate(raw) + + def test_flow_config_unknown_agent_in_phase(): raw = _minimal_config() raw["phases"]["p1"]["agent"] = "nonexistent" @@ -295,3 +302,109 @@ def test_from_yaml_invalid_yaml(tmp_path): def test_from_yaml_missing_file(tmp_path): with pytest.raises(FlowConfigError, match="Failed to load"): FlowConfig.from_yaml(tmp_path / "nonexistent.yaml", tmp_path) + + +# --------------------------------------------------------------------------- +# FlowConfig cycle detection via model_validate +# --------------------------------------------------------------------------- + + +def _three_phase_config() -> dict: + agent = {"tools": []} + task = {"name": "t", "prompt": "Do it."} + return { + "agents": {"writer": agent}, + "phases": { + "p1": {"agent": "writer", "tasks": [task]}, + "p2": {"agent": "writer", "tasks": [task]}, + "p3": {"agent": "writer", "tasks": [task]}, + }, + } + + +def test_flow_config_direct_cycle_raises(): + raw = _three_phase_config() + raw["flow"] = [ + {"phase": "p1", "dependencies": ["p2"]}, + {"phase": "p2", "dependencies": ["p1"]}, + ] + with pytest.raises(ValidationError, match="Cycle"): + FlowConfig.model_validate(raw) + + +def test_flow_config_three_node_cycle_raises(): + raw = _three_phase_config() + raw["flow"] = [ + {"phase": "p1", "dependencies": ["p3"]}, + {"phase": "p2", "dependencies": ["p1"]}, + {"phase": "p3", "dependencies": ["p2"]}, + ] + with pytest.raises(ValidationError, match="Cycle"): + FlowConfig.model_validate(raw) + + +def test_flow_config_acyclic_chain_ok(): + raw = _three_phase_config() + raw["flow"] = [ + {"phase": "p1"}, + {"phase": "p2", "dependencies": ["p1"]}, + {"phase": "p3", "dependencies": ["p1"]}, + ] + config = FlowConfig.model_validate(raw) + assert len(config.flow) == 3 + + +def test_flow_disjoined_graphs_ok(): + agent = {"tools": []} + task = {"name": "t", "prompt": "Do it."} + raw = { + "agents": {"writer": agent}, + "phases": { + "p1": {"agent": "writer", "tasks": [task]}, + "p2": {"agent": "writer", "tasks": [task]}, + "p3": {"agent": "writer", "tasks": [task]}, + "p4": {"agent": "writer", "tasks": [task]}, + }, + "flow": [ + {"phase": "p1"}, + {"phase": "p2", "dependencies": ["p1"]}, + {"phase": "p3"}, + {"phase": "p4", "dependencies": ["p3"]}, + ], + } + config = FlowConfig.model_validate(raw) + assert len(config.flow) == 4 + + +def test_flow_config_self_dependency_raises(): + raw = _minimal_config() + raw["flow"] = [{"phase": "p1", "dependencies": ["p1"]}] + with pytest.raises(ValidationError, match="Cycle"): + FlowConfig.model_validate(raw) + + +def test_flow_config_two_independent_cycles_reports_both(): + agent = {"tools": []} + task = {"name": "t", "prompt": "Do it."} + raw = { + "agents": {"writer": agent}, + "phases": { + "p1": {"agent": "writer", "tasks": [task]}, + "p2": {"agent": "writer", "tasks": [task]}, + "p3": {"agent": "writer", "tasks": [task]}, + "p4": {"agent": "writer", "tasks": [task]}, + }, + "flow": [ + # dependency edges: p1→p3→p2→p1 and p1→p4→p2→p1 + {"phase": "p1", "dependencies": ["p3", "p4"]}, + {"phase": "p2", "dependencies": ["p1"]}, + {"phase": "p3", "dependencies": ["p2"]}, + {"phase": "p4", "dependencies": ["p2"]}, + ], + } + with pytest.raises(ValidationError) as exc_info: + FlowConfig.model_validate(raw) + error = str(exc_info.value) + assert "Cycle" in error + assert "p1 → p3 → p2 → p1" in error + assert "p1 → p4 → p2 → p1" in error From da616729391c94a0c12dab01725249c6f8b0c72f Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Thu, 7 May 2026 17:19:58 +0200 Subject: [PATCH 34/44] Enable prompt caching on AnthropicAgent (#23627) * Add cache breakpoints to anthropic_client * Fix little bugs * Move imports only used for type checking to if TYPE_CHECKING --- ddev/src/ddev/ai/agent/anthropic_client.py | 68 ++++++++- ddev/tests/ai/agent/test_anthropic_client.py | 148 +++++++++++++++++++ 2 files changed, 208 insertions(+), 8 deletions(-) diff --git a/ddev/src/ddev/ai/agent/anthropic_client.py b/ddev/src/ddev/ai/agent/anthropic_client.py index 6d6af0838bbe1..ae5de3a8e6d76 100644 --- a/ddev/src/ddev/ai/agent/anthropic_client.py +++ b/ddev/src/ddev/ai/agent/anthropic_client.py @@ -2,19 +2,35 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from typing import Final +from __future__ import annotations + +from typing import TYPE_CHECKING, Final, overload import anthropic -from anthropic.types import MessageParam, ToolParam, ToolResultBlockParam +from anthropic.types import MessageParam from ddev.ai.agent.base import BaseAgent from ddev.ai.agent.exceptions import AgentAPIError, AgentConnectionError, AgentError, AgentRateLimitError from ddev.ai.agent.types import AgentResponse, ContextUsage, StopReason, TokenUsage, ToolCall, ToolResultMessage from ddev.ai.tools.registry import ToolRegistry +if TYPE_CHECKING: + from anthropic.types import ( + CacheControlEphemeralParam, + TextBlockParam, + ToolParam, + ToolResultBlockParam, + ) + DEFAULT_MODEL: Final[str] = "claude-sonnet-4-6" DEFAULT_MAX_TOKENS: Final[int] = 8192 # max tokens per response +# 1h TTL for the static prefix (system + tools): paid once, read for the whole session. +STATIC_CACHE_CONTROL: Final[CacheControlEphemeralParam] = {"type": "ephemeral", "ttl": "1h"} +# Default TTL (currently 5 min, but Anthropic may change it) for the sliding breakpoint +# on the last user message: re-written each turn, so a longer TTL would be wasted. +SLIDING_CACHE_CONTROL: Final[CacheControlEphemeralParam] = {"type": "ephemeral"} + class AnthropicAgent(BaseAgent[MessageParam]): """A wrapper around the Anthropic API that provides a simple interface for interacting with agents.""" @@ -102,6 +118,32 @@ def _to_tool_result_params(self, messages: list[ToolResultMessage]) -> list[Tool for msg in messages ] + @overload + @staticmethod + def _with_user_cache_breakpoint(content: str) -> list[TextBlockParam]: ... + + @overload + @staticmethod + def _with_user_cache_breakpoint(content: list[ToolResultBlockParam]) -> list[ToolResultBlockParam]: ... + + @staticmethod + def _with_user_cache_breakpoint( + content: str | list[ToolResultBlockParam], + ) -> list[TextBlockParam] | list[ToolResultBlockParam]: + """Return a block list with a sliding cache breakpoint on the last block.""" + if isinstance(content, str): + return [{"type": "text", "text": content, "cache_control": SLIDING_CACHE_CONTROL}] + if not content: + return [] + return [*content[:-1], {**content[-1], "cache_control": SLIDING_CACHE_CONTROL}] + + @staticmethod + def _with_tools_cache_breakpoint(tool_defs: list[ToolParam]) -> list[ToolParam]: + """Return a tool list with a static cache breakpoint on the last tool.""" + if not tool_defs: + return tool_defs + return [*tool_defs[:-1], {**tool_defs[-1], "cache_control": STATIC_CACHE_CONTROL}] + async def send( self, content: str | list[ToolResultMessage], @@ -114,19 +156,27 @@ async def send( Returns: An AgentResponse object containing the response from the agent. """ - tool_defs = self._get_tool_definitions(allowed_tools) + tool_defs = self._with_tools_cache_breakpoint(self._get_tool_definitions(allowed_tools)) api_content: str | list[ToolResultBlockParam] = ( self._to_tool_result_params(content) if isinstance(content, list) else content ) - user_msg: MessageParam = {"role": "user", "content": api_content} - messages = [*self._history, user_msg] + user_msg_for_history: MessageParam = {"role": "user", "content": api_content} + user_msg_for_request: MessageParam = { + "role": "user", + "content": self._with_user_cache_breakpoint(api_content), + } + messages = [*self._history, user_msg_for_request] + + system_param: list[TextBlockParam] = [ + {"type": "text", "text": self._system_prompt, "cache_control": STATIC_CACHE_CONTROL} + ] try: response = await self._client.messages.create( model=self._model, max_tokens=self._max_tokens, - system=self._system_prompt, + system=system_param, messages=messages, tools=tool_defs if tool_defs else anthropic.NOT_GIVEN, ) @@ -174,7 +224,9 @@ async def send( usage=usage, ) - # Save to history only after a successful response. - self._history.extend([user_msg, {"role": "assistant", "content": response.content}]) + # Save to history only after a successful response. Use the unmarked form so the + # cache_control breakpoint is only ever on the latest user message — this keeps the + # request below the 4-marker limit regardless of conversation length. + self._history.extend([user_msg_for_history, {"role": "assistant", "content": response.content}]) return agent_response diff --git a/ddev/tests/ai/agent/test_anthropic_client.py b/ddev/tests/ai/agent/test_anthropic_client.py index 739f99d53c74a..5ec4892923220 100644 --- a/ddev/tests/ai/agent/test_anthropic_client.py +++ b/ddev/tests/ai/agent/test_anthropic_client.py @@ -485,3 +485,151 @@ async def test_error_mid_conversation_leaves_history_unchanged() -> None: await agent.send("Second message") assert agent.history == history_after_first + + +# --------------------------------------------------------------------------- +# Prompt caching: static breakpoints (system + last tool, 1h TTL) +# --------------------------------------------------------------------------- + + +async def test_system_prompt_sent_as_block_with_static_cache_control() -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(mock_response=resp) + + await agent.send("Hi") + + assert create_mock.call_args.kwargs["system"] == [ + { + "type": "text", + "text": "You are helpful.", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ] + + +@pytest.mark.parametrize( + "tool_names", + [["only"], ["a", "b"], ["a", "b", "c", "d"]], + ids=["single_tool", "two_tools", "four_tools"], +) +async def test_only_last_tool_carries_static_cache_control(tool_names: list[str]) -> None: + registry = ToolRegistry([FakeTool(n) for n in tool_names]) + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(tools=registry, mock_response=resp) + + await agent.send("Hi") + + sent_tools = create_mock.call_args.kwargs["tools"] + assert all("cache_control" not in t for t in sent_tools[:-1]) + assert sent_tools[-1]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + + +async def test_allowed_tools_subset_places_cache_control_on_last_of_subset() -> None: + registry = ToolRegistry([FakeTool(n) for n in ["a", "b", "c"]]) + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(tools=registry, mock_response=resp) + + await agent.send("Hi", allowed_tools=["a", "b"]) + + sent_tools = create_mock.call_args.kwargs["tools"] + assert [t["name"] for t in sent_tools] == ["a", "b"] + assert "cache_control" not in sent_tools[0] + assert sent_tools[-1]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + + +# --------------------------------------------------------------------------- +# Prompt caching: sliding breakpoint on the last user message block (default TTL) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "content", + [ + pytest.param("Hi there", id="str"), + pytest.param( + [ToolResultMessage(tool_call_id="t1", result=ToolResult(success=True, data="r1"))], + id="single_tool_result", + ), + pytest.param( + [ToolResultMessage(tool_call_id=f"t{i}", result=ToolResult(success=True, data=f"r{i}")) for i in range(3)], + id="multiple_tool_results", + ), + ], +) +async def test_sliding_cache_control_on_last_user_block_only( + content: str | list[ToolResultMessage], +) -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(mock_response=resp) + + await agent.send(content) + + blocks = create_mock.call_args.kwargs["messages"][-1]["content"] + assert isinstance(blocks, list) and blocks + assert all("cache_control" not in b for b in blocks[:-1]) + assert blocks[-1]["cache_control"] == {"type": "ephemeral"} + assert "ttl" not in blocks[-1]["cache_control"] + + +async def test_empty_tool_results_produces_empty_content_block() -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(mock_response=resp) + + await agent.send([]) + + blocks = create_mock.call_args.kwargs["messages"][-1]["content"] + assert blocks == [] + + +# --------------------------------------------------------------------------- +# Prompt caching: history must not retain cache_control markers +# (otherwise multi-turn requests would exceed the 4-marker limit) +# --------------------------------------------------------------------------- + + +async def test_history_str_message_keeps_raw_str_form() -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, _ = make_agent(mock_response=resp) + + await agent.send("Hi") + + assert agent.history[0] == {"role": "user", "content": "Hi"} + + +async def test_history_tool_result_blocks_have_no_cache_control() -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, _ = make_agent(mock_response=resp) + + tool_results = [ + ToolResultMessage(tool_call_id=f"t{i}", result=ToolResult(success=True, data=f"r{i}")) for i in range(2) + ] + await agent.send(tool_results) + + blocks = agent.history[0]["content"] + assert all("cache_control" not in b for b in blocks) + + +async def test_multi_turn_only_latest_user_message_in_request_has_cache_control() -> None: + first_resp = make_response("tool_use", [make_tool_use_block(id="t1")]) + second_resp = make_response("end_turn", [make_text_block("done")]) + + client = MagicMock(spec=anthropic.AsyncAnthropic) + client.messages = MagicMock() + client.messages.create = AsyncMock(side_effect=[first_resp, second_resp]) + client.models = MagicMock() + client.models.retrieve = AsyncMock(return_value=SimpleNamespace(max_input_tokens=FAKE_CONTEXT_WINDOW)) + agent = AnthropicAgent(client=client, tools=ToolRegistry([]), system_prompt="sp", name="t") + + await agent.send("First") + await agent.send([ToolResultMessage(tool_call_id="t1", result=ToolResult(success=True, data="r"))]) + + first_call_messages = client.messages.create.call_args_list[0].kwargs["messages"] + assert first_call_messages[-1]["content"] == [ + {"type": "text", "text": "First", "cache_control": {"type": "ephemeral"}} + ] + + second_call_messages = client.messages.create.call_args_list[1].kwargs["messages"] + assert second_call_messages[0] == {"role": "user", "content": "First"} + latest_blocks = second_call_messages[-1]["content"] + assert all("cache_control" not in b for b in latest_blocks[:-1]) + assert latest_blocks[-1]["cache_control"] == {"type": "ephemeral"} From 180688a361391226f7ccb538ae9d5991dc36d027 Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Wed, 20 May 2026 12:50:18 +0200 Subject: [PATCH 35/44] Add ddev validate to ddev tools (#23636) * Add ddev validate * Drop ddev validate all and fix some bugs * Use --sync for every subcommand * Add ddev validate all again --- ddev/src/ddev/ai/tools/registry.py | 1 + ddev/src/ddev/ai/tools/shell/ddev/validate.py | 61 +++++++++++++++++++ .../ai/tools/shell/ddev/test_ddev_tools.py | 26 ++++++++ 3 files changed, 88 insertions(+) create mode 100644 ddev/src/ddev/ai/tools/shell/ddev/validate.py diff --git a/ddev/src/ddev/ai/tools/registry.py b/ddev/src/ddev/ai/tools/registry.py index f8249bfb7fd93..3fd255f7bda1b 100644 --- a/ddev/src/ddev/ai/tools/registry.py +++ b/ddev/src/ddev/ai/tools/registry.py @@ -70,6 +70,7 @@ class ToolSpec: "ddev_env_stop": ToolSpec("shell.ddev.env_stop", "DdevEnvStopTool"), "ddev_env_test": ToolSpec("shell.ddev.env_test", "DdevEnvTestTool"), "ddev_release_changelog": ToolSpec("shell.ddev.release_changelog", "DdevReleaseChangelogTool"), + "ddev_validate": ToolSpec("shell.ddev.validate", "DdevValidateTool"), } diff --git a/ddev/src/ddev/ai/tools/shell/ddev/validate.py b/ddev/src/ddev/ai/tools/shell/ddev/validate.py new file mode 100644 index 0000000000000..9d6d87a050e84 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/validate.py @@ -0,0 +1,61 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated, Literal + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + +ValidateSubcommand = Literal["config", "models", "metadata", "all"] + + +class DdevValidateInput(BaseToolInput): + subcommand: Annotated[ + ValidateSubcommand, + Field( + description=( + "Which validator to run. Options:" + "- 'config': validates assets/configuration/spec.yaml against data/conf.yaml.example. " + "- 'models': validates spec.yaml against datadog_checks//config_models/. " + "- 'metadata': validates metadata.csv. " + "- 'all': runs all ~20 validators in parallel. Some of these always scan the entire " + "repository, so the output may include failures for files outside of . " + "IGNORE those unrelated failures — only act on failures that reference files inside " + "your integration's directory. It might take a long time to run. Use 'all' only as " + "a final sweep to catch issues the targeted validators do not cover." + ) + ), + ] + integration: Annotated[str, Field(description="Integration name to validate")] + sync: Annotated[ + bool, + Field( + description=( + "Regenerate / auto-fix derived files instead of only checking. " + "For 'config', regenerates conf.yaml.example. " + "For 'models', regenerates config_models/. " + "For 'metadata', rewrites metadata.csv into canonical form. " + "For 'all', auto-fixes every validator that supports it." + ) + ), + ] = False + + +class DdevValidateTool(CmdTool[DdevValidateInput]): + """Validates an integration's spec, config example, config models, or metadata.csv. + Set `sync=true` to regenerate the derived files from spec.yaml.""" + + timeout = 660 + + @property + def name(self) -> str: + return "ddev_validate" + + def cmd(self, tool_input: DdevValidateInput) -> list[str]: + cmd = ["ddev", "--no-interactive", "validate", tool_input.subcommand] + if tool_input.sync: + cmd.append("--fix" if tool_input.subcommand == "all" else "--sync") + cmd.append(tool_input.integration) + return cmd diff --git a/ddev/tests/ai/tools/shell/ddev/test_ddev_tools.py b/ddev/tests/ai/tools/shell/ddev/test_ddev_tools.py index cc4eb72a7e182..fa7f30378225e 100644 --- a/ddev/tests/ai/tools/shell/ddev/test_ddev_tools.py +++ b/ddev/tests/ai/tools/shell/ddev/test_ddev_tools.py @@ -11,6 +11,7 @@ from ddev.ai.tools.shell.ddev.env_stop import DdevEnvStopTool, EnvStopInput from ddev.ai.tools.shell.ddev.env_test import DdevEnvTestTool, EnvTestInput from ddev.ai.tools.shell.ddev.release_changelog import DdevReleaseChangelogTool, ReleaseChangelogInput +from ddev.ai.tools.shell.ddev.validate import DdevValidateInput, DdevValidateTool # --- ddev create --- @@ -163,3 +164,28 @@ def test_release_changelog_cmd_message_placement(): def test_release_changelog_invalid_change_type_raises(): with pytest.raises(ValidationError): ReleaseChangelogInput(change_type="patch", integration="mycheck", message="Some message") + + +# --- ddev validate --- + + +@pytest.mark.parametrize("subcommand", ["config", "models", "metadata", "all"]) +def test_validate_cmd_all_subcommands(subcommand: str): + cmd = DdevValidateTool().cmd(DdevValidateInput(subcommand=subcommand, integration="mycheck")) + assert cmd == ["ddev", "--no-interactive", "validate", subcommand, "mycheck"] + + +@pytest.mark.parametrize("subcommand", ["config", "models", "metadata"]) +def test_validate_cmd_sync_flag_per_subcommand(subcommand: str): + cmd = DdevValidateTool().cmd(DdevValidateInput(subcommand=subcommand, integration="mycheck", sync=True)) + assert cmd == ["ddev", "--no-interactive", "validate", subcommand, "--sync", "mycheck"] + + +def test_validate_cmd_all_uses_fix_flag(): + cmd = DdevValidateTool().cmd(DdevValidateInput(subcommand="all", integration="mycheck", sync=True)) + assert cmd == ["ddev", "--no-interactive", "validate", "all", "--fix", "mycheck"] + + +def test_validate_invalid_subcommand_raises(): + with pytest.raises(ValidationError): + DdevValidateInput(subcommand="lint", integration="mycheck") From 04df008817b01421db3ea0286afd37868277f3ce Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Thu, 21 May 2026 17:46:28 +0200 Subject: [PATCH 36/44] [ddev/ai/phases]: Introduce AgenticPhase and make Phase an abstract lifecycle base (#23663) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(ai/phases): introduce PhaseOutcome and abstract Phase.execute() - Add PhaseOutcome dataclass (memory_text, token counts, extra_checkpoint) - Add validate_config() classmethod to Phase (no-op default) - Add execute() method that implements the agent pipeline (later to be overridden by AgentPhase) - Rewrite process_message() to call execute() and assemble the checkpoint from PhaseOutcome * refactor(ai/phases): extract AgentPhase from Phase - Create agent_phase.py with AgentPhase(Phase) that owns the LLM pipeline: before_react/after_react hooks, run_tasks, execute() - Move render_task_prompt and render_memory_prompt to agent_phase.py - AgentPhase.validate_config enforces agent, known-agent, and non-empty tasks - Phase.execute() now raises NotImplementedError — subclasses must implement it - Strip base.py of all agent-specific code and imports - Split test_base.py into lifecycle-only tests (using _StubPhase) and test_agent_phase.py for the agent-driven behaviour tests * refactor(ai/phases): make PhaseConfig.agent and .tasks optional - type default: "Phase" → "AgentPhase" - agent: str (required) → str | None = None - tasks: list[TaskConfig] (required) → list[TaskConfig] = [] - Remove at_least_one_task field validator (now enforced by AgentPhase.validate_config) - FlowConfig.cross_references: skip unknown-agent check when agent is None - orchestrator: guard agent_config lookup against None, import AgentConfig - test_config.py: update type assertion, remove empty_tasks test, add test_flow_config_phase_without_agent_validates - test_base.py / test_agent_phase.py: drop model_construct workarounds * refactor(ai/phases): invoke Phase.validate_config from orchestrator - Call phase_cls.validate_config(phase_id, config, agents) immediately after resolving the phase class in on_initialize — only for phases scheduled in flow: - Orphan phases (defined but absent from flow:) are skipped before the call - test_orchestrator.py: drop explicit type: Phase lines from fixtures (use AgentPhase default), assert AgentPhase is registered by discovery, add tests for validate_config invocation and orphan-skip behaviour * Rename AgentPhase to AgenticPhase * Split AgenticPhase's execute into smaller functions and added tests for them * Move agent and client parameters to AgenticPhase and make Phase abstract * Add e2e Phase contract test * Move some tests from agentic phase to conftest * Phase not registered and improve tests * Prevent extra_checkpoint from overriding checkpoint_payload * Make Phase and Orchestrator model-agnostic * Add Phase.extra_init_kwargs and agent/build.py tests --- ddev/src/ddev/ai/agent/build.py | 77 +++ ddev/src/ddev/ai/phases/agentic_phase.py | 185 +++++++ ddev/src/ddev/ai/phases/base.py | 203 ++----- ddev/src/ddev/ai/phases/checkpoint.py | 6 + ddev/src/ddev/ai/phases/config.py | 16 +- ddev/src/ddev/ai/phases/orchestrator.py | 52 +- ddev/tests/ai/agent/test_build.py | 76 +++ ddev/tests/ai/phases/conftest.py | 69 ++- ddev/tests/ai/phases/test_agentic_phase.py | 502 +++++++++++++++++ ddev/tests/ai/phases/test_base.py | 611 +++------------------ ddev/tests/ai/phases/test_checkpoint.py | 14 + ddev/tests/ai/phases/test_config.py | 24 +- ddev/tests/ai/phases/test_orchestrator.py | 301 +++++----- 13 files changed, 1245 insertions(+), 891 deletions(-) create mode 100644 ddev/src/ddev/ai/agent/build.py create mode 100644 ddev/src/ddev/ai/phases/agentic_phase.py create mode 100644 ddev/tests/ai/agent/test_build.py create mode 100644 ddev/tests/ai/phases/test_agentic_phase.py diff --git a/ddev/src/ddev/ai/agent/build.py b/ddev/src/ddev/ai/agent/build.py new file mode 100644 index 0000000000000..97e24aecb5183 --- /dev/null +++ b/ddev/src/ddev/ai/agent/build.py @@ -0,0 +1,77 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING, Any + +from ddev.ai.agent.anthropic_client import AnthropicAgent +from ddev.ai.agent.base import BaseAgent +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.registry import ToolRegistry + +if TYPE_CHECKING: + from ddev.ai.phases.config import AgentConfig + +AgentBuilder = Callable[[str, str], tuple[BaseAgent[Any], ToolRegistry]] + + +def _resolve_client(agent_clients: dict[str, Any], provider: str) -> Any: + client = agent_clients.get(provider) + if client is None: + raise ValueError(f"No client provided for agent provider {provider!r}") + return client + + +def build_agent( + agent_config: AgentConfig, + agent_clients: dict[str, Any], + system_prompt: str, + owner_id: str, + file_registry: FileRegistry, +) -> tuple[BaseAgent[Any], ToolRegistry]: + """Construct a provider-specific BaseAgent and its ToolRegistry from an AgentConfig.""" + + tool_registry = ToolRegistry.from_names( + agent_config.tools, + owner_id=owner_id, + file_registry=file_registry, + ) + + if agent_config.provider == "anthropic": + kwargs: dict[str, Any] = {} + if agent_config.model is not None: + kwargs["model"] = agent_config.model + if agent_config.max_tokens is not None: + kwargs["max_tokens"] = agent_config.max_tokens + agent: BaseAgent[Any] = AnthropicAgent( + client=_resolve_client(agent_clients, "anthropic"), + tools=tool_registry, + system_prompt=system_prompt, + name=owner_id, + **kwargs, + ) + return agent, tool_registry + + raise ValueError(f"Unknown agent provider: {agent_config.provider!r}") + + +def make_agent_builder( + agent_config: AgentConfig, + agent_clients: dict[str, Any], + file_registry: FileRegistry, +) -> AgentBuilder: + """Return a closure that builds an agent+registry given a rendered system_prompt and owner_id.""" + + def builder(system_prompt: str, owner_id: str) -> tuple[BaseAgent[Any], ToolRegistry]: + return build_agent( + agent_config=agent_config, + agent_clients=agent_clients, + system_prompt=system_prompt, + owner_id=owner_id, + file_registry=file_registry, + ) + + return builder diff --git a/ddev/src/ddev/ai/phases/agentic_phase.py b/ddev/src/ddev/ai/phases/agentic_phase.py new file mode 100644 index 0000000000000..c8a36767cd24b --- /dev/null +++ b/ddev/src/ddev/ai/phases/agentic_phase.py @@ -0,0 +1,185 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import logging +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from ddev.ai.agent.base import BaseAgent +from ddev.ai.agent.build import AgentBuilder, make_agent_builder +from ddev.ai.callbacks.callbacks import Callbacks +from ddev.ai.phases.base import Phase, PhaseOutcome +from ddev.ai.phases.checkpoint import CheckpointManager +from ddev.ai.phases.config import AgentConfig, CheckpointConfig, FlowConfigError, PhaseConfig, TaskConfig +from ddev.ai.phases.template import render_inline, render_prompt +from ddev.ai.react.process import ReActProcess +from ddev.ai.tools.fs.file_registry import FileRegistry + + +def render_task_prompt( + task: TaskConfig, + config_dir: Path, + context: dict[str, Any], + resolver: Callable[[str], str] | None = None, +) -> str: + """Render a task prompt — from file if prompt_path is set, inline otherwise.""" + if task.prompt_path is not None: + return render_prompt(config_dir / task.prompt_path, context, resolver) + if task.prompt is None: + raise FlowConfigError("TaskConfig must set either 'prompt' or 'prompt_path'") + return render_inline(task.prompt, context, resolver) + + +def render_memory_prompt( + checkpoint: CheckpointConfig, + config_dir: Path, + context: dict[str, Any], +) -> str: + """Render a checkpoint memory prompt — from file if memory_prompt_path is set, inline otherwise.""" + if checkpoint.memory_prompt_path is not None: + return render_prompt(config_dir / checkpoint.memory_prompt_path, context) + if checkpoint.memory_prompt is None: + raise FlowConfigError("CheckpointConfig must set either 'memory_prompt' or 'memory_prompt_path'") + return render_inline(checkpoint.memory_prompt, context) + + +class AgenticPhase(Phase): + """Phase that owns an LLM agent and drives one or more ReAct loops.""" + + def __init__( + self, + phase_id: str, + dependencies: list[str], + config: PhaseConfig, + agent_builder: AgentBuilder, + checkpoint_manager: CheckpointManager, + runtime_variables: dict[str, str], + flow_variables: dict[str, str], + config_dir: Path, + file_registry: FileRegistry, + callbacks: Callbacks | None = None, + logger: logging.Logger | None = None, + ) -> None: + super().__init__( + phase_id=phase_id, + dependencies=dependencies, + config=config, + checkpoint_manager=checkpoint_manager, + runtime_variables=runtime_variables, + flow_variables=flow_variables, + config_dir=config_dir, + file_registry=file_registry, + callbacks=callbacks, + logger=logger, + ) + self._agent_builder = agent_builder + + @classmethod + def validate_config( + cls, + phase_id: str, + config: PhaseConfig, + agents: dict[str, AgentConfig], + ) -> None: + if config.agent is None: + raise FlowConfigError(f"Phase {phase_id!r} (AgenticPhase) requires 'agent'") + if config.agent not in agents: + raise FlowConfigError(f"Phase {phase_id!r} references unknown agent: {config.agent!r}") + if not config.tasks: + raise FlowConfigError(f"Phase {phase_id!r} (AgenticPhase) must have at least one task") + + @classmethod + def extra_init_kwargs( + cls, + phase_id: str, + phase_config: PhaseConfig, + agents: dict[str, AgentConfig], + agent_clients: dict[str, Any], + file_registry: FileRegistry, + ) -> dict[str, Any]: + if phase_config.agent is None: + raise FlowConfigError(f"Phase {phase_id!r} (AgenticPhase) requires 'agent'") + return { + "agent_builder": make_agent_builder( + agent_config=agents[phase_config.agent], + agent_clients=agent_clients, + file_registry=file_registry, + ) + } + + def before_react(self) -> None: + """Called once before agent/tools are created. Override for phase-specific setup.""" + + def after_react(self) -> None: + """Called once after all tasks complete. Override for phase-specific teardown.""" + + async def run_tasks( + self, + process: ReActProcess, + context: dict[str, Any], + ) -> tuple[int, int]: + """Run the task loop. Returns (total_input_tokens, total_output_tokens). + + Override to customize task execution — e.g. add retries, change ordering, etc. + Default implementation iterates through config.tasks sequentially. + """ + total_input = total_output = 0 + last_result = None + for task in self._config.tasks: + if last_result is not None and last_result.context_usage is not None: + if last_result.context_usage.context_pct >= self._config.context_compact_threshold_pct: + compact_in, compact_out = await process.compact() + total_input += compact_in + total_output += compact_out + prompt = render_task_prompt(task, self._config_dir, context, self._resolver) + last_result = await process.start(prompt) + total_input += last_result.total_input_tokens + total_output += last_result.total_output_tokens + return total_input, total_output + + def _build_agent_and_process(self, context: dict[str, Any]) -> tuple[BaseAgent[Any], ReActProcess]: + """Build the agent and ReAct process used to drive task execution.""" + system_prompt = render_prompt( + self._config_dir / "prompts" / f"{self._config.agent}.md", + context, + self._resolver, + ) + agent, tool_registry = self._agent_builder(system_prompt, self._phase_id) + process = ReActProcess( + agent=agent, + tool_registry=tool_registry, + callbacks=self._callbacks, + ) + return agent, process + + async def _run_memory_step( + self, + agent: BaseAgent[Any], + context: dict[str, Any], + ) -> tuple[str, int, int]: + """Run the final summary turn. Returns (memory_text, input_tokens, output_tokens).""" + user_additions = None + if self._config.checkpoint is not None: + user_additions = render_memory_prompt(self._config.checkpoint, self._config_dir, context) + memory_prompt = self._checkpoint_manager.build_memory_prompt(user_additions) + + await self._callbacks.fire_before_agent_send(1) + response = await agent.send(memory_prompt, allowed_tools=[]) + await self._callbacks.fire_agent_response(response, 1) + return response.text, response.usage.input_tokens, response.usage.output_tokens + + async def execute(self, context: dict[str, Any]) -> PhaseOutcome: + self.before_react() + agent, process = self._build_agent_and_process(context) + total_input, total_output = await self.run_tasks(process, context) + self.after_react() + + memory_text, mem_in, mem_out = await self._run_memory_step(agent, context) + + return PhaseOutcome( + memory_text=memory_text, + total_input_tokens=total_input + mem_in, + total_output_tokens=total_output + mem_out, + ) diff --git a/ddev/src/ddev/ai/phases/base.py b/ddev/src/ddev/ai/phases/base.py index c0f1c9fe226c9..126d5772ba1ea 100644 --- a/ddev/src/ddev/ai/phases/base.py +++ b/ddev/src/ddev/ai/phases/base.py @@ -3,26 +3,30 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import logging +from abc import abstractmethod from collections.abc import Callable +from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path from typing import Any -import anthropic - -from ddev.ai.agent.anthropic_client import AnthropicAgent from ddev.ai.callbacks.callbacks import Callbacks from ddev.ai.phases.checkpoint import CheckpointManager -from ddev.ai.phases.config import AgentConfig, CheckpointConfig, FlowConfigError, PhaseConfig, TaskConfig +from ddev.ai.phases.config import AgentConfig, PhaseConfig from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger -from ddev.ai.phases.template import render_inline, render_prompt -from ddev.ai.react.process import ReActProcess from ddev.ai.tools.fs.file_registry import FileRegistry -from ddev.ai.tools.registry import ToolRegistry from ddev.event_bus.exceptions import MessageProcessingError, ProcessorHookError from ddev.event_bus.orchestrator import AsyncProcessor, BaseMessage +@dataclass +class PhaseOutcome: + memory_text: str + total_input_tokens: int = 0 + total_output_tokens: int = 0 + extra_checkpoint: dict[str, Any] = field(default_factory=dict) + + class PhaseRegistry: def __init__(self) -> None: self._registry: dict[str, type["Phase"]] = {} @@ -39,45 +43,11 @@ def get(self, name: str) -> type["Phase"]: return self._registry[name] -def _make_memory_resolver(checkpoint_manager: CheckpointManager) -> Callable[[str], str]: - """Build a resolver that reads phase memory files on demand for template substitution.""" - - def resolve(key: str) -> str: - if key.endswith("_memory"): - return checkpoint_manager.memory_content(key.removesuffix("_memory")) - return f"" - - return resolve - - -def render_task_prompt( - task: TaskConfig, - config_dir: Path, - context: dict[str, Any], - resolver: Callable[[str], str] | None = None, -) -> str: - """Render a task prompt -- from file if prompt_path is set, inline otherwise.""" - if task.prompt_path is not None: - return render_prompt(config_dir / task.prompt_path, context, resolver) - if task.prompt is None: - raise FlowConfigError("TaskConfig must set either 'prompt' or 'prompt_path'") - return render_inline(task.prompt, context, resolver) - - -def render_memory_prompt(checkpoint: CheckpointConfig, config_dir: Path, context: dict[str, Any]) -> str: - """Render a checkpoint memory prompt -- from file if memory_prompt_path is set, inline otherwise.""" - if checkpoint.memory_prompt_path is not None: - return render_prompt(config_dir / checkpoint.memory_prompt_path, context) - if checkpoint.memory_prompt is None: - raise FlowConfigError("CheckpointConfig must set either 'memory_prompt' or 'memory_prompt_path'") - return render_inline(checkpoint.memory_prompt, context) - - class Phase(AsyncProcessor[PhaseTrigger]): - """Concrete base for all phases. + """Lifecycle base for all phases. process_message() implements the immutable pipeline skeleton. - Override before_react(), after_react(), and run_tasks() to customize phase behaviour. + Subclasses implement execute() to provide phase-specific logic. Registered in PhaseRegistry by _discover_and_register_phases() at startup. """ @@ -86,8 +56,6 @@ def __init__( phase_id: str, dependencies: list[str], config: PhaseConfig, - agent_config: AgentConfig, - anthropic_client: anthropic.AsyncAnthropic, checkpoint_manager: CheckpointManager, runtime_variables: dict[str, str], flow_variables: dict[str, str], @@ -101,8 +69,6 @@ def __init__( self._dependencies = set(dependencies) self._remaining_dependencies = set(dependencies) self._config = config - self._agent_config = agent_config - self._anthropic_client = anthropic_client self._checkpoint_manager = checkpoint_manager self._runtime_variables = runtime_variables self._flow_variables = flow_variables @@ -132,122 +98,65 @@ def should_process_message(self, message: BaseMessage) -> bool: self._executed = True return True - def before_react(self) -> None: - """Called once before agent/tools are created. Override for phase-specific setup.""" - - def after_react(self) -> None: - """Called once after all tasks complete. Override for phase-specific teardown.""" + @classmethod + def validate_config( + cls, + phase_id: str, + config: PhaseConfig, + agents: dict[str, AgentConfig], + ) -> None: + """Override to enforce per-subclass config invariants. Raise FlowConfigError on mismatch.""" + return None - async def run_tasks( - self, - process: ReActProcess, - context: dict[str, Any], - ) -> tuple[int, int]: - """Run the task loop. Returns (total_input_tokens, total_output_tokens). + @classmethod + def extra_init_kwargs( + cls, + phase_id: str, + phase_config: PhaseConfig, + agents: dict[str, AgentConfig], + agent_clients: dict[str, Any], + file_registry: FileRegistry, + ) -> dict[str, Any]: + """Override to inject subclass-specific kwargs into __init__ at construction time.""" + return {} - Override to customize task execution -- e.g. add retries, change ordering, etc. - Default implementation iterates through config.tasks sequentially. - """ - total_input = total_output = 0 - last_result = None - for task in self._config.tasks: - if last_result is not None and last_result.context_usage is not None: - if last_result.context_usage.context_pct >= self._config.context_compact_threshold_pct: - compact_in, compact_out = await process.compact() - total_input += compact_in - total_output += compact_out - prompt = render_task_prompt(task, self._config_dir, context, self._resolver) - last_result = await process.start(prompt) - total_input += last_result.total_input_tokens - total_output += last_result.total_output_tokens - return total_input, total_output + @abstractmethod + async def execute(self, context: dict[str, Any]) -> PhaseOutcome: ... async def process_message(self, message: PhaseTrigger) -> None: - """Full phase pipeline. Not intended to be overridden -- customise via the extension points.""" - # 1. Record start time and notify observers + """Immutable pipeline skeleton. Not intended to be overridden — implement execute() instead.""" self._started_at = datetime.now(UTC) await self._callbacks.fire_phase_start(self._phase_id) - # 2. Build template context and memory resolver context: dict[str, Any] = { **self._flow_variables, **self._runtime_variables, "phase_name": self._phase_id, "checkpoints": self._checkpoint_manager.read(), } - self._resolver = _make_memory_resolver(self._checkpoint_manager) + self._resolver = self._checkpoint_manager.resolve_template_variable - # 3. Call before_react() - self.before_react() + outcome = await self.execute(context) - # 4. Create system prompt, ToolRegistry, AnthropicAgent - system_prompt = render_prompt( - self._config_dir / "prompts" / f"{self._config.agent}.md", - context, - self._resolver, - ) - tool_registry = ToolRegistry.from_names( - self._agent_config.tools, - owner_id=self._phase_id, - file_registry=self._file_registry, - ) - - agent_kwargs: dict[str, Any] = {} - if self._agent_config.model is not None: - agent_kwargs["model"] = self._agent_config.model - if self._agent_config.max_tokens is not None: - agent_kwargs["max_tokens"] = self._agent_config.max_tokens - - agent = AnthropicAgent( - client=self._anthropic_client, - tools=tool_registry, - system_prompt=system_prompt, - name=self._phase_id, - **agent_kwargs, - ) - - # 5. Build ReActProcess - process = ReActProcess( - agent=agent, - tool_registry=tool_registry, - callbacks=self._callbacks, - ) - - # 6. Call run_tasks() - total_input, total_output = await self.run_tasks(process, context) - - # 7. Call after_react() - self.after_react() - - # 8. Build memory prompt (template errors fail the phase) - user_additions = None - if self._config.checkpoint is not None: - user_additions = render_memory_prompt(self._config.checkpoint, self._config_dir, context) - memory_prompt = self._checkpoint_manager.build_memory_prompt(user_additions) - - # 9. Call the agent for the summary — text-only (allowed_tools=[]) - await self._callbacks.fire_before_agent_send(1) - - response = await agent.send(memory_prompt, allowed_tools=[]) - total_input += response.usage.input_tokens - total_output += response.usage.output_tokens - - await self._callbacks.fire_agent_response(response, 1) - - # 10. Persist the memory file - self._checkpoint_manager.write_memory(self._phase_id, response.text) - - # 11. Write the success checkpoint (with memory_path and final token totals) - self._checkpoint_manager.write_phase_checkpoint( - self._phase_id, - { - "status": "success", - "started_at": self._started_at.isoformat(), - "finished_at": datetime.now(UTC).isoformat(), - "tokens": {"total_input": total_input, "total_output": total_output}, - "memory_path": str(self._checkpoint_manager.memory_path(self._phase_id)), + checkpoint_payload: dict[str, Any] = { + "status": "success", + "started_at": self._started_at.isoformat(), + "finished_at": datetime.now(UTC).isoformat(), + "tokens": { + "total_input": outcome.total_input_tokens, + "total_output": outcome.total_output_tokens, }, - ) + "memory_path": str(self._checkpoint_manager.memory_path(self._phase_id)), + } + reserved = set(checkpoint_payload) & set(outcome.extra_checkpoint) + if reserved: + raise ValueError( + f"Phase {self._phase_id!r}: extra_checkpoint cannot override reserved keys: {sorted(reserved)}" + ) + checkpoint_payload.update(outcome.extra_checkpoint) + + self._checkpoint_manager.write_memory(self._phase_id, outcome.memory_text) + self._checkpoint_manager.write_phase_checkpoint(self._phase_id, checkpoint_payload) await self._callbacks.fire_phase_finish(self._phase_id) async def on_success(self, message: PhaseTrigger) -> None: diff --git a/ddev/src/ddev/ai/phases/checkpoint.py b/ddev/src/ddev/ai/phases/checkpoint.py index 6951a7ff2b3ea..1a23b8ce63b7f 100644 --- a/ddev/src/ddev/ai/phases/checkpoint.py +++ b/ddev/src/ddev/ai/phases/checkpoint.py @@ -55,3 +55,9 @@ def memory_content(self, phase_id: str) -> str: """Return the contents of a phase's memory file, or a NOT FOUND placeholder.""" path = self.memory_path(phase_id) return path.read_text(encoding="utf-8") if path.exists() else f"" + + def resolve_template_variable(self, key: str) -> str: + """Resolve a template variable. ``_memory`` keys read the matching memory file.""" + if key.endswith("_memory"): + return self.memory_content(key.removesuffix("_memory")) + return f"" diff --git a/ddev/src/ddev/ai/phases/config.py b/ddev/src/ddev/ai/phases/config.py index 1f44cf7452634..5c2564e19066b 100644 --- a/ddev/src/ddev/ai/phases/config.py +++ b/ddev/src/ddev/ai/phases/config.py @@ -80,6 +80,7 @@ def exactly_one_source(self) -> CheckpointConfig: class AgentConfig(BaseModel): model_config = ConfigDict(extra="forbid") + provider: str = "anthropic" model: str | None = None max_tokens: int | None = None tools: list[str] = [] @@ -95,19 +96,12 @@ def tools_must_be_known(cls, tools: list[str]) -> list[str]: class PhaseConfig(BaseModel): model_config = ConfigDict(extra="forbid") - type: str = "Phase" - agent: str - tasks: list[TaskConfig] + type: str = "AgenticPhase" + agent: str | None = None + tasks: list[TaskConfig] = [] context_compact_threshold_pct: int = 80 checkpoint: CheckpointConfig | None = None - @field_validator("tasks", mode="after") - @classmethod - def at_least_one_task(cls, tasks: list[TaskConfig]) -> list[TaskConfig]: - if not tasks: - raise ValueError("A phase must have at least one task") - return tasks - class FlowEntry(BaseModel): model_config = ConfigDict(extra="forbid") @@ -140,7 +134,7 @@ def cross_references(self) -> FlowConfig: raise ValueError(f"Phase {entry.phase!r} depends on {dep!r} which is not scheduled in flow") for phase_id, phase in self.phases.items(): - if phase.agent not in self.agents: + if phase.agent is not None and phase.agent not in self.agents: raise ValueError(f"Phase {phase_id!r} references unknown agent: {phase.agent!r}") dependency_map = {entry.phase: entry.dependencies for entry in self.flow} diff --git a/ddev/src/ddev/ai/phases/orchestrator.py b/ddev/src/ddev/ai/phases/orchestrator.py index cfdc8851f3062..914d6aadf557e 100644 --- a/ddev/src/ddev/ai/phases/orchestrator.py +++ b/ddev/src/ddev/ai/phases/orchestrator.py @@ -6,8 +6,7 @@ import inspect import logging from pathlib import Path - -import anthropic +from typing import Any from ddev.ai.callbacks.callbacks import Callbacks from ddev.ai.phases.base import Phase, PhaseRegistry @@ -36,7 +35,7 @@ def _discover_and_register_phases( except Exception as e: raise FlowConfigError(f"Failed to import phase module '{py_file.stem}': {e}") from e for _, obj in inspect.getmembers(module, inspect.isclass): - if issubclass(obj, Phase) and obj.__module__ == module.__name__: + if issubclass(obj, Phase) and not inspect.isabstract(obj) and obj.__module__ == module.__name__: registry.register(obj.__name__, obj) @@ -46,7 +45,7 @@ def __init__( flow_yaml_path: Path, checkpoint_path: Path, runtime_variables: dict[str, str], - anthropic_client: anthropic.AsyncAnthropic, + agent_clients: dict[str, Any], file_access_policy: FileAccessPolicy, callbacks: Callbacks | None = None, grace_period: float = 10, @@ -54,6 +53,10 @@ def __init__( ) -> None: """Initialize the orchestrator. + ``agent_clients`` maps provider name (e.g. ``"anthropic"``) to a constructed + provider client. ``build_agent`` looks up the right one based on each + ``AgentConfig.provider``. + ``file_access_policy`` must have ``write_root`` set to the integration output directory so that agent writes are confined to that path. """ @@ -61,7 +64,7 @@ def __init__( self._flow_yaml_path = flow_yaml_path self._checkpoint_path = checkpoint_path self._runtime_variables = runtime_variables - self._anthropic_client = anthropic_client + self._agent_clients = agent_clients self._callbacks: Callbacks = callbacks or Callbacks() self._phase_registry = PhaseRegistry() self._failed_phase: str | None = None @@ -82,9 +85,10 @@ async def on_initialize(self) -> None: self._logger.warning("Phase %r is defined but not referenced in flow — it will not run", phase_id) continue try: - self._phase_registry.get(phase_config.type) + phase_cls = self._phase_registry.get(phase_config.type) except ValueError as e: raise FlowConfigError(str(e)) from e + phase_cls.validate_config(phase_id, phase_config, config.agents) checkpoint_manager = CheckpointManager(self._checkpoint_path) @@ -93,25 +97,33 @@ async def on_initialize(self) -> None: for entry in config.flow: phase_id = entry.phase phase_config = config.phases[phase_id] - agent_config = config.agents[phase_config.agent] dependencies = dependency_map[phase_id] phase_cls = self._phase_registry.get(phase_config.type) - phase = phase_cls( - phase_id=phase_id, - dependencies=dependencies, - config=phase_config, - agent_config=agent_config, - anthropic_client=self._anthropic_client, - checkpoint_manager=checkpoint_manager, - runtime_variables=self._runtime_variables, - flow_variables=config.variables, - config_dir=config_dir, - file_registry=self._file_registry, - callbacks=self._callbacks, - logger=self._logger, + phase_kwargs: dict[str, Any] = { + "phase_id": phase_id, + "dependencies": dependencies, + "config": phase_config, + "checkpoint_manager": checkpoint_manager, + "runtime_variables": self._runtime_variables, + "flow_variables": config.variables, + "config_dir": config_dir, + "file_registry": self._file_registry, + "callbacks": self._callbacks, + "logger": self._logger, + } + phase_kwargs.update( + phase_cls.extra_init_kwargs( + phase_id=phase_id, + phase_config=phase_config, + agents=config.agents, + agent_clients=self._agent_clients, + file_registry=self._file_registry, + ) ) + phase = phase_cls(**phase_kwargs) + self.register_processor(phase, [PhaseTrigger]) self.submit_message(PhaseTrigger(id="start", phase_id=None)) diff --git a/ddev/tests/ai/agent/test_build.py b/ddev/tests/ai/agent/test_build.py new file mode 100644 index 0000000000000..254e1530dc11b --- /dev/null +++ b/ddev/tests/ai/agent/test_build.py @@ -0,0 +1,76 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from unittest.mock import MagicMock + +import pytest + +from ddev.ai.agent.anthropic_client import AnthropicAgent +from ddev.ai.agent.build import build_agent, make_agent_builder +from ddev.ai.phases.config import AgentConfig +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.registry import ToolRegistry + + +@pytest.fixture +def file_registry(tmp_path) -> FileRegistry: + return FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) + + +def test_build_agent_anthropic_returns_agent_and_registry(file_registry): + agent_config = AgentConfig(provider="anthropic", model="claude-test", max_tokens=1024, tools=[]) + agent_clients = {"anthropic": MagicMock()} + + agent, registry = build_agent( + agent_config=agent_config, + agent_clients=agent_clients, + system_prompt="hello", + owner_id="p1", + file_registry=file_registry, + ) + + assert isinstance(agent, AnthropicAgent) + assert isinstance(registry, ToolRegistry) + assert agent.name == "p1" + + +def test_build_agent_missing_client_raises(file_registry): + agent_config = AgentConfig(provider="anthropic", tools=[]) + with pytest.raises(ValueError, match="No client provided for agent provider 'anthropic'"): + build_agent( + agent_config=agent_config, + agent_clients={}, + system_prompt="hello", + owner_id="p1", + file_registry=file_registry, + ) + + +def test_build_agent_unknown_provider_raises(file_registry): + agent_config = AgentConfig(provider="openai", tools=[]) + with pytest.raises(ValueError, match="Unknown agent provider: 'openai'"): + build_agent( + agent_config=agent_config, + agent_clients={"openai": MagicMock()}, + system_prompt="hello", + owner_id="p1", + file_registry=file_registry, + ) + + +def test_make_agent_builder_returns_callable_that_delegates_to_build_agent(file_registry): + agent_config = AgentConfig(provider="anthropic", tools=[]) + agent_clients = {"anthropic": MagicMock()} + + builder = make_agent_builder( + agent_config=agent_config, + agent_clients=agent_clients, + file_registry=file_registry, + ) + + agent, registry = builder("system prompt", "p2") + assert isinstance(agent, AnthropicAgent) + assert isinstance(registry, ToolRegistry) + assert agent.name == "p2" diff --git a/ddev/tests/ai/phases/conftest.py b/ddev/tests/ai/phases/conftest.py index 0174d2907350f..7f07c1734ffdc 100644 --- a/ddev/tests/ai/phases/conftest.py +++ b/ddev/tests/ai/phases/conftest.py @@ -8,7 +8,12 @@ import pytest from ddev.ai.agent.types import AgentResponse, ContextUsage, StopReason, TokenUsage, ToolResultMessage +from ddev.ai.phases.agentic_phase import AgenticPhase +from ddev.ai.phases.checkpoint import CheckpointManager +from ddev.ai.phases.config import PhaseConfig, TaskConfig from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.registry import ToolRegistry # --------------------------------------------------------------------------- # Helpers @@ -80,14 +85,66 @@ async def compact_preserving_last_turn(self) -> AgentResponse | None: return None -def make_agent_factory(mock_agent: MockAgent): - """Create a callable that replaces AnthropicAgent constructor, returning the given mock.""" +def make_agent_builder(mock_agent: MockAgent, captured_kwargs: dict[str, Any] | None = None): + """Create an agent_builder that returns the given mock and an empty ToolRegistry. - def factory(**kwargs: Any) -> MockAgent: - mock_agent.name = kwargs.get("name", "mock") - return mock_agent + If ``captured_kwargs`` is provided, every call records the system_prompt and + owner_id passed in — useful for asserting on prompt rendering. + """ - return factory + def builder(system_prompt: str, owner_id: str) -> tuple[MockAgent, ToolRegistry]: + if captured_kwargs is not None: + captured_kwargs["system_prompt"] = system_prompt + captured_kwargs["owner_id"] = owner_id + mock_agent.name = owner_id + return mock_agent, ToolRegistry([]) + + return builder + + +def make_agent_phase( + flow_dir, + mock_agent: MockAgent, + monkeypatch, + message_queue, + *, + phase_id: str = "p1", + dependencies: list[str] | None = None, + tasks: list[TaskConfig] | None = None, + checkpoint=None, + flow_variables: dict[str, str] | None = None, + runtime_variables: dict[str, str] | None = None, + context_compact_threshold_pct: int = 80, + callbacks=None, + captured_agent_kwargs: dict[str, Any] | None = None, +) -> tuple[AgenticPhase, CheckpointManager]: + """Build an AgenticPhase ready for process_message-driven tests. + + Injects a mock agent_builder so no real LLM or tools are constructed. Pass + ``captured_agent_kwargs`` (a dict) to record the rendered system_prompt and owner_id. + """ + config = PhaseConfig( + agent="writer", + tasks=tasks or [TaskConfig(name="t1", prompt="Do the work.")], + checkpoint=checkpoint, + context_compact_threshold_pct=context_compact_threshold_pct, + ) + checkpoint_manager = CheckpointManager(flow_dir / "checkpoints.yaml") + + phase = AgenticPhase( + phase_id=phase_id, + dependencies=dependencies or [], + config=config, + agent_builder=make_agent_builder(mock_agent, captured_agent_kwargs), + checkpoint_manager=checkpoint_manager, + runtime_variables=runtime_variables or {}, + flow_variables=flow_variables or {}, + config_dir=flow_dir, + file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), + callbacks=callbacks, + ) + phase.queue = message_queue + return phase, checkpoint_manager # --------------------------------------------------------------------------- diff --git a/ddev/tests/ai/phases/test_agentic_phase.py b/ddev/tests/ai/phases/test_agentic_phase.py new file mode 100644 index 0000000000000..e41188c46e8f2 --- /dev/null +++ b/ddev/tests/ai/phases/test_agentic_phase.py @@ -0,0 +1,502 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from pathlib import Path + +import pytest + +from ddev.ai.callbacks.callbacks import Callbacks, CallbackSet +from ddev.ai.phases.agentic_phase import AgenticPhase, render_memory_prompt, render_task_prompt +from ddev.ai.phases.config import AgentConfig, CheckpointConfig, FlowConfigError, PhaseConfig, TaskConfig +from ddev.ai.phases.messages import PhaseTrigger + +from .conftest import MockAgent, make_agent_phase, make_response, resolve_key + +# --------------------------------------------------------------------------- +# render_task_prompt +# --------------------------------------------------------------------------- + + +def test_render_task_prompt_from_file(tmp_path): + prompt_file = tmp_path / "task.md" + prompt_file.write_text("Hello ${name}.") + task = TaskConfig(name="t1", prompt_path="task.md") + result = render_task_prompt(task, tmp_path, {"name": "Alice"}) + assert result == "Hello Alice." + + +def test_render_task_prompt_inline(): + task = TaskConfig(name="t1", prompt="Hello ${name}.") + result = render_task_prompt(task, None, {"name": "Bob"}) + assert result == "Hello Bob." + + +def test_render_task_prompt_forwards_resolver(tmp_path): + prompt_file = tmp_path / "task.md" + prompt_file.write_text("Memory: ${draft_memory}") + task = TaskConfig(name="t1", prompt_path="task.md") + result = render_task_prompt(task, tmp_path, {}, resolve_key) + assert result == "Memory: resolved(draft_memory)" + + +def test_render_task_prompt_raises_when_both_unset(): + task = TaskConfig.model_construct(name="t1", prompt=None, prompt_path=None) + with pytest.raises(FlowConfigError, match="prompt"): + render_task_prompt(task, None, {}) + + +# --------------------------------------------------------------------------- +# render_memory_prompt +# --------------------------------------------------------------------------- + + +def test_render_memory_prompt_from_file(tmp_path): + mem_file = tmp_path / "mem.md" + mem_file.write_text("List files for ${phase_name}.") + checkpoint = CheckpointConfig(memory_prompt_path="mem.md") + result = render_memory_prompt(checkpoint, tmp_path, {"phase_name": "draft"}) + assert result == "List files for draft." + + +def test_render_memory_prompt_inline(): + checkpoint = CheckpointConfig(memory_prompt="List files for ${phase_name}.") + result = render_memory_prompt(checkpoint, None, {"phase_name": "draft"}) + assert result == "List files for draft." + + +def test_render_memory_prompt_raises_when_both_unset(): + checkpoint = CheckpointConfig.model_construct(memory_prompt=None, memory_prompt_path=None) + with pytest.raises(FlowConfigError, match="memory_prompt"): + render_memory_prompt(checkpoint, None, {}) + + +# --------------------------------------------------------------------------- +# AgenticPhase.validate_config +# --------------------------------------------------------------------------- + + +def test_agentic_phase_validate_config_rejects_missing_agent(): + config = PhaseConfig(tasks=[TaskConfig(name="t1", prompt="x")]) + with pytest.raises(FlowConfigError, match="requires 'agent'"): + AgenticPhase.validate_config("p1", config, {}) + + +def test_agentic_phase_validate_config_rejects_unknown_agent(): + config = PhaseConfig(agent="ghost", tasks=[TaskConfig(name="t1", prompt="x")]) + with pytest.raises(FlowConfigError, match="unknown agent"): + AgenticPhase.validate_config("p1", config, {"writer": AgentConfig()}) + + +def test_agentic_phase_validate_config_rejects_empty_tasks(): + config = PhaseConfig(agent="writer") + with pytest.raises(FlowConfigError, match="at least one task"): + AgenticPhase.validate_config("p1", config, {"writer": AgentConfig()}) + + +def test_agentic_phase_validate_config_accepts_valid(): + config = PhaseConfig(agent="writer", tasks=[TaskConfig(name="t1", prompt="x")]) + AgenticPhase.validate_config("p1", config, {"writer": AgentConfig()}) + + +# --------------------------------------------------------------------------- +# AgenticPhase.process_message — happy path +# --------------------------------------------------------------------------- + + +async def test_happy_path_single_task(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task done", 100, 50), # task 1 via ReActProcess + make_response("summary", 10, 5), # memory step + ] + mock_agent = MockAgent(responses) + phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.memory_content("p1") == "summary" + + checkpoint = mgr.read()["p1"] + assert checkpoint["status"] == "success" + assert checkpoint["tokens"]["total_input"] == 110 + assert checkpoint["tokens"]["total_output"] == 55 + assert checkpoint["memory_path"] + + assert len(mock_agent.send_calls) == 2 + assert mock_agent.send_calls[0] == "Do the work." + assert "Write a brief summary" in mock_agent.send_calls[1] + + +async def test_happy_path_two_tasks(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task1 done", 100, 50), + make_response("task2 done", 200, 80), + make_response("summary", 10, 5), + ] + mock_agent = MockAgent(responses) + phase, mgr = make_agent_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + tasks=[ + TaskConfig(name="t1", prompt="First task."), + TaskConfig(name="t2", prompt="Second task."), + ], + ) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + checkpoint = mgr.read()["p1"] + assert checkpoint["tokens"]["total_input"] == 310 + assert checkpoint["tokens"]["total_output"] == 135 + assert checkpoint["memory_path"] + + +# --------------------------------------------------------------------------- +# AgenticPhase.process_message — memory step with checkpoint config +# --------------------------------------------------------------------------- + + +async def test_memory_step_with_checkpoint_config(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task done", 100, 50), + make_response("summary with files", 10, 5), + ] + mock_agent = MockAgent(responses) + phase, mgr = make_agent_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + checkpoint=CheckpointConfig(memory_prompt="Also list the files."), + ) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + memory_prompt = mock_agent.send_calls[1] + assert "Also list the files." in memory_prompt + assert "Write a brief summary" in memory_prompt + + +async def test_memory_step_without_checkpoint_config(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task done", 100, 50), + make_response("summary", 10, 5), + ] + mock_agent = MockAgent(responses) + phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + memory_prompt = mock_agent.send_calls[1] + assert memory_prompt == "Write a brief summary of what you accomplished in this phase." + + +# --------------------------------------------------------------------------- +# AgenticPhase.process_message — context compaction between tasks +# --------------------------------------------------------------------------- + + +async def test_compact_between_tasks_when_above_threshold(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task1 done", 100, 50, context_pct=85), # above 80% threshold + make_response("task2 done", 200, 80), + make_response("summary", 10, 5), + ] + mock_agent = MockAgent(responses) + phase, mgr = make_agent_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + tasks=[ + TaskConfig(name="t1", prompt="First task."), + TaskConfig(name="t2", prompt="Second task."), + ], + ) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + checkpoint = mgr.read()["p1"] + assert checkpoint["status"] == "success" + assert checkpoint["memory_path"] + assert mock_agent.compact_call_count >= 1 + + +async def test_no_compact_when_below_threshold(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task1 done", 100, 50, context_pct=50), # below 80% threshold + make_response("task2 done", 200, 80), + make_response("summary", 10, 5), + ] + mock_agent = MockAgent(responses) + phase, mgr = make_agent_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + tasks=[ + TaskConfig(name="t1", prompt="First task."), + TaskConfig(name="t2", prompt="Second task."), + ], + ) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + checkpoint = mgr.read()["p1"] + assert checkpoint["status"] == "success" + assert checkpoint["memory_path"] + assert mock_agent.compact_call_count == 0 + + +# --------------------------------------------------------------------------- +# AgenticPhase.process_message — template context +# --------------------------------------------------------------------------- + + +async def test_flow_variables_in_system_prompt(flow_dir, monkeypatch, message_queue): + (flow_dir / "prompts" / "writer.md").write_text("Project: ${project}") + mock_agent = MockAgent([make_response("done", 100, 50), make_response("summary", 10, 5)]) + captured_kwargs: dict = {} + phase, _ = make_agent_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + flow_variables={"project": "myproj"}, + captured_agent_kwargs=captured_kwargs, + ) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert captured_kwargs["system_prompt"] == "Project: myproj" + + +async def test_runtime_variables_override_flow_variables(flow_dir, monkeypatch, message_queue): + (flow_dir / "prompts" / "writer.md").write_text("Project: ${project}") + mock_agent = MockAgent([make_response("done", 100, 50), make_response("summary", 10, 5)]) + captured_kwargs: dict = {} + phase, _ = make_agent_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + flow_variables={"project": "flow_default"}, + runtime_variables={"project": "runtime_override"}, + captured_agent_kwargs=captured_kwargs, + ) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert captured_kwargs["system_prompt"] == "Project: runtime_override" + + +# --------------------------------------------------------------------------- +# AgenticPhase.process_message — before_react / after_react errors +# --------------------------------------------------------------------------- + + +async def test_before_react_raises_propagates(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([]) + phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + def failing_hook(): + raise RuntimeError("setup failed") + + phase.before_react = failing_hook + + with pytest.raises(RuntimeError, match="setup failed"): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.read() == {} + + +async def test_after_react_raises_propagates(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("done", 100, 50), + ] + mock_agent = MockAgent(responses) + phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + def failing_hook(): + raise RuntimeError("teardown failed") + + phase.after_react = failing_hook + + with pytest.raises(RuntimeError, match="teardown failed"): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.read() == {} + + +# --------------------------------------------------------------------------- +# AgenticPhase.process_message — resolver integration with memory files +# --------------------------------------------------------------------------- + + +async def test_task_prompt_resolves_memory_variable(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([make_response("done", 100, 50), make_response("summary", 10, 5)]) + phase, mgr = make_agent_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + phase_id="review", + tasks=[TaskConfig(name="t1", prompt="Review: ${draft_memory}")], + ) + mgr.write_phase_checkpoint("draft", {"status": "success"}) + mgr.write_memory("draft", "Created file.py") + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mock_agent.send_calls[0] == "Review: Created file.py" + + +# --------------------------------------------------------------------------- +# AgenticPhase.process_message — memory step failure behaviour +# --------------------------------------------------------------------------- + + +async def test_memory_api_failure_fails_phase(flow_dir, monkeypatch, message_queue): + responses = [make_response("task done", 100, 50)] + mock_agent = MockAgent(responses) + phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + with pytest.raises(IndexError): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.read() == {} + + +async def test_memory_template_error_fails_phase(flow_dir, monkeypatch, message_queue): + responses = [make_response("task done", 100, 50)] + mock_agent = MockAgent(responses) + phase, mgr = make_agent_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + checkpoint=CheckpointConfig(memory_prompt="Summarize."), + ) + + def raise_render_error(*args, **kwargs): + raise ValueError("template error") + + monkeypatch.setattr("ddev.ai.phases.agentic_phase.render_memory_prompt", raise_render_error) + + with pytest.raises(ValueError, match="template error"): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.read() == {} + + +async def test_successful_phase_writes_memory_path_into_checkpoint(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task done", 100, 50), + make_response("summary text", 10, 5), + ] + mock_agent = MockAgent(responses) + phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + checkpoint = mgr.read()["p1"] + assert "memory_path" in checkpoint + memory_path = Path(checkpoint["memory_path"]) + assert memory_path.is_absolute() + assert memory_path.exists() + assert memory_path.name == "p1_memory.md" + assert memory_path.read_text() == "summary text" + + +# --------------------------------------------------------------------------- +# AgenticPhase._run_memory_step +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "checkpoint, expected_build_arg", + [ + (None, None), + (CheckpointConfig(memory_prompt="anything"), "USER_ADDITIONS"), + ], + ids=["no_checkpoint", "with_checkpoint"], +) +async def test_run_memory_step_forwards_user_additions_to_build( + flow_dir, monkeypatch, message_queue, checkpoint, expected_build_arg +): + mock_agent = MockAgent([make_response("ok", 0, 0)]) + phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue, checkpoint=checkpoint) + + monkeypatch.setattr("ddev.ai.phases.agentic_phase.render_memory_prompt", lambda *a, **kw: "USER_ADDITIONS") + build_calls: list = [] + monkeypatch.setattr( + mgr, "build_memory_prompt", lambda user_additions: build_calls.append(user_additions) or "PROMPT" + ) + + await phase._run_memory_step(mock_agent, {}) + + assert build_calls == [expected_build_arg] + + +async def test_run_memory_step_sends_built_prompt_with_no_tools(flow_dir, monkeypatch, message_queue): + captured: dict = {} + + class CapturingAgent(MockAgent): + async def send(self, content, allowed_tools=None): + captured["content"] = content + captured["allowed_tools"] = allowed_tools + return await super().send(content, allowed_tools) + + agent = CapturingAgent([make_response("ok", 0, 0)]) + phase, mgr = make_agent_phase(flow_dir, agent, monkeypatch, message_queue) + monkeypatch.setattr(mgr, "build_memory_prompt", lambda user_additions: "BUILT") + + await phase._run_memory_step(agent, {}) + + assert captured == {"content": "BUILT", "allowed_tools": []} + + +async def test_run_memory_step_returns_response_data_and_fires_callbacks(flow_dir, monkeypatch, message_queue): + events: list = [] + cb_set = CallbackSet() + + @cb_set.on_before_agent_send + async def _before(iteration): + events.append(("before", iteration)) + + @cb_set.on_agent_response + async def _response(response, iteration): + events.append(("response", iteration, response.text)) + + mock_agent = MockAgent([make_response("summary text", 7, 3)]) + phase, _ = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue, callbacks=Callbacks([cb_set])) + + result = await phase._run_memory_step(mock_agent, {}) + + assert result == ("summary text", 7, 3) + assert events == [("before", 1), ("response", 1, "summary text")] + + +# --------------------------------------------------------------------------- +# AgenticPhase.process_message — disk failure regression +# --------------------------------------------------------------------------- + + +async def test_write_memory_disk_failure_fails_phase(flow_dir, monkeypatch, message_queue): + responses = [ + make_response("task done", 100, 50), + make_response("summary text", 10, 5), + ] + mock_agent = MockAgent(responses) + phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + def raise_permission_error(*args, **kwargs): + raise PermissionError("disk is read-only") + + monkeypatch.setattr("ddev.ai.phases.checkpoint.CheckpointManager.write_memory", raise_permission_error) + + with pytest.raises(PermissionError, match="disk is read-only"): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.read() == {} diff --git a/ddev/tests/ai/phases/test_base.py b/ddev/tests/ai/phases/test_base.py index 683a1f4ba9e61..b776aa11448a1 100644 --- a/ddev/tests/ai/phases/test_base.py +++ b/ddev/tests/ai/phases/test_base.py @@ -3,433 +3,51 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from datetime import UTC, datetime -from pathlib import Path -from unittest.mock import MagicMock import pytest -from ddev.ai.phases.base import Phase, _make_memory_resolver, render_memory_prompt, render_task_prompt +from ddev.ai.phases.base import Phase, PhaseOutcome from ddev.ai.phases.checkpoint import CheckpointManager -from ddev.ai.phases.config import AgentConfig, CheckpointConfig, FlowConfigError, PhaseConfig, TaskConfig +from ddev.ai.phases.config import PhaseConfig from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy from ddev.ai.tools.fs.file_registry import FileRegistry -from ddev.ai.tools.registry import ToolRegistry from ddev.event_bus.exceptions import HookName, MessageProcessingError, ProcessorHookError -from .conftest import MockAgent, make_agent_factory, make_response, resolve_key +class _StubPhase(Phase): + """Concrete Phase for lifecycle tests; execute() returns a deterministic PhaseOutcome.""" -def _empty_registry_from_names(cls, names, *, owner_id, file_registry): - return ToolRegistry([]) + def __init__(self, *args, outcome: PhaseOutcome | None = None, **kwargs): + super().__init__(*args, **kwargs) + self._outcome = outcome or PhaseOutcome(memory_text="stub-memory") - -# --------------------------------------------------------------------------- -# _make_memory_resolver -# --------------------------------------------------------------------------- - - -def test_resolver_memory_suffix(tmp_path): - mgr = CheckpointManager(tmp_path / "checkpoints.yaml") - mgr.write_phase_checkpoint("x", {}) - mgr.write_memory("draft", "Draft memory content.") - resolver = _make_memory_resolver(mgr) - assert resolver("draft_memory") == "Draft memory content." - - -def test_resolver_non_memory_key(): - mgr = MagicMock() - resolver = _make_memory_resolver(mgr) - assert resolver("some_variable") == "" - mgr.memory_content.assert_not_called() - - -def test_resolver_absent_memory(tmp_path): - mgr = CheckpointManager(tmp_path / "checkpoints.yaml") - resolver = _make_memory_resolver(mgr) - assert resolver("nonexistent_memory") == "" - - -# --------------------------------------------------------------------------- -# render_task_prompt -# --------------------------------------------------------------------------- - - -def test_render_task_prompt_from_file(tmp_path): - prompt_file = tmp_path / "task.md" - prompt_file.write_text("Hello ${name}.") - task = TaskConfig(name="t1", prompt_path="task.md") - result = render_task_prompt(task, tmp_path, {"name": "Alice"}) - assert result == "Hello Alice." - - -def test_render_task_prompt_inline(): - task = TaskConfig(name="t1", prompt="Hello ${name}.") - result = render_task_prompt(task, None, {"name": "Bob"}) - assert result == "Hello Bob." - - -def test_render_task_prompt_forwards_resolver(tmp_path): - prompt_file = tmp_path / "task.md" - prompt_file.write_text("Memory: ${draft_memory}") - task = TaskConfig(name="t1", prompt_path="task.md") - result = render_task_prompt(task, tmp_path, {}, resolve_key) - assert result == "Memory: resolved(draft_memory)" - - -# --------------------------------------------------------------------------- -# render_memory_prompt -# --------------------------------------------------------------------------- + async def execute(self, context): + return self._outcome -def test_render_memory_prompt_from_file(tmp_path): - mem_file = tmp_path / "mem.md" - mem_file.write_text("List files for ${phase_name}.") - checkpoint = CheckpointConfig(memory_prompt_path="mem.md") - result = render_memory_prompt(checkpoint, tmp_path, {"phase_name": "draft"}) - assert result == "List files for draft." - - -def test_render_memory_prompt_inline(): - checkpoint = CheckpointConfig(memory_prompt="List files for ${phase_name}.") - result = render_memory_prompt(checkpoint, None, {"phase_name": "draft"}) - assert result == "List files for draft." - - -def test_render_task_prompt_raises_when_both_unset(): - task = TaskConfig.model_construct(name="t1", prompt=None, prompt_path=None) - with pytest.raises(FlowConfigError, match="prompt"): - render_task_prompt(task, None, {}) - - -def test_render_memory_prompt_raises_when_both_unset(): - checkpoint = CheckpointConfig.model_construct(memory_prompt=None, memory_prompt_path=None) - with pytest.raises(FlowConfigError, match="memory_prompt"): - render_memory_prompt(checkpoint, None, {}) - - -# --------------------------------------------------------------------------- -# Phase helpers -# --------------------------------------------------------------------------- - - -def _make_phase( +def _make_stub_phase( flow_dir, - mock_agent, - monkeypatch, message_queue, *, phase_id="p1", dependencies=None, - tasks=None, - checkpoint=None, - agent_tools=None, - flow_variables=None, - runtime_variables=None, - context_compact_threshold_pct=80, + outcome=None, ): - monkeypatch.setattr("ddev.ai.phases.base.AnthropicAgent", make_agent_factory(mock_agent)) - monkeypatch.setattr(ToolRegistry, "from_names", classmethod(_empty_registry_from_names)) - - config = PhaseConfig( - agent="writer", - tasks=tasks or [TaskConfig(name="t1", prompt="Do the work.")], - checkpoint=checkpoint, - context_compact_threshold_pct=context_compact_threshold_pct, - ) - agent_config = AgentConfig(tools=agent_tools or []) checkpoint_manager = CheckpointManager(flow_dir / "checkpoints.yaml") - - phase = Phase( + phase = _StubPhase( phase_id=phase_id, dependencies=dependencies or [], - config=config, - agent_config=agent_config, - anthropic_client=MagicMock(), + config=PhaseConfig(), checkpoint_manager=checkpoint_manager, - runtime_variables=runtime_variables or {}, - flow_variables=flow_variables or {}, - config_dir=flow_dir, - file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), - callbacks=None, - ) - phase.queue = message_queue - return phase, checkpoint_manager - - -# --------------------------------------------------------------------------- -# Phase.process_message — happy path -# --------------------------------------------------------------------------- - - -async def test_happy_path_single_task(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task done", 100, 50), # task 1 via ReActProcess - make_response("summary", 10, 5), # memory step - ] - mock_agent = MockAgent(responses) - phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) - - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - # Memory was written - assert mgr.memory_content("p1") == "summary" - - # Checkpoint was written with memory_path and final token totals (including memory step) - checkpoint = mgr.read()["p1"] - assert checkpoint["status"] == "success" - assert checkpoint["tokens"]["total_input"] == 110 - assert checkpoint["tokens"]["total_output"] == 55 - assert checkpoint["memory_path"] # non-empty string - - # on_success is called by _task_wrapper, not process_message directly. - # But we verify it would work by checking the send calls. - assert len(mock_agent.send_calls) == 2 - assert mock_agent.send_calls[0] == "Do the work." - assert "Write a brief summary" in mock_agent.send_calls[1] - - -async def test_happy_path_two_tasks(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task1 done", 100, 50), - make_response("task2 done", 200, 80), - make_response("summary", 10, 5), - ] - mock_agent = MockAgent(responses) - phase, mgr = _make_phase( - flow_dir, - mock_agent, - monkeypatch, - message_queue, - tasks=[ - TaskConfig(name="t1", prompt="First task."), - TaskConfig(name="t2", prompt="Second task."), - ], - ) - - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - checkpoint = mgr.read()["p1"] - assert checkpoint["tokens"]["total_input"] == 310 - assert checkpoint["tokens"]["total_output"] == 135 - assert checkpoint["memory_path"] - - -# --------------------------------------------------------------------------- -# Phase.process_message — memory step with checkpoint config -# --------------------------------------------------------------------------- - - -async def test_memory_step_with_checkpoint_config(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task done", 100, 50), - make_response("summary with files", 10, 5), - ] - mock_agent = MockAgent(responses) - phase, mgr = _make_phase( - flow_dir, - mock_agent, - monkeypatch, - message_queue, - checkpoint=CheckpointConfig(memory_prompt="Also list the files."), - ) - - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - # Memory prompt should include user additions - memory_prompt = mock_agent.send_calls[1] - assert "Also list the files." in memory_prompt - assert "Write a brief summary" in memory_prompt - - -async def test_memory_step_without_checkpoint_config(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task done", 100, 50), - make_response("summary", 10, 5), - ] - mock_agent = MockAgent(responses) - phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) - - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - memory_prompt = mock_agent.send_calls[1] - assert memory_prompt == "Write a brief summary of what you accomplished in this phase." - - -# --------------------------------------------------------------------------- -# Phase.process_message — context compaction between tasks -# --------------------------------------------------------------------------- - - -async def test_compact_between_tasks_when_above_threshold(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task1 done", 100, 50, context_pct=85), # above 80% threshold - make_response("task2 done", 200, 80), - make_response("summary", 10, 5), - ] - mock_agent = MockAgent(responses) - phase, mgr = _make_phase( - flow_dir, - mock_agent, - monkeypatch, - message_queue, - tasks=[ - TaskConfig(name="t1", prompt="First task."), - TaskConfig(name="t2", prompt="Second task."), - ], - ) - - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - checkpoint = mgr.read()["p1"] - assert checkpoint["status"] == "success" - assert checkpoint["memory_path"] - assert mock_agent.compact_call_count >= 1 - - -async def test_no_compact_when_below_threshold(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task1 done", 100, 50, context_pct=50), # below 80% threshold - make_response("task2 done", 200, 80), - make_response("summary", 10, 5), - ] - mock_agent = MockAgent(responses) - phase, mgr = _make_phase( - flow_dir, - mock_agent, - monkeypatch, - message_queue, - tasks=[ - TaskConfig(name="t1", prompt="First task."), - TaskConfig(name="t2", prompt="Second task."), - ], - ) - - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - checkpoint = mgr.read()["p1"] - assert checkpoint["status"] == "success" - assert checkpoint["memory_path"] - assert mock_agent.compact_call_count == 0 - - -# --------------------------------------------------------------------------- -# Phase.process_message — template context -# --------------------------------------------------------------------------- - - -async def test_flow_variables_in_system_prompt(flow_dir, monkeypatch, message_queue): - # System prompt references ${project} - (flow_dir / "prompts" / "writer.md").write_text("Project: ${project}") - responses = [ - make_response("done", 100, 50), - make_response("summary", 10, 5), - ] - mock_agent = MockAgent(responses) - captured_kwargs = {} - original_factory = make_agent_factory(mock_agent) - - def capturing_factory(**kwargs): - captured_kwargs.update(kwargs) - return original_factory(**kwargs) - - monkeypatch.setattr("ddev.ai.phases.base.AnthropicAgent", capturing_factory) - monkeypatch.setattr(ToolRegistry, "from_names", classmethod(_empty_registry_from_names)) - - config = PhaseConfig( - agent="writer", - tasks=[TaskConfig(name="t1", prompt="Do it.")], - ) - phase = Phase( - phase_id="p1", - dependencies=[], - config=config, - agent_config=AgentConfig(), - anthropic_client=MagicMock(), - checkpoint_manager=CheckpointManager(flow_dir / "checkpoints.yaml"), runtime_variables={}, - flow_variables={"project": "myproj"}, - config_dir=flow_dir, - file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), - ) - phase.queue = message_queue - - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - assert "Project: myproj" == captured_kwargs["system_prompt"] - - -async def test_runtime_variables_override_flow_variables(flow_dir, monkeypatch, message_queue): - (flow_dir / "prompts" / "writer.md").write_text("Project: ${project}") - responses = [ - make_response("done", 100, 50), - make_response("summary", 10, 5), - ] - mock_agent = MockAgent(responses) - captured_kwargs = {} - original_factory = make_agent_factory(mock_agent) - - def capturing_factory(**kwargs): - captured_kwargs.update(kwargs) - return original_factory(**kwargs) - - monkeypatch.setattr("ddev.ai.phases.base.AnthropicAgent", capturing_factory) - monkeypatch.setattr(ToolRegistry, "from_names", classmethod(_empty_registry_from_names)) - - config = PhaseConfig( - agent="writer", - tasks=[TaskConfig(name="t1", prompt="Do it.")], - ) - phase = Phase( - phase_id="p1", - dependencies=[], - config=config, - agent_config=AgentConfig(), - anthropic_client=MagicMock(), - checkpoint_manager=CheckpointManager(flow_dir / "checkpoints.yaml"), - runtime_variables={"project": "runtime_override"}, - flow_variables={"project": "flow_default"}, + flow_variables={}, config_dir=flow_dir, file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), + outcome=outcome, ) phase.queue = message_queue - - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - assert captured_kwargs["system_prompt"] == "Project: runtime_override" - - -# --------------------------------------------------------------------------- -# Phase.process_message — before_react / after_react errors -# --------------------------------------------------------------------------- - - -async def test_before_react_raises_propagates(flow_dir, monkeypatch, message_queue): - mock_agent = MockAgent([]) - phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) - - def failing_hook(): - raise RuntimeError("setup failed") - - phase.before_react = failing_hook - - with pytest.raises(RuntimeError, match="setup failed"): - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - -async def test_after_react_raises_propagates(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("done", 100, 50), - ] - mock_agent = MockAgent(responses) - phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) - - def failing_hook(): - raise RuntimeError("teardown failed") - - phase.after_react = failing_hook - - with pytest.raises(RuntimeError, match="teardown failed"): - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + return phase, checkpoint_manager # --------------------------------------------------------------------------- @@ -437,9 +55,8 @@ def failing_hook(): # --------------------------------------------------------------------------- -async def test_on_success_emits_finished_message(flow_dir, monkeypatch, message_queue): - mock_agent = MockAgent([]) - phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) +async def test_on_success_emits_finished_message(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue) await phase.on_success(PhaseTrigger(id="start", phase_id=None)) @@ -454,9 +71,8 @@ async def test_on_success_emits_finished_message(flow_dir, monkeypatch, message_ # --------------------------------------------------------------------------- -async def test_on_error_writes_failed_checkpoint(flow_dir, monkeypatch, message_queue): - mock_agent = MockAgent([]) - phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) +async def test_on_error_writes_failed_checkpoint(flow_dir, message_queue): + phase, mgr = _make_stub_phase(flow_dir, message_queue) wrapped = MessageProcessingError("p1", PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) await phase.on_error(wrapped) @@ -467,9 +83,8 @@ async def test_on_error_writes_failed_checkpoint(flow_dir, monkeypatch, message_ assert checkpoint["started_at"] is None # not started yet -async def test_on_error_emits_failed_message(flow_dir, monkeypatch, message_queue): - mock_agent = MockAgent([]) - phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) +async def test_on_error_emits_failed_message(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue) wrapped = ProcessorHookError( HookName.ON_SUCCESS, "p1", PhaseTrigger(id="start", phase_id=None), RuntimeError("boom") @@ -482,9 +97,8 @@ async def test_on_error_emits_failed_message(flow_dir, monkeypatch, message_queu assert msg.error == "boom" -async def test_on_error_writes_failed_checkpoint_after_start(flow_dir, monkeypatch, message_queue): - mock_agent = MockAgent([]) - phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) +async def test_on_error_writes_failed_checkpoint_after_start(flow_dir, message_queue): + phase, mgr = _make_stub_phase(flow_dir, message_queue) phase._started_at = datetime.now(UTC) wrapped = MessageProcessingError("p1", PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) @@ -495,58 +109,13 @@ async def test_on_error_writes_failed_checkpoint_after_start(flow_dir, monkeypat assert checkpoint["started_at"] is not None -# --------------------------------------------------------------------------- -# Phase.process_message — resolver integration with memory files -# --------------------------------------------------------------------------- - - -async def test_task_prompt_resolves_memory_variable(flow_dir, monkeypatch, message_queue): - # Create a memory file for "draft" phase - mgr = CheckpointManager(flow_dir / "checkpoints.yaml") - mgr.write_phase_checkpoint("draft", {"status": "success"}) - mgr.write_memory("draft", "Created file.py") - - # Task prompt references ${draft_memory} - responses = [ - make_response("done", 100, 50), - make_response("summary", 10, 5), - ] - mock_agent = MockAgent(responses) - - monkeypatch.setattr("ddev.ai.phases.base.AnthropicAgent", make_agent_factory(mock_agent)) - monkeypatch.setattr(ToolRegistry, "from_names", classmethod(_empty_registry_from_names)) - - config = PhaseConfig( - agent="writer", - tasks=[TaskConfig(name="t1", prompt="Review: ${draft_memory}")], - ) - phase = Phase( - phase_id="review", - dependencies=[], - config=config, - agent_config=AgentConfig(), - anthropic_client=MagicMock(), - checkpoint_manager=mgr, - runtime_variables={}, - flow_variables={}, - config_dir=flow_dir, - file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), - ) - phase.queue = message_queue - - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - assert mock_agent.send_calls[0] == "Review: Created file.py" - - # --------------------------------------------------------------------------- # Phase.should_process_message # --------------------------------------------------------------------------- -def test_should_process_returns_true_for_initial_trigger_on_root_phase(flow_dir, monkeypatch, message_queue): - mock_agent = MockAgent([]) - phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) +def test_should_process_returns_true_for_initial_trigger_on_root_phase(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue) result = phase.should_process_message(PhaseTrigger(id="start", phase_id=None)) @@ -554,9 +123,8 @@ def test_should_process_returns_true_for_initial_trigger_on_root_phase(flow_dir, assert phase._executed is True -def test_should_process_returns_false_for_initial_trigger_on_dependent_phase(flow_dir, monkeypatch, message_queue): - mock_agent = MockAgent([]) - phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue, dependencies=["dep1"]) +def test_should_process_returns_false_for_initial_trigger_on_dependent_phase(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue, dependencies=["dep1"]) result = phase.should_process_message(PhaseTrigger(id="start", phase_id=None)) @@ -564,9 +132,8 @@ def test_should_process_returns_false_for_initial_trigger_on_dependent_phase(flo assert phase._executed is False -def test_should_process_returns_false_for_unrelated_dep(flow_dir, monkeypatch, message_queue): - mock_agent = MockAgent([]) - phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue, dependencies=["dep1"]) +def test_should_process_returns_false_for_unrelated_dep(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue, dependencies=["dep1"]) result = phase.should_process_message(PhaseTrigger(id="msg1", phase_id="other")) @@ -574,9 +141,8 @@ def test_should_process_returns_false_for_unrelated_dep(flow_dir, monkeypatch, m assert phase._executed is False -def test_should_process_returns_false_while_deps_pending(flow_dir, monkeypatch, message_queue): - mock_agent = MockAgent([]) - phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue, dependencies=["dep1", "dep2"]) +def test_should_process_returns_false_while_deps_pending(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue, dependencies=["dep1", "dep2"]) result = phase.should_process_message(PhaseTrigger(id="msg1", phase_id="dep1")) @@ -585,9 +151,8 @@ def test_should_process_returns_false_while_deps_pending(flow_dir, monkeypatch, assert phase._executed is False -def test_should_process_returns_true_when_last_dep_arrives(flow_dir, monkeypatch, message_queue): - mock_agent = MockAgent([]) - phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue, dependencies=["dep1", "dep2"]) +def test_should_process_returns_true_when_last_dep_arrives(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue, dependencies=["dep1", "dep2"]) phase.should_process_message(PhaseTrigger(id="msg1", phase_id="dep1")) result = phase.should_process_message(PhaseTrigger(id="msg2", phase_id="dep2")) @@ -596,9 +161,8 @@ def test_should_process_returns_true_when_last_dep_arrives(flow_dir, monkeypatch assert phase._executed is True -def test_should_process_returns_false_after_already_executed(flow_dir, monkeypatch, message_queue): - mock_agent = MockAgent([]) - phase, _ = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) +def test_should_process_returns_false_after_already_executed(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue) phase.should_process_message(PhaseTrigger(id="start", phase_id=None)) result = phase.should_process_message(PhaseTrigger(id="start2", phase_id=None)) @@ -607,89 +171,56 @@ def test_should_process_returns_false_after_already_executed(flow_dir, monkeypat # --------------------------------------------------------------------------- -# Phase.process_message — memory step failure behaviour +# Phase lifecycle — memory path # --------------------------------------------------------------------------- -async def test_memory_api_failure_fails_phase(flow_dir, monkeypatch, message_queue): - # Only 1 response provided; second send (memory step) raises IndexError. - responses = [make_response("task done", 100, 50)] - mock_agent = MockAgent(responses) - phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) - - with pytest.raises(IndexError): - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - # Checkpoint must not have been written (exception before checkpoint write) - assert mgr.read() == {} - - -async def test_memory_template_error_fails_phase(flow_dir, monkeypatch, message_queue): - responses = [make_response("task done", 100, 50)] - mock_agent = MockAgent(responses) - phase, mgr = _make_phase( - flow_dir, - mock_agent, - monkeypatch, - message_queue, - checkpoint=CheckpointConfig(memory_prompt="Summarize."), +async def test_process_message_writes_memory_and_checkpoint(flow_dir, message_queue): + """End-to-end Phase contract: memory_text is persisted, extra_checkpoint merges, + token totals land in the checkpoint, and the success metadata is recorded. + """ + outcome = PhaseOutcome( + memory_text="stub-memory-body", + total_input_tokens=123, + total_output_tokens=45, + extra_checkpoint={"custom_field": "custom_value", "count": 7}, ) + phase, mgr = _make_stub_phase(flow_dir, message_queue, outcome=outcome) - def raise_render_error(*args, **kwargs): - raise ValueError("template error") + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - monkeypatch.setattr("ddev.ai.phases.base.render_memory_prompt", raise_render_error) + assert mgr.memory_content("p1") == "stub-memory-body" - with pytest.raises(ValueError, match="template error"): + checkpoint = mgr.read()["p1"] + assert checkpoint["status"] == "success" + assert checkpoint["tokens"] == {"total_input": 123, "total_output": 45} + assert checkpoint["memory_path"] == str(mgr.memory_path("p1")) + assert checkpoint["custom_field"] == "custom_value" + assert checkpoint["count"] == 7 + assert checkpoint["started_at"] + assert checkpoint["finished_at"] + + +@pytest.mark.parametrize( + "reserved_key", + ["status", "started_at", "finished_at", "tokens", "memory_path"], +) +async def test_extra_checkpoint_cannot_override_reserved_keys(flow_dir, message_queue, reserved_key): + outcome = PhaseOutcome(memory_text="m", extra_checkpoint={reserved_key: "evil"}) + phase, mgr = _make_stub_phase(flow_dir, message_queue, outcome=outcome) + + with pytest.raises(ValueError, match=f"reserved keys.*{reserved_key}"): await phase.process_message(PhaseTrigger(id="start", phase_id=None)) assert mgr.read() == {} + assert not mgr.memory_path("p1").exists() -async def test_successful_phase_writes_memory_path_into_checkpoint(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task done", 100, 50), - make_response("summary text", 10, 5), - ] - mock_agent = MockAgent(responses) - phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) - - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - checkpoint = mgr.read()["p1"] - assert "memory_path" in checkpoint - memory_path = Path(checkpoint["memory_path"]) - assert memory_path.is_absolute() - assert memory_path.exists() - assert memory_path.name == "p1_memory.md" - assert memory_path.read_text() == "summary text" - - -async def test_failed_phase_omits_memory_path(flow_dir, monkeypatch, message_queue): - mock_agent = MockAgent([]) - phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) +async def test_failed_phase_omits_memory_path(flow_dir, message_queue): + phase, mgr = _make_stub_phase(flow_dir, message_queue) wrapped = MessageProcessingError("p1", PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) await phase.on_error(wrapped) checkpoint = mgr.read()["p1"] assert "memory_path" not in checkpoint - - -async def test_write_memory_disk_failure_fails_phase(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task done", 100, 50), - make_response("summary text", 10, 5), - ] - mock_agent = MockAgent(responses) - phase, mgr = _make_phase(flow_dir, mock_agent, monkeypatch, message_queue) - - def raise_permission_error(*args, **kwargs): - raise PermissionError("disk is read-only") - - monkeypatch.setattr("ddev.ai.phases.checkpoint.CheckpointManager.write_memory", raise_permission_error) - - with pytest.raises(PermissionError, match="disk is read-only"): - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - assert mgr.read() == {} diff --git a/ddev/tests/ai/phases/test_checkpoint.py b/ddev/tests/ai/phases/test_checkpoint.py index 571ea642065da..41ec010e3481f 100644 --- a/ddev/tests/ai/phases/test_checkpoint.py +++ b/ddev/tests/ai/phases/test_checkpoint.py @@ -127,3 +127,17 @@ def test_memory_file_location(manager): assert expected_path.exists() assert expected_path.read_text() == "content" assert manager.memory_path("phase1") == expected_path.resolve() + + +# --------------------------------------------------------------------------- +# resolve_template_variable +# --------------------------------------------------------------------------- + + +def test_resolve_template_variable_memory_suffix(manager): + manager.write_memory("draft", "Draft memory content.") + assert manager.resolve_template_variable("draft_memory") == "Draft memory content." + + +def test_resolve_template_variable_non_memory_key(manager): + assert manager.resolve_template_variable("some_variable") == "" diff --git a/ddev/tests/ai/phases/test_config.py b/ddev/tests/ai/phases/test_config.py index 82770b723b892..79deda7b989b5 100644 --- a/ddev/tests/ai/phases/test_config.py +++ b/ddev/tests/ai/phases/test_config.py @@ -104,16 +104,11 @@ def test_agent_config_optional_fields(): def test_phase_config_defaults(): pc = PhaseConfig(agent="writer", tasks=[TaskConfig(name="t1", prompt="Do it.")]) - assert pc.type == "Phase" + assert pc.type == "AgenticPhase" assert pc.context_compact_threshold_pct == 80 assert pc.checkpoint is None -def test_phase_config_empty_tasks_raises(): - with pytest.raises(ValidationError, match="at least one task"): - PhaseConfig(agent="writer", tasks=[]) - - def test_phase_config_with_checkpoint(): pc = PhaseConfig( agent="writer", @@ -184,6 +179,23 @@ def test_flow_config_unknown_agent_in_phase(): FlowConfig.model_validate(raw) +def test_flow_config_phase_without_agent_validates(): + raw = { + "agents": {"writer": {"tools": []}}, + "phases": { + "p1": {"agent": "writer", "tasks": [{"name": "t1", "prompt": "Do it."}]}, + "noop": {"type": "SomeCustomPhase"}, + }, + "flow": [ + {"phase": "p1"}, + {"phase": "noop", "dependencies": ["p1"]}, + ], + } + config = FlowConfig.model_validate(raw) + assert config.phases["noop"].agent is None + assert config.phases["noop"].tasks == [] + + def test_flow_config_with_variables(): raw = _minimal_config(variables={"project": "myproj"}) config = FlowConfig.model_validate(raw) diff --git a/ddev/tests/ai/phases/test_orchestrator.py b/ddev/tests/ai/phases/test_orchestrator.py index b36700a7cd121..85c9758a95d1b 100644 --- a/ddev/tests/ai/phases/test_orchestrator.py +++ b/ddev/tests/ai/phases/test_orchestrator.py @@ -2,28 +2,55 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import logging from pathlib import Path from textwrap import dedent +from typing import Any from unittest.mock import MagicMock import pytest +from ddev.ai.phases.agentic_phase import AgenticPhase from ddev.ai.phases.base import Phase, PhaseRegistry from ddev.ai.phases.config import FlowConfigError from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger from ddev.ai.phases.orchestrator import PhaseOrchestrator, _discover_and_register_phases from ddev.event_bus.exceptions import FatalProcessingError + +@pytest.fixture +def make_orchestrator(file_access_policy): + """Factory that builds a PhaseOrchestrator with test defaults. + + Pass a ``base_dir`` to anchor ``flow.yaml`` / ``checkpoints.yaml`` (defaults to + ``/fake`` for tests that never touch disk). Any constructor kwarg can be overridden. + """ + + def _make(base_dir: Path | None = None, **overrides: Any) -> PhaseOrchestrator: + base_dir = base_dir if base_dir is not None else Path("/fake") + kwargs: dict[str, Any] = { + "flow_yaml_path": base_dir / "flow.yaml", + "checkpoint_path": base_dir / "checkpoints.yaml", + "runtime_variables": {}, + "agent_clients": {"anthropic": MagicMock()}, + "file_access_policy": file_access_policy, + **overrides, + } + return PhaseOrchestrator(**kwargs) + + return _make + + # --------------------------------------------------------------------------- # _discover_and_register_phases # --------------------------------------------------------------------------- -def test_discover_registers_phase_itself(): +def test_discover_registers_agentic_phase(): registry = PhaseRegistry() _discover_and_register_phases(registry) - assert "Phase" in registry.known_names() - assert registry.get("Phase") is Phase + assert "AgenticPhase" in registry.known_names() + assert registry.get("AgenticPhase") is AgenticPhase def test_discover_registers_custom_subclass(tmp_path, monkeypatch): @@ -31,14 +58,16 @@ def test_discover_registers_custom_subclass(tmp_path, monkeypatch): fake_dir = tmp_path / "fake_phases" fake_dir.mkdir() (fake_dir / "__init__.py").write_text("") - (fake_dir / "custom.py").write_text("from ddev.ai.phases.base import Phase\nclass CustomPhase(Phase):\n pass\n") + (fake_dir / "custom.py").write_text( + "from ddev.ai.phases.agentic_phase import AgenticPhase\nclass CustomPhase(AgenticPhase):\n pass\n" + ) monkeypatch.syspath_prepend(str(tmp_path)) registry = PhaseRegistry() _discover_and_register_phases(registry, phases_dir=fake_dir, import_prefix="fake_phases") assert "CustomPhase" in registry.known_names() - assert issubclass(registry.get("CustomPhase"), Phase) + assert issubclass(registry.get("CustomPhase"), AgenticPhase) def test_discover_ignores_module_without_phase_subclass(tmp_path, monkeypatch): @@ -59,21 +88,33 @@ def test_discover_does_not_register_imported_phase_class(tmp_path, monkeypatch): fake_dir = tmp_path / "importer_pkg" fake_dir.mkdir() (fake_dir / "__init__.py").write_text("") - (fake_dir / "importer.py").write_text("from ddev.ai.phases.base import Phase\n") + (fake_dir / "importer.py").write_text("from ddev.ai.phases.agentic_phase import AgenticPhase\n") monkeypatch.syspath_prepend(str(tmp_path)) registry = PhaseRegistry() _discover_and_register_phases(registry, phases_dir=fake_dir, import_prefix="importer_pkg") - assert "Phase" not in registry.known_names() + assert "AgenticPhase" not in registry.known_names() -def test_discover_skips_underscore_prefixed_files(): - """After discovery, only non-underscore files are imported. - __init__.py is underscore-prefixed and is skipped.""" +def test_discover_skips_underscore_prefixed_files(tmp_path, monkeypatch): + """Classes defined in underscore-prefixed files (e.g. _private.py) are never registered.""" + fake_dir = tmp_path / "underscore_pkg" + fake_dir.mkdir() + (fake_dir / "__init__.py").write_text("") + (fake_dir / "_private.py").write_text( + "from ddev.ai.phases.agentic_phase import AgenticPhase\nclass PrivatePhase(AgenticPhase):\n pass\n" + ) + (fake_dir / "public.py").write_text( + "from ddev.ai.phases.agentic_phase import AgenticPhase\nclass PublicPhase(AgenticPhase):\n pass\n" + ) + monkeypatch.syspath_prepend(str(tmp_path)) + registry = PhaseRegistry() - _discover_and_register_phases(registry) - assert "Phase" in registry.known_names() + _discover_and_register_phases(registry, phases_dir=fake_dir, import_prefix="underscore_pkg") + + assert "PrivatePhase" not in registry.known_names() + assert "PublicPhase" in registry.known_names() def test_discover_idempotent(): @@ -99,22 +140,10 @@ def test_imported_class_not_registered(): assert "BaseMessage" not in registry.known_names() -def test_two_orchestrators_have_independent_registries(tmp_path, file_access_policy): +def test_two_orchestrators_have_independent_registries(tmp_path, make_orchestrator): """Each PhaseOrchestrator owns its own registry; registering in one does not affect the other.""" - o1 = PhaseOrchestrator( - flow_yaml_path=tmp_path / "flow.yaml", - checkpoint_path=tmp_path / "checkpoints.yaml", - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - ) - o2 = PhaseOrchestrator( - flow_yaml_path=tmp_path / "flow.yaml", - checkpoint_path=tmp_path / "checkpoints.yaml", - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - ) + o1 = make_orchestrator(tmp_path) + o2 = make_orchestrator(tmp_path) class ExclusivePhase(Phase): pass @@ -138,28 +167,16 @@ def test_discover_does_not_mutate_global_state(): # --------------------------------------------------------------------------- -async def test_on_message_received_fatal_on_phase_failed(file_access_policy): - orchestrator = PhaseOrchestrator( - flow_yaml_path=Path("/fake/flow.yaml"), - checkpoint_path=Path("/fake/checkpoints.yaml"), - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - ) +async def test_on_message_received_fatal_on_phase_failed(make_orchestrator): + orchestrator = make_orchestrator() msg = PhaseFailedMessage(id="f1", phase_id="p1", error="something broke") with pytest.raises(FatalProcessingError, match="Phase 'p1' failed"): await orchestrator.on_message_received(msg) -async def test_on_message_received_ignores_other_messages(file_access_policy): - orchestrator = PhaseOrchestrator( - flow_yaml_path=Path("/fake/flow.yaml"), - checkpoint_path=Path("/fake/checkpoints.yaml"), - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - ) +async def test_on_message_received_ignores_other_messages(make_orchestrator): + orchestrator = make_orchestrator() # These should not raise await orchestrator.on_message_received(PhaseTrigger(id="start", phase_id=None)) await orchestrator.on_message_received(PhaseTrigger(id="f1", phase_id="p1")) @@ -182,13 +199,11 @@ def minimal_flow(tmp_path): tools: [] phases: a: - type: Phase agent: writer tasks: - name: task_a prompt: task a b: - type: Phase agent: writer tasks: - name: task_b @@ -202,14 +217,8 @@ def minimal_flow(tmp_path): return tmp_path -async def test_on_initialize_registers_all_flow_phases(minimal_flow, file_access_policy): - orchestrator = PhaseOrchestrator( - flow_yaml_path=minimal_flow / "flow.yaml", - checkpoint_path=minimal_flow / "checkpoints.yaml", - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - ) +async def test_on_initialize_registers_all_flow_phases(minimal_flow, make_orchestrator): + orchestrator = make_orchestrator(minimal_flow) await orchestrator.on_initialize() processors = orchestrator._subscribers.get(PhaseTrigger, []) @@ -217,14 +226,8 @@ async def test_on_initialize_registers_all_flow_phases(minimal_flow, file_access assert phase_names == {"a", "b"} -async def test_on_initialize_wires_dependencies(minimal_flow, file_access_policy): - orchestrator = PhaseOrchestrator( - flow_yaml_path=minimal_flow / "flow.yaml", - checkpoint_path=minimal_flow / "checkpoints.yaml", - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - ) +async def test_on_initialize_wires_dependencies(minimal_flow, make_orchestrator): + orchestrator = make_orchestrator(minimal_flow) await orchestrator.on_initialize() processors = orchestrator._subscribers.get(PhaseTrigger, []) @@ -233,14 +236,8 @@ async def test_on_initialize_wires_dependencies(minimal_flow, file_access_policy assert phases_by_name["b"]._dependencies == {"a"} -async def test_on_initialize_submits_initial_phase_trigger(minimal_flow, file_access_policy): - orchestrator = PhaseOrchestrator( - flow_yaml_path=minimal_flow / "flow.yaml", - checkpoint_path=minimal_flow / "checkpoints.yaml", - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - ) +async def test_on_initialize_submits_initial_phase_trigger(minimal_flow, make_orchestrator): + orchestrator = make_orchestrator(minimal_flow) await orchestrator.on_initialize() assert not orchestrator._queue.empty() @@ -249,7 +246,7 @@ async def test_on_initialize_submits_initial_phase_trigger(minimal_flow, file_ac assert msg.phase_id is None -async def test_on_initialize_unknown_phase_type_raises_flow_config_error(tmp_path, file_access_policy): +async def test_on_initialize_unknown_phase_type_raises_flow_config_error(tmp_path, make_orchestrator): (tmp_path / "prompts").mkdir() (tmp_path / "prompts" / "writer.md").write_text("system prompt") (tmp_path / "flow.yaml").write_text( @@ -268,18 +265,12 @@ async def test_on_initialize_unknown_phase_type_raises_flow_config_error(tmp_pat - phase: a """) ) - orchestrator = PhaseOrchestrator( - flow_yaml_path=tmp_path / "flow.yaml", - checkpoint_path=tmp_path / "checkpoints.yaml", - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - ) + orchestrator = make_orchestrator(tmp_path) with pytest.raises(FlowConfigError, match="Unknown phase type"): await orchestrator.on_initialize() -async def test_on_initialize_missing_agent_raises(tmp_path, file_access_policy): +async def test_on_initialize_missing_agent_raises(tmp_path, make_orchestrator): (tmp_path / "prompts").mkdir() (tmp_path / "flow.yaml").write_text( dedent("""\ @@ -288,7 +279,6 @@ async def test_on_initialize_missing_agent_raises(tmp_path, file_access_policy): tools: [] phases: a: - type: Phase agent: nonexistent_agent tasks: - name: task_a @@ -297,25 +287,13 @@ async def test_on_initialize_missing_agent_raises(tmp_path, file_access_policy): - phase: a """) ) - orchestrator = PhaseOrchestrator( - flow_yaml_path=tmp_path / "flow.yaml", - checkpoint_path=tmp_path / "checkpoints.yaml", - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - ) + orchestrator = make_orchestrator(tmp_path) with pytest.raises(FlowConfigError): await orchestrator.on_initialize() -async def test_on_initialize_phases_share_file_registry(minimal_flow, file_access_policy): - orchestrator = PhaseOrchestrator( - flow_yaml_path=minimal_flow / "flow.yaml", - checkpoint_path=minimal_flow / "checkpoints.yaml", - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - ) +async def test_on_initialize_phases_share_file_registry(minimal_flow, make_orchestrator): + orchestrator = make_orchestrator(minimal_flow) await orchestrator.on_initialize() phases = orchestrator._subscribers.get(PhaseTrigger, []) assert len(phases) >= 2 @@ -327,7 +305,7 @@ async def test_on_initialize_phases_share_file_registry(minimal_flow, file_acces # --------------------------------------------------------------------------- -async def test_orphan_phase_with_unknown_type_does_not_block_init(tmp_path, file_access_policy): +async def test_orphan_phase_with_unknown_type_does_not_block_init(tmp_path, make_orchestrator): """A phase defined in phases: but absent from flow: may have an unknown type — no error.""" (tmp_path / "prompts").mkdir() (tmp_path / "prompts" / "writer.md").write_text("system prompt") @@ -338,7 +316,6 @@ async def test_orphan_phase_with_unknown_type_does_not_block_init(tmp_path, file tools: [] phases: real: - type: Phase agent: writer tasks: - name: t1 @@ -353,20 +330,14 @@ async def test_orphan_phase_with_unknown_type_does_not_block_init(tmp_path, file - phase: real """) ) - orchestrator = PhaseOrchestrator( - flow_yaml_path=tmp_path / "flow.yaml", - checkpoint_path=tmp_path / "checkpoints.yaml", - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - ) + orchestrator = make_orchestrator(tmp_path) await orchestrator.on_initialize() processors = orchestrator._subscribers.get(PhaseTrigger, []) assert {p.name for p in processors} == {"real"} -async def test_phase_in_flow_with_unknown_type_raises(tmp_path, file_access_policy): +async def test_phase_in_flow_with_unknown_type_raises(tmp_path, make_orchestrator): """A phase referenced from flow: with an unknown type must still raise FlowConfigError.""" (tmp_path / "prompts").mkdir() (tmp_path / "prompts" / "writer.md").write_text("system prompt") @@ -386,21 +357,13 @@ async def test_phase_in_flow_with_unknown_type_raises(tmp_path, file_access_poli - phase: a """) ) - orchestrator = PhaseOrchestrator( - flow_yaml_path=tmp_path / "flow.yaml", - checkpoint_path=tmp_path / "checkpoints.yaml", - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - ) + orchestrator = make_orchestrator(tmp_path) with pytest.raises(FlowConfigError, match="Unknown phase type"): await orchestrator.on_initialize() -async def test_orphan_phase_logs_warning(tmp_path, file_access_policy, caplog): +async def test_orphan_phase_logs_warning(tmp_path, make_orchestrator, caplog): """An orphan phase must emit a warning containing its phase id.""" - import logging - (tmp_path / "prompts").mkdir() (tmp_path / "prompts" / "writer.md").write_text("system prompt") (tmp_path / "flow.yaml").write_text( @@ -410,13 +373,11 @@ async def test_orphan_phase_logs_warning(tmp_path, file_access_policy, caplog): tools: [] phases: real: - type: Phase agent: writer tasks: - name: t1 prompt: do it orphan: - type: Phase agent: writer tasks: - name: t2 @@ -425,13 +386,7 @@ async def test_orphan_phase_logs_warning(tmp_path, file_access_policy, caplog): - phase: real """) ) - orchestrator = PhaseOrchestrator( - flow_yaml_path=tmp_path / "flow.yaml", - checkpoint_path=tmp_path / "checkpoints.yaml", - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - ) + orchestrator = make_orchestrator(tmp_path) with caplog.at_level(logging.WARNING): await orchestrator.on_initialize() @@ -439,31 +394,70 @@ async def test_orphan_phase_logs_warning(tmp_path, file_access_policy, caplog): # --------------------------------------------------------------------------- -# PhaseOrchestrator.on_finalize +# PhaseOrchestrator.on_initialize — validate_config invocation # --------------------------------------------------------------------------- -async def test_on_finalize_no_failure_is_noop(tmp_path, file_access_policy): - orchestrator = PhaseOrchestrator( - flow_yaml_path=Path("/fake/flow.yaml"), - checkpoint_path=Path("/fake/checkpoints.yaml"), - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, +async def test_on_initialize_invokes_validate_config(tmp_path, make_orchestrator): + """validate_config is called for each scheduled phase; raising propagates as FlowConfigError.""" + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + a: + agent: writer + tasks: [] + flow: + - phase: a + """) + ) + orchestrator = make_orchestrator(tmp_path) + with pytest.raises(FlowConfigError, match="at least one task"): + await orchestrator.on_initialize() + + +async def test_on_initialize_skips_validate_config_for_orphan(tmp_path, make_orchestrator): + """A phase defined but not in flow must not trigger its validate_config.""" + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + real: + agent: writer + tasks: + - name: t1 + prompt: do it + orphan: + agent: writer + tasks: [] + flow: + - phase: real + """) ) - await orchestrator.on_finalize(None) # must not raise + orchestrator = make_orchestrator(tmp_path) + await orchestrator.on_initialize() # must not raise -async def test_on_finalize_after_phase_failed_logs(tmp_path, file_access_policy, caplog): - import logging +# --------------------------------------------------------------------------- +# PhaseOrchestrator.on_finalize +# --------------------------------------------------------------------------- - orchestrator = PhaseOrchestrator( - flow_yaml_path=Path("/fake/flow.yaml"), - checkpoint_path=Path("/fake/checkpoints.yaml"), - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - ) + +async def test_on_finalize_no_failure_is_noop(tmp_path, make_orchestrator): + orchestrator = make_orchestrator() + await orchestrator.on_finalize(None) # must not raise + + +async def test_on_finalize_after_phase_failed_logs(tmp_path, make_orchestrator, caplog): + orchestrator = make_orchestrator() msg = PhaseFailedMessage(id="f1", phase_id="p1", error="boom") exc = FatalProcessingError("Phase 'p1' failed: boom") with pytest.raises(FatalProcessingError): @@ -475,16 +469,8 @@ async def test_on_finalize_after_phase_failed_logs(tmp_path, file_access_policy, assert any("Pipeline aborted" in r.message and "p1" in r.message and "boom" in r.message for r in caplog.records) -async def test_on_finalize_no_exception_no_log(tmp_path, file_access_policy, caplog): - import logging - - orchestrator = PhaseOrchestrator( - flow_yaml_path=Path("/fake/flow.yaml"), - checkpoint_path=Path("/fake/checkpoints.yaml"), - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - ) +async def test_on_finalize_no_exception_no_log(tmp_path, make_orchestrator, caplog): + orchestrator = make_orchestrator() msg = PhaseFailedMessage(id="f1", phase_id="p1", error="boom") with pytest.raises(FatalProcessingError): await orchestrator.on_message_received(msg) @@ -495,7 +481,7 @@ async def test_on_finalize_no_exception_no_log(tmp_path, file_access_policy, cap assert not any("Pipeline aborted" in r.message for r in caplog.records) -def test_run_raises_runtime_error_when_phase_fails(tmp_path, file_access_policy): +def test_run_raises_runtime_error_when_phase_fails(tmp_path, make_orchestrator): """Full pipeline: a failing phase must cause run() to raise RuntimeError.""" (tmp_path / "prompts").mkdir() (tmp_path / "prompts" / "writer.md").write_text("system prompt") @@ -517,17 +503,10 @@ def test_run_raises_runtime_error_when_phase_fails(tmp_path, file_access_policy) ) class FailingPhase(Phase): - async def process_message(self, message: PhaseTrigger) -> None: + async def execute(self, context): raise RuntimeError("intentional failure") - orchestrator = PhaseOrchestrator( - flow_yaml_path=tmp_path / "flow.yaml", - checkpoint_path=tmp_path / "checkpoints.yaml", - runtime_variables={}, - anthropic_client=MagicMock(), - file_access_policy=file_access_policy, - grace_period=0.1, - ) + orchestrator = make_orchestrator(tmp_path, grace_period=0.1) orchestrator._phase_registry.register("FailingPhase", FailingPhase) with pytest.raises(FatalProcessingError, match="Phase 'failing' failed"): From 0886870c6a634da9e450d0bc58848cd280bd8d8b Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Thu, 21 May 2026 17:25:50 +0200 Subject: [PATCH 37/44] Implement spawn subagent tool --- ddev/src/ddev/ai/agent/build.py | 101 ++++- ddev/src/ddev/ai/callbacks/file_logger.py | 105 +++++ ddev/src/ddev/ai/phases/agentic_phase.py | 30 +- ddev/src/ddev/ai/phases/base.py | 17 +- ddev/src/ddev/ai/phases/checkpoint.py | 9 +- ddev/src/ddev/ai/tools/agents/__init__.py | 3 + .../ddev/ai/tools/agents/spawn_subagent.py | 156 +++++++ ddev/src/ddev/ai/tools/registry.py | 31 +- ddev/tests/ai/agent/test_build.py | 140 ++++-- ddev/tests/ai/callbacks/test_file_logger.py | 114 +++++ ddev/tests/ai/phases/conftest.py | 4 +- ddev/tests/ai/phases/test_agentic_phase.py | 428 ++++++++---------- ddev/tests/ai/tools/agents/__init__.py | 3 + .../ai/tools/agents/test_spawn_subagent.py | 310 +++++++++++++ ddev/tests/ai/tools/test_registry.py | 16 +- 15 files changed, 1140 insertions(+), 327 deletions(-) create mode 100644 ddev/src/ddev/ai/callbacks/file_logger.py create mode 100644 ddev/src/ddev/ai/tools/agents/__init__.py create mode 100644 ddev/src/ddev/ai/tools/agents/spawn_subagent.py create mode 100644 ddev/tests/ai/callbacks/test_file_logger.py create mode 100644 ddev/tests/ai/tools/agents/__init__.py create mode 100644 ddev/tests/ai/tools/agents/test_spawn_subagent.py diff --git a/ddev/src/ddev/ai/agent/build.py b/ddev/src/ddev/ai/agent/build.py index 97e24aecb5183..6ab2e30fb9428 100644 --- a/ddev/src/ddev/ai/agent/build.py +++ b/ddev/src/ddev/ai/agent/build.py @@ -5,17 +5,26 @@ from __future__ import annotations from collections.abc import Callable +from pathlib import Path from typing import TYPE_CHECKING, Any from ddev.ai.agent.anthropic_client import AnthropicAgent from ddev.ai.agent.base import BaseAgent +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy from ddev.ai.tools.fs.file_registry import FileRegistry from ddev.ai.tools.registry import ToolRegistry if TYPE_CHECKING: from ddev.ai.phases.config import AgentConfig -AgentBuilder = Callable[[str, str], tuple[BaseAgent[Any], ToolRegistry]] +SubagentBuilder = Callable[ + [str, str, list[str]], # (system_prompt, owner_id, tool_names) + tuple[BaseAgent[Any], ToolRegistry], +] +AgentBuilder = Callable[ + [str, str, SubagentBuilder | None, Path | None], # system_prompt, owner_id, subagent_builder, log_dir + tuple[BaseAgent[Any], ToolRegistry], +] def _resolve_client(agent_clients: dict[str, Any], provider: str) -> Any: @@ -25,19 +34,22 @@ def _resolve_client(agent_clients: dict[str, Any], provider: str) -> Any: return client -def build_agent( +def _build_agent_and_registry( agent_config: AgentConfig, agent_clients: dict[str, Any], system_prompt: str, owner_id: str, + tool_names: list[str], file_registry: FileRegistry, + subagent_builder: SubagentBuilder | None = None, + log_dir: Path | None = None, ) -> tuple[BaseAgent[Any], ToolRegistry]: - """Construct a provider-specific BaseAgent and its ToolRegistry from an AgentConfig.""" - tool_registry = ToolRegistry.from_names( - agent_config.tools, + tool_names, owner_id=owner_id, file_registry=file_registry, + subagent_builder=subagent_builder, + log_dir=log_dir, ) if agent_config.provider == "anthropic": @@ -58,20 +70,93 @@ def build_agent( raise ValueError(f"Unknown agent provider: {agent_config.provider!r}") +def build_agent( + agent_config: AgentConfig, + agent_clients: dict[str, Any], + system_prompt: str, + owner_id: str, + file_registry: FileRegistry, + subagent_builder: SubagentBuilder | None = None, + log_dir: Path | None = None, +) -> tuple[BaseAgent[Any], ToolRegistry]: + """Construct a provider-specific BaseAgent and its ToolRegistry from an AgentConfig.""" + return _build_agent_and_registry( + agent_config=agent_config, + agent_clients=agent_clients, + system_prompt=system_prompt, + owner_id=owner_id, + tool_names=agent_config.tools, + file_registry=file_registry, + subagent_builder=subagent_builder, + log_dir=log_dir, + ) + + +def build_subagent( + parent_agent_config: AgentConfig, + agent_clients: dict[str, Any], + file_access_policy: FileAccessPolicy, + system_prompt: str, + owner_id: str, + tool_names: list[str], +) -> tuple[BaseAgent[Any], ToolRegistry]: + """Build a subagent + ToolRegistry. Always uses a fresh FileRegistry. + + Reuses the parent's provider/model/max_tokens. No subagent_builder or + log_dir is forwarded, so the subagent cannot recursively spawn subagents — + ToolRegistry.from_names will raise if spawn_subagent is in tool_names. + """ + return _build_agent_and_registry( + agent_config=parent_agent_config, + agent_clients=agent_clients, + system_prompt=system_prompt, + owner_id=owner_id, + tool_names=tool_names, + file_registry=FileRegistry(policy=file_access_policy), + ) + + def make_agent_builder( agent_config: AgentConfig, agent_clients: dict[str, Any], file_registry: FileRegistry, ) -> AgentBuilder: - """Return a closure that builds an agent+registry given a rendered system_prompt and owner_id.""" - - def builder(system_prompt: str, owner_id: str) -> tuple[BaseAgent[Any], ToolRegistry]: + """Return a closure that builds an agent+registry given system_prompt, owner_id, subagent_builder, log_dir.""" + + def builder( + system_prompt: str, + owner_id: str, + subagent_builder: SubagentBuilder | None, + log_dir: Path | None, + ) -> tuple[BaseAgent[Any], ToolRegistry]: return build_agent( agent_config=agent_config, agent_clients=agent_clients, system_prompt=system_prompt, owner_id=owner_id, file_registry=file_registry, + subagent_builder=subagent_builder, + log_dir=log_dir, + ) + + return builder + + +def make_subagent_builder( + parent_agent_config: AgentConfig, + agent_clients: dict[str, Any], + file_access_policy: FileAccessPolicy, +) -> SubagentBuilder: + """Return a closure that builds a subagent+registry given (system_prompt, owner_id, tool_names).""" + + def builder(system_prompt: str, owner_id: str, tool_names: list[str]) -> tuple[BaseAgent[Any], ToolRegistry]: + return build_subagent( + parent_agent_config=parent_agent_config, + agent_clients=agent_clients, + file_access_policy=file_access_policy, + system_prompt=system_prompt, + owner_id=owner_id, + tool_names=tool_names, ) return builder diff --git a/ddev/src/ddev/ai/callbacks/file_logger.py b/ddev/src/ddev/ai/callbacks/file_logger.py new file mode 100644 index 0000000000000..2e58e4057e915 --- /dev/null +++ b/ddev/src/ddev/ai/callbacks/file_logger.py @@ -0,0 +1,105 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import json +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from ddev.ai.agent.types import AgentResponse, ToolCall +from ddev.ai.callbacks.callbacks import Callbacks, CallbackSet +from ddev.ai.tools.core.types import ToolResult + + +class FileLogger: + """Append-only JSONL writer for ReAct events plus subagent start/finish bookkeeping. + + Owns the file handle. Call build_callbacks() to obtain a Callbacks object whose + handlers route ReAct events through _emit. Call close() in a finally to release + the handle. Assumes log_path.parent already exists. + """ + + def __init__(self, log_path: Path) -> None: + self._log_path = log_path + self._fh = log_path.open("a", encoding="utf-8") + self._closed = False + + @property + def log_path(self) -> Path: + return self._log_path + + def _emit(self, event: dict[str, Any]) -> None: + if self._closed: + return + record = {"ts": datetime.now(UTC).isoformat(), **event} + self._fh.write(json.dumps(record, default=str) + "\n") + self._fh.flush() + + def log_start(self, *, system_prompt: str, prompt: str, tools: list[str]) -> None: + self._emit({"event": "start", "system_prompt": system_prompt, "prompt": prompt, "tools": tools}) + + def log_finish(self, *, success: bool, **fields: Any) -> None: + self._emit({"event": "finish", "success": success, **fields}) + + def close(self) -> None: + if not self._closed: + self._fh.close() + self._closed = True + + def build_callbacks(self) -> Callbacks: + cb_set = CallbackSet() + + @cb_set.on_before_agent_send + async def _on_before_send(iteration: int) -> None: + self._emit({"event": "before_agent_send", "iter": iteration}) + + @cb_set.on_agent_response + async def _on_agent_response(response: AgentResponse, iteration: int) -> None: + self._emit( + { + "event": "agent_response", + "iter": iteration, + "text": response.text, + "tool_calls": [{"id": tc.id, "name": tc.name, "input": tc.input} for tc in response.tool_calls], + "stop_reason": str(response.stop_reason), + "tokens": { + "input": response.usage.input_tokens, + "output": response.usage.output_tokens, + "cache_read": response.usage.cache_read_input_tokens, + "cache_creation": response.usage.cache_creation_input_tokens, + }, + } + ) + + @cb_set.on_tool_call + async def _on_tool_call(tool_call: ToolCall, result: ToolResult, iteration: int) -> None: + self._emit( + { + "event": "tool_call", + "iter": iteration, + "tool_call_id": tool_call.id, + "name": tool_call.name, + "input": tool_call.input, + "result": { + "success": result.success, + "data": result.data, + "error": result.error, + "truncated": result.truncated, + }, + } + ) + + @cb_set.on_before_compact + async def _on_before_compact() -> None: + self._emit({"event": "before_compact"}) + + @cb_set.on_after_compact + async def _on_after_compact() -> None: + self._emit({"event": "after_compact"}) + + @cb_set.on_error + async def _on_error(error: BaseException) -> None: + self._emit({"event": "error", "exception": f"{type(error).__name__}: {error}"}) + + return Callbacks([cb_set]) diff --git a/ddev/src/ddev/ai/phases/agentic_phase.py b/ddev/src/ddev/ai/phases/agentic_phase.py index c8a36767cd24b..25431f7a44c20 100644 --- a/ddev/src/ddev/ai/phases/agentic_phase.py +++ b/ddev/src/ddev/ai/phases/agentic_phase.py @@ -8,7 +8,7 @@ from typing import Any from ddev.ai.agent.base import BaseAgent -from ddev.ai.agent.build import AgentBuilder, make_agent_builder +from ddev.ai.agent.build import AgentBuilder, SubagentBuilder, make_agent_builder, make_subagent_builder from ddev.ai.callbacks.callbacks import Callbacks from ddev.ai.phases.base import Phase, PhaseOutcome from ddev.ai.phases.checkpoint import CheckpointManager @@ -59,6 +59,7 @@ def __init__( flow_variables: dict[str, str], config_dir: Path, file_registry: FileRegistry, + subagent_builder: SubagentBuilder | None = None, callbacks: Callbacks | None = None, logger: logging.Logger | None = None, ) -> None: @@ -75,6 +76,7 @@ def __init__( logger=logger, ) self._agent_builder = agent_builder + self._subagent_builder = subagent_builder @classmethod def validate_config( @@ -91,22 +93,36 @@ def validate_config( raise FlowConfigError(f"Phase {phase_id!r} (AgenticPhase) must have at least one task") @classmethod - def extra_init_kwargs( + def extra_init_kwargs( # type: ignore[override] cls, + *, phase_id: str, phase_config: PhaseConfig, agents: dict[str, AgentConfig], agent_clients: dict[str, Any], file_registry: FileRegistry, + **_: Any, ) -> dict[str, Any]: if phase_config.agent is None: raise FlowConfigError(f"Phase {phase_id!r} (AgenticPhase) requires 'agent'") + agent_config = agents[phase_config.agent] + + subagent_builder = None + # TODO: generalize this dispatch if more agent-meta tools appear in tools/agents/. + if "spawn_subagent" in agent_config.tools: + subagent_builder = make_subagent_builder( + parent_agent_config=agent_config, + agent_clients=agent_clients, + file_access_policy=file_registry.policy, + ) + return { "agent_builder": make_agent_builder( - agent_config=agents[phase_config.agent], + agent_config=agent_config, agent_clients=agent_clients, file_registry=file_registry, - ) + ), + "subagent_builder": subagent_builder, } def before_react(self) -> None: @@ -146,7 +162,11 @@ def _build_agent_and_process(self, context: dict[str, Any]) -> tuple[BaseAgent[A context, self._resolver, ) - agent, tool_registry = self._agent_builder(system_prompt, self._phase_id) + log_dir = None + if self._subagent_builder is not None: + log_dir = self._checkpoint_manager.root / "subagents" / self._phase_id + + agent, tool_registry = self._agent_builder(system_prompt, self._phase_id, self._subagent_builder, log_dir) process = ReActProcess( agent=agent, tool_registry=tool_registry, diff --git a/ddev/src/ddev/ai/phases/base.py b/ddev/src/ddev/ai/phases/base.py index 126d5772ba1ea..227b92560a60f 100644 --- a/ddev/src/ddev/ai/phases/base.py +++ b/ddev/src/ddev/ai/phases/base.py @@ -109,15 +109,14 @@ def validate_config( return None @classmethod - def extra_init_kwargs( - cls, - phase_id: str, - phase_config: PhaseConfig, - agents: dict[str, AgentConfig], - agent_clients: dict[str, Any], - file_registry: FileRegistry, - ) -> dict[str, Any]: - """Override to inject subclass-specific kwargs into __init__ at construction time.""" + def extra_init_kwargs(cls, **kwargs: Any) -> dict[str, Any]: + """Override to inject subclass-specific kwargs into __init__ at construction time. + + The orchestrator passes every framework-level dep (phase_id, phase_config, agents, + agent_clients, file_registry, checkpoint_manager, ...) as keyword arguments. + Subclasses pick the ones they need by declaring them explicitly and accept the + rest via **kwargs. + """ return {} @abstractmethod diff --git a/ddev/src/ddev/ai/phases/checkpoint.py b/ddev/src/ddev/ai/phases/checkpoint.py index 1a23b8ce63b7f..3f32edb158a5b 100644 --- a/ddev/src/ddev/ai/phases/checkpoint.py +++ b/ddev/src/ddev/ai/phases/checkpoint.py @@ -18,8 +18,13 @@ class CheckpointManager: def __init__(self, path: Path) -> None: self._path = path + @property + def root(self) -> Path: + """Directory that holds checkpoints.yaml, per-phase memory files, and any side artifacts.""" + return self._path.parent + def _ensure_dir(self) -> None: - self._path.parent.mkdir(parents=True, exist_ok=True) + self.root.mkdir(parents=True, exist_ok=True) def read(self) -> dict[str, Any]: """Return full checkpoint data, keyed by phase_id. Empty dict if file absent.""" @@ -44,7 +49,7 @@ def build_memory_prompt(self, user_additions: str | None) -> str: def memory_path(self, phase_id: str) -> Path: """Return the resolved path to a phase's memory file.""" - return (self._path.parent / f"{phase_id}_memory.md").resolve() + return (self.root / f"{phase_id}_memory.md").resolve() def write_memory(self, phase_id: str, text: str) -> None: """Write agent-authored text to this phase's memory file.""" diff --git a/ddev/src/ddev/ai/tools/agents/__init__.py b/ddev/src/ddev/ai/tools/agents/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/tools/agents/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/tools/agents/spawn_subagent.py b/ddev/src/ddev/ai/tools/agents/spawn_subagent.py new file mode 100644 index 0000000000000..ec9b03f66cd85 --- /dev/null +++ b/ddev/src/ddev/ai/tools/agents/spawn_subagent.py @@ -0,0 +1,156 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from pathlib import Path +from typing import Annotated + +from pydantic import Field + +from ddev.ai.agent.build import SubagentBuilder +from ddev.ai.agent.types import StopReason +from ddev.ai.callbacks.file_logger import FileLogger +from ddev.ai.react.process import ReActProcess +from ddev.ai.tools.core.base import BaseTool, BaseToolInput +from ddev.ai.tools.core.types import ToolResult + + +class SpawnSubagentInput(BaseToolInput): + system_prompt: Annotated[ + str, + Field(description="System prompt that defines the subagent's role and behavior."), + ] + prompt: Annotated[ + str, + Field(description="The task prompt sent to the subagent as its first (and only) user turn."), + ] + tools: Annotated[ + list[str], + Field( + description=( + "Names of tools the subagent may use. Must be a subset of your tool list and " + "may not include 'spawn_subagent'. May be empty if the subagent should answer " + "from the prompt alone." + ), + ), + ] = [] + name: Annotated[ + str | None, + Field( + description=("Optional short human-readable name for the subagent."), + ), + ] = None + + +class SpawnSubagentTool(BaseTool[SpawnSubagentInput]): + """Delegate a self-contained subtask to a fresh subagent. + + The subagent runs one Reason-Action loop with the provided system prompt, user prompt, and tool subset. + Only the subagent's final assistant message is returned to you. Instruct the subagent in your prompt + to put anything you need in its final message. Include every piece of context the subagent needs + inside the system prompt and the user prompt.""" + + def __init__( + self, + owner_id: str, + subagent_builder: SubagentBuilder, + allowed_tools: list[str], + log_dir: Path, + ) -> None: + self._owner_id = owner_id + self._subagent_builder = subagent_builder + # Parent may itself have spawn_subagent; never offer it to children. + self._allowed_tools = set(allowed_tools) - {self.name} + self._log_dir = log_dir + self._counter = 0 + + @property + def name(self) -> str: + return "spawn_subagent" + + def _label(self, tool_input: SpawnSubagentInput) -> str: + return tool_input.name or "unnamed" + + async def __call__(self, tool_input: SpawnSubagentInput) -> ToolResult: + label = self._label(tool_input) + + # Subset validation — return failed ToolResult; no log file is opened. + if self.name in tool_input.tools: + return ToolResult( + success=False, + error=( + f"Subagent {label!r} not spawned: subagents cannot spawn further subagents " + f"('{self.name}' is not allowed in 'tools')." + ), + ) + disallowed = sorted(set(tool_input.tools) - self._allowed_tools) + if disallowed: + return ToolResult( + success=False, + error=( + f"Subagent {label!r} not spawned: disallowed tools requested: {disallowed}. " + f"Allowed subset: {sorted(self._allowed_tools)}." + ), + ) + + try: + self._log_dir.mkdir(parents=True, exist_ok=True) + except OSError as e: + return ToolResult( + success=False, + error=(f"Subagent {label!r} not spawned: cannot create log directory {self._log_dir}: {e}"), + ) + + self._counter += 1 + subagent_id = f"{self._owner_id}.sub.{self._counter:03d}-{label}" + log_path = self._log_dir / f"{self._counter:03d}-{label}.jsonl" + + logger = FileLogger(log_path) + try: + logger.log_start( + system_prompt=tool_input.system_prompt, + prompt=tool_input.prompt, + tools=tool_input.tools, + ) + + try: + agent, tool_registry = self._subagent_builder( + tool_input.system_prompt, + subagent_id, + tool_input.tools, + ) + except Exception as e: + logger.log_finish(success=False, error=f"build failed: {type(e).__name__}: {e}") + return ToolResult( + success=False, + error=f"Subagent {label!r} failed to build: {type(e).__name__}: {e}", + ) + + process = ReActProcess( + agent=agent, + tool_registry=tool_registry, + callbacks=logger.build_callbacks(), + ) + try: + result = await process.start(tool_input.prompt) + except Exception as e: + logger.log_finish(success=False, error=f"{type(e).__name__}: {e}") + return ToolResult( + success=False, + error=f"Subagent {label!r} failed: {type(e).__name__}: {e}", + ) + + logger.log_finish( + success=True, + iterations=result.iterations, + total_input_tokens=result.total_input_tokens, + total_output_tokens=result.total_output_tokens, + stop_reason=str(result.final_response.stop_reason), + ) + + data = result.final_response.text + if result.final_response.stop_reason == StopReason.MAX_TOKENS: + data = "[SUBAGENT HIT MAX_TOKENS — RESPONSE MAY BE TRUNCATED]\n\n" + data + return ToolResult(success=True, data=data) + finally: + logger.close() diff --git a/ddev/src/ddev/ai/tools/registry.py b/ddev/src/ddev/ai/tools/registry.py index 3fd255f7bda1b..828694eed6c11 100644 --- a/ddev/src/ddev/ai/tools/registry.py +++ b/ddev/src/ddev/ai/tools/registry.py @@ -4,8 +4,10 @@ from __future__ import annotations from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from importlib import import_module +from pathlib import Path +from typing import TYPE_CHECKING from anthropic.types import ToolParam @@ -15,6 +17,9 @@ from .core.protocol import ToolProtocol from .core.types import ToolResult +if TYPE_CHECKING: + from ddev.ai.agent.build import SubagentBuilder + @dataclass class ToolContext: @@ -22,6 +27,9 @@ class ToolContext: file_registry: FileRegistry owner_id: str + allowed_tool_names: tuple[str, ...] = field(default_factory=tuple) + subagent_builder: SubagentBuilder | None = None + log_dir: Path | None = None @property def policy(self) -> FileAccessPolicy: @@ -40,6 +48,21 @@ def _file_policy_factory(tool_cls: type, ctx: ToolContext) -> ToolProtocol: return tool_cls(ctx.policy) +def _spawn_subagent_factory(tool_cls: type, ctx: ToolContext) -> ToolProtocol: + if ctx.subagent_builder is None or ctx.log_dir is None: + raise ValueError( + "Tool 'spawn_subagent' requires both 'subagent_builder' and 'log_dir' to be " + "passed to ToolRegistry.from_names." + ) + allowed = [name for name in ctx.allowed_tool_names if name != "spawn_subagent"] + return tool_cls( + owner_id=ctx.owner_id, + subagent_builder=ctx.subagent_builder, + allowed_tools=allowed, + log_dir=ctx.log_dir, + ) + + @dataclass(frozen=True) class ToolSpec: """Lazy pointer to a tool class and how to construct it. @@ -71,6 +94,7 @@ class ToolSpec: "ddev_env_test": ToolSpec("shell.ddev.env_test", "DdevEnvTestTool"), "ddev_release_changelog": ToolSpec("shell.ddev.release_changelog", "DdevReleaseChangelogTool"), "ddev_validate": ToolSpec("shell.ddev.validate", "DdevValidateTool"), + "spawn_subagent": ToolSpec("agents.spawn_subagent", "SpawnSubagentTool", factory=_spawn_subagent_factory), } @@ -92,6 +116,8 @@ def from_names( *, owner_id: str, file_registry: FileRegistry, + subagent_builder: SubagentBuilder | None = None, + log_dir: Path | None = None, ) -> ToolRegistry: """Build a ToolRegistry from a list of tool name strings. @@ -102,6 +128,9 @@ def from_names( ctx = ToolContext( file_registry=file_registry, owner_id=owner_id, + allowed_tool_names=tuple(tool_names), + subagent_builder=subagent_builder, + log_dir=log_dir, ) tools: list[ToolProtocol] = [] for name in tool_names: diff --git a/ddev/tests/ai/agent/test_build.py b/ddev/tests/ai/agent/test_build.py index 254e1530dc11b..9cff25a6f8881 100644 --- a/ddev/tests/ai/agent/test_build.py +++ b/ddev/tests/ai/agent/test_build.py @@ -7,7 +7,7 @@ import pytest from ddev.ai.agent.anthropic_client import AnthropicAgent -from ddev.ai.agent.build import build_agent, make_agent_builder +from ddev.ai.agent.build import build_agent, build_subagent, make_agent_builder, make_subagent_builder from ddev.ai.phases.config import AgentConfig from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy from ddev.ai.tools.fs.file_registry import FileRegistry @@ -15,62 +15,102 @@ @pytest.fixture -def file_registry(tmp_path) -> FileRegistry: - return FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) +def policy(tmp_path) -> FileAccessPolicy: + return FileAccessPolicy(write_root=tmp_path) -def test_build_agent_anthropic_returns_agent_and_registry(file_registry): - agent_config = AgentConfig(provider="anthropic", model="claude-test", max_tokens=1024, tools=[]) - agent_clients = {"anthropic": MagicMock()} +@pytest.fixture +def file_registry(policy) -> FileRegistry: + return FileRegistry(policy=policy) + + +@pytest.fixture +def clients() -> dict: + return {"anthropic": MagicMock()} + + +# --------------------------------------------------------------------------- +# Core builder behaviour +# --------------------------------------------------------------------------- + + +def test_unknown_provider_raises(file_registry, clients): + config = AgentConfig.model_construct(provider="bad_provider", tools=[]) + with pytest.raises(ValueError, match="Unknown agent provider: 'bad_provider'"): + build_agent(config, clients, "sys", "p1", file_registry) - agent, registry = build_agent( - agent_config=agent_config, - agent_clients=agent_clients, - system_prompt="hello", - owner_id="p1", - file_registry=file_registry, - ) +def test_missing_client_raises(file_registry): + config = AgentConfig(provider="anthropic", tools=[]) + with pytest.raises(ValueError, match="No client provided for agent provider 'anthropic'"): + build_agent(config, {}, "sys", "p1", file_registry) + + +def test_builds_anthropic_agent_with_correct_types_and_name(file_registry, clients): + config = AgentConfig(provider="anthropic", tools=[]) + agent, registry = build_agent(config, clients, "sys", "p1", file_registry) assert isinstance(agent, AnthropicAgent) assert isinstance(registry, ToolRegistry) assert agent.name == "p1" -def test_build_agent_missing_client_raises(file_registry): - agent_config = AgentConfig(provider="anthropic", tools=[]) - with pytest.raises(ValueError, match="No client provided for agent provider 'anthropic'"): - build_agent( - agent_config=agent_config, - agent_clients={}, - system_prompt="hello", - owner_id="p1", - file_registry=file_registry, - ) - - -def test_build_agent_unknown_provider_raises(file_registry): - agent_config = AgentConfig(provider="openai", tools=[]) - with pytest.raises(ValueError, match="Unknown agent provider: 'openai'"): - build_agent( - agent_config=agent_config, - agent_clients={"openai": MagicMock()}, - system_prompt="hello", - owner_id="p1", - file_registry=file_registry, - ) - - -def test_make_agent_builder_returns_callable_that_delegates_to_build_agent(file_registry): - agent_config = AgentConfig(provider="anthropic", tools=[]) - agent_clients = {"anthropic": MagicMock()} - - builder = make_agent_builder( - agent_config=agent_config, - agent_clients=agent_clients, - file_registry=file_registry, - ) - - agent, registry = builder("system prompt", "p2") +@pytest.mark.parametrize( + "model,max_tokens", + [ + ("claude-opus-4-7", 2048), + ("claude-haiku-4-5", 512), + ], +) +def test_model_and_max_tokens_forwarded(file_registry, clients, model, max_tokens): + config = AgentConfig(provider="anthropic", model=model, max_tokens=max_tokens, tools=[]) + agent, _ = build_agent(config, clients, "sys", "p1", file_registry) + assert agent._model == model + assert agent._max_tokens == max_tokens + + +def test_build_agent_uses_config_tools(file_registry, clients): + config = AgentConfig(provider="anthropic", tools=["read_file"]) + _, registry = build_agent(config, clients, "sys", "p1", file_registry) + assert len(registry.definitions) == 1 + assert registry.definitions[0]["name"] == "read_file" + + +# --------------------------------------------------------------------------- +# build_subagent +# --------------------------------------------------------------------------- + + +def test_build_subagent_creates_fresh_file_registry(policy, clients): + config = AgentConfig(provider="anthropic", tools=[]) + caller_registry = FileRegistry(policy=policy) + _, reg_a = build_subagent(config, clients, policy, "sys", "a", []) + _, reg_b = build_subagent(config, clients, policy, "sys", "b", []) + assert reg_a is not reg_b + assert reg_a is not caller_registry + + +def test_build_subagent_recursion_guard(policy, clients): + config = AgentConfig.model_construct(provider="anthropic", tools=[]) + with pytest.raises(ValueError): + build_subagent(config, clients, policy, "sys", "sub", ["spawn_subagent"]) + + +# --------------------------------------------------------------------------- +# Closures — verify delegation works and signatures are correct +# --------------------------------------------------------------------------- + + +def test_make_agent_builder(file_registry, clients): + config = AgentConfig(provider="anthropic", tools=[]) + builder = make_agent_builder(config, clients, file_registry) + agent, registry = builder("sys", "p1", None, None) assert isinstance(agent, AnthropicAgent) - assert isinstance(registry, ToolRegistry) - assert agent.name == "p2" + assert agent.name == "p1" + + +def test_make_subagent_builder(policy, clients): + config = AgentConfig(provider="anthropic", tools=[]) + builder = make_subagent_builder(config, clients, policy) + agent, registry = builder("sys", "sub-1", []) + assert isinstance(agent, AnthropicAgent) + assert agent.name == "sub-1" diff --git a/ddev/tests/ai/callbacks/test_file_logger.py b/ddev/tests/ai/callbacks/test_file_logger.py new file mode 100644 index 0000000000000..1b4f52bf251cb --- /dev/null +++ b/ddev/tests/ai/callbacks/test_file_logger.py @@ -0,0 +1,114 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import json + +import pytest + +from ddev.ai.agent.types import AgentResponse, StopReason, TokenUsage, ToolCall +from ddev.ai.callbacks.file_logger import FileLogger +from ddev.ai.tools.core.types import ToolResult + + +def make_response(text: str = "", stop_reason: StopReason = StopReason.END_TURN) -> AgentResponse: + return AgentResponse( + stop_reason=stop_reason, + text=text, + tool_calls=[], + usage=TokenUsage(input_tokens=10, output_tokens=5, cache_read_input_tokens=0, cache_creation_input_tokens=0), + ) + + +def read_events(log_path) -> list[dict]: + return [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +# --------------------------------------------------------------------------- +# File mechanics +# --------------------------------------------------------------------------- + + +def test_log_entries_are_valid_jsonl_with_timestamp(tmp_path): + log_path = tmp_path / "log.jsonl" + logger = FileLogger(log_path) + logger.log_start(system_prompt="sys", prompt="go", tools=["read_file"]) + logger.log_finish(success=True, iterations=1) + logger.close() + + events = read_events(log_path) + assert len(events) == 2 + assert all("ts" in e for e in events) + assert events[0]["event"] == "start" + assert events[1]["event"] == "finish" + + +def test_flush_after_each_write(tmp_path): + log_path = tmp_path / "log.jsonl" + logger = FileLogger(log_path) + logger.log_start(system_prompt="s", prompt="p", tools=[]) + # A second file handle reads the line without closing the logger first + assert log_path.read_text(encoding="utf-8").strip() != "" + logger.close() + + +def test_close_is_idempotent_and_prevents_further_writes(tmp_path): + log_path = tmp_path / "log.jsonl" + logger = FileLogger(log_path) + logger.log_start(system_prompt="s", prompt="p", tools=[]) + logger.close() + logger.close() # must not raise + logger.log_finish(success=False) # must not write + assert len(read_events(log_path)) == 1 + + +def test_constructor_requires_existing_parent(tmp_path): + with pytest.raises(OSError): + FileLogger(tmp_path / "doesnotexist" / "log.jsonl") + + +def test_non_serializable_values_use_str_repr(tmp_path): + log_path = tmp_path / "log.jsonl" + logger = FileLogger(log_path) + + class Unserializable: + def __repr__(self): + return "" + + logger.log_finish(success=True, extra=Unserializable()) + logger.close() + + assert read_events(log_path)[0]["extra"] == "" + + +# --------------------------------------------------------------------------- +# Callbacks wiring +# --------------------------------------------------------------------------- + + +async def test_build_callbacks_fires_all_event_types(tmp_path): + log_path = tmp_path / "log.jsonl" + logger = FileLogger(log_path) + callbacks = logger.build_callbacks() + + tool_call = ToolCall(id="tc1", name="read_file", input={"path": "/f"}) + tool_result = ToolResult(success=True, data="content") + + await callbacks.fire_before_agent_send(2) + await callbacks.fire_agent_response(make_response("hi"), 2) + await callbacks.fire_tool_call(tool_call, tool_result, 2) + await callbacks.fire_before_compact() + await callbacks.fire_after_compact() + await callbacks.fire_error(ValueError("oops")) + logger.close() + + events = {e["event"]: e for e in read_events(log_path)} + + assert events["before_agent_send"]["iter"] == 2 + assert events["agent_response"]["text"] == "hi" + assert events["agent_response"]["iter"] == 2 + assert events["tool_call"]["tool_call_id"] == "tc1" + assert events["tool_call"]["result"]["success"] is True + assert "before_compact" in events + assert "after_compact" in events + assert "ValueError" in events["error"]["exception"] diff --git a/ddev/tests/ai/phases/conftest.py b/ddev/tests/ai/phases/conftest.py index 7f07c1734ffdc..96149455e1385 100644 --- a/ddev/tests/ai/phases/conftest.py +++ b/ddev/tests/ai/phases/conftest.py @@ -92,7 +92,9 @@ def make_agent_builder(mock_agent: MockAgent, captured_kwargs: dict[str, Any] | owner_id passed in — useful for asserting on prompt rendering. """ - def builder(system_prompt: str, owner_id: str) -> tuple[MockAgent, ToolRegistry]: + def builder( + system_prompt: str, owner_id: str, subagent_builder=None, log_dir=None + ) -> tuple[MockAgent, ToolRegistry]: if captured_kwargs is not None: captured_kwargs["system_prompt"] = system_prompt captured_kwargs["owner_id"] = owner_id diff --git a/ddev/tests/ai/phases/test_agentic_phase.py b/ddev/tests/ai/phases/test_agentic_phase.py index e41188c46e8f2..bc333f965aee9 100644 --- a/ddev/tests/ai/phases/test_agentic_phase.py +++ b/ddev/tests/ai/phases/test_agentic_phase.py @@ -2,17 +2,28 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import json from pathlib import Path import pytest +from ddev.ai.agent.types import AgentResponse, StopReason, TokenUsage, ToolCall from ddev.ai.callbacks.callbacks import Callbacks, CallbackSet from ddev.ai.phases.agentic_phase import AgenticPhase, render_memory_prompt, render_task_prompt +from ddev.ai.phases.checkpoint import CheckpointManager from ddev.ai.phases.config import AgentConfig, CheckpointConfig, FlowConfigError, PhaseConfig, TaskConfig -from ddev.ai.phases.messages import PhaseTrigger +from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.registry import ToolRegistry from .conftest import MockAgent, make_agent_phase, make_response, resolve_key + +def read_jsonl(path: Path) -> list[dict]: + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + # --------------------------------------------------------------------------- # render_task_prompt # --------------------------------------------------------------------------- @@ -21,29 +32,24 @@ def test_render_task_prompt_from_file(tmp_path): prompt_file = tmp_path / "task.md" prompt_file.write_text("Hello ${name}.") - task = TaskConfig(name="t1", prompt_path="task.md") - result = render_task_prompt(task, tmp_path, {"name": "Alice"}) + result = render_task_prompt(TaskConfig(name="t1", prompt_path="task.md"), tmp_path, {"name": "Alice"}) assert result == "Hello Alice." def test_render_task_prompt_inline(): - task = TaskConfig(name="t1", prompt="Hello ${name}.") - result = render_task_prompt(task, None, {"name": "Bob"}) + result = render_task_prompt(TaskConfig(name="t1", prompt="Hello ${name}."), None, {"name": "Bob"}) assert result == "Hello Bob." def test_render_task_prompt_forwards_resolver(tmp_path): - prompt_file = tmp_path / "task.md" - prompt_file.write_text("Memory: ${draft_memory}") - task = TaskConfig(name="t1", prompt_path="task.md") - result = render_task_prompt(task, tmp_path, {}, resolve_key) + (tmp_path / "task.md").write_text("Memory: ${draft_memory}") + result = render_task_prompt(TaskConfig(name="t1", prompt_path="task.md"), tmp_path, {}, resolve_key) assert result == "Memory: resolved(draft_memory)" -def test_render_task_prompt_raises_when_both_unset(): - task = TaskConfig.model_construct(name="t1", prompt=None, prompt_path=None) +def test_render_task_prompt_raises_when_no_source(): with pytest.raises(FlowConfigError, match="prompt"): - render_task_prompt(task, None, {}) + render_task_prompt(TaskConfig.model_construct(name="t1", prompt=None, prompt_path=None), None, {}) # --------------------------------------------------------------------------- @@ -52,23 +58,21 @@ def test_render_task_prompt_raises_when_both_unset(): def test_render_memory_prompt_from_file(tmp_path): - mem_file = tmp_path / "mem.md" - mem_file.write_text("List files for ${phase_name}.") - checkpoint = CheckpointConfig(memory_prompt_path="mem.md") - result = render_memory_prompt(checkpoint, tmp_path, {"phase_name": "draft"}) + (tmp_path / "mem.md").write_text("List files for ${phase_name}.") + result = render_memory_prompt(CheckpointConfig(memory_prompt_path="mem.md"), tmp_path, {"phase_name": "draft"}) assert result == "List files for draft." def test_render_memory_prompt_inline(): - checkpoint = CheckpointConfig(memory_prompt="List files for ${phase_name}.") - result = render_memory_prompt(checkpoint, None, {"phase_name": "draft"}) + result = render_memory_prompt( + CheckpointConfig(memory_prompt="List files for ${phase_name}."), None, {"phase_name": "draft"} + ) assert result == "List files for draft." -def test_render_memory_prompt_raises_when_both_unset(): - checkpoint = CheckpointConfig.model_construct(memory_prompt=None, memory_prompt_path=None) +def test_render_memory_prompt_raises_when_no_source(): with pytest.raises(FlowConfigError, match="memory_prompt"): - render_memory_prompt(checkpoint, None, {}) + render_memory_prompt(CheckpointConfig.model_construct(memory_prompt=None, memory_prompt_path=None), None, {}) # --------------------------------------------------------------------------- @@ -76,262 +80,153 @@ def test_render_memory_prompt_raises_when_both_unset(): # --------------------------------------------------------------------------- -def test_agentic_phase_validate_config_rejects_missing_agent(): - config = PhaseConfig(tasks=[TaskConfig(name="t1", prompt="x")]) - with pytest.raises(FlowConfigError, match="requires 'agent'"): - AgenticPhase.validate_config("p1", config, {}) - - -def test_agentic_phase_validate_config_rejects_unknown_agent(): - config = PhaseConfig(agent="ghost", tasks=[TaskConfig(name="t1", prompt="x")]) - with pytest.raises(FlowConfigError, match="unknown agent"): - AgenticPhase.validate_config("p1", config, {"writer": AgentConfig()}) - - -def test_agentic_phase_validate_config_rejects_empty_tasks(): - config = PhaseConfig(agent="writer") - with pytest.raises(FlowConfigError, match="at least one task"): +@pytest.mark.parametrize( + "config,match", + [ + (PhaseConfig(tasks=[TaskConfig(name="t1", prompt="x")]), "requires 'agent'"), + (PhaseConfig(agent="ghost", tasks=[TaskConfig(name="t1", prompt="x")]), "unknown agent"), + (PhaseConfig(agent="writer"), "at least one task"), + ], + ids=["missing_agent", "unknown_agent", "empty_tasks"], +) +def test_validate_config_rejects_invalid(config, match): + with pytest.raises(FlowConfigError, match=match): AgenticPhase.validate_config("p1", config, {"writer": AgentConfig()}) -def test_agentic_phase_validate_config_accepts_valid(): - config = PhaseConfig(agent="writer", tasks=[TaskConfig(name="t1", prompt="x")]) - AgenticPhase.validate_config("p1", config, {"writer": AgentConfig()}) +def test_validate_config_accepts_valid(): + AgenticPhase.validate_config( + "p1", PhaseConfig(agent="writer", tasks=[TaskConfig(name="t1", prompt="x")]), {"writer": AgentConfig()} + ) # --------------------------------------------------------------------------- -# AgenticPhase.process_message — happy path +# process_message — happy path # --------------------------------------------------------------------------- async def test_happy_path_single_task(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task done", 100, 50), # task 1 via ReActProcess - make_response("summary", 10, 5), # memory step - ] - mock_agent = MockAgent(responses) + mock_agent = MockAgent([make_response("task done", 100, 50), make_response("summary", 10, 5)]) phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) await phase.process_message(PhaseTrigger(id="start", phase_id=None)) assert mgr.memory_content("p1") == "summary" - checkpoint = mgr.read()["p1"] assert checkpoint["status"] == "success" - assert checkpoint["tokens"]["total_input"] == 110 - assert checkpoint["tokens"]["total_output"] == 55 - assert checkpoint["memory_path"] - - assert len(mock_agent.send_calls) == 2 + assert checkpoint["tokens"] == {"total_input": 110, "total_output": 55} assert mock_agent.send_calls[0] == "Do the work." assert "Write a brief summary" in mock_agent.send_calls[1] + # checkpoint memory_path points to the written file + memory_path = Path(checkpoint["memory_path"]) + assert memory_path.is_absolute() and memory_path.exists() and memory_path.name == "p1_memory.md" -async def test_happy_path_two_tasks(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task1 done", 100, 50), - make_response("task2 done", 200, 80), - make_response("summary", 10, 5), - ] - mock_agent = MockAgent(responses) +async def test_happy_path_two_tasks_accumulates_tokens(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent( + [ + make_response("t1 done", 100, 50), + make_response("t2 done", 200, 80), + make_response("summary", 10, 5), + ] + ) phase, mgr = make_agent_phase( flow_dir, mock_agent, monkeypatch, message_queue, - tasks=[ - TaskConfig(name="t1", prompt="First task."), - TaskConfig(name="t2", prompt="Second task."), - ], + tasks=[TaskConfig(name="t1", prompt="First."), TaskConfig(name="t2", prompt="Second.")], ) await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - checkpoint = mgr.read()["p1"] - assert checkpoint["tokens"]["total_input"] == 310 - assert checkpoint["tokens"]["total_output"] == 135 - assert checkpoint["memory_path"] + assert mgr.read()["p1"]["tokens"] == {"total_input": 310, "total_output": 135} # --------------------------------------------------------------------------- -# AgenticPhase.process_message — memory step with checkpoint config +# process_message — context compaction # --------------------------------------------------------------------------- -async def test_memory_step_with_checkpoint_config(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task done", 100, 50), - make_response("summary with files", 10, 5), - ] - mock_agent = MockAgent(responses) - phase, mgr = make_agent_phase( +@pytest.mark.parametrize("context_pct,expect_compact", [(85, True), (50, False)], ids=["above", "below"]) +async def test_compact_between_tasks(flow_dir, monkeypatch, message_queue, context_pct, expect_compact): + mock_agent = MockAgent( + [ + make_response("t1 done", 100, 50, context_pct=context_pct), + make_response("t2 done", 200, 80), + make_response("summary", 10, 5), + ] + ) + phase, _ = make_agent_phase( flow_dir, mock_agent, monkeypatch, message_queue, - checkpoint=CheckpointConfig(memory_prompt="Also list the files."), + tasks=[TaskConfig(name="t1", prompt="First."), TaskConfig(name="t2", prompt="Second.")], ) await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - memory_prompt = mock_agent.send_calls[1] - assert "Also list the files." in memory_prompt - assert "Write a brief summary" in memory_prompt - - -async def test_memory_step_without_checkpoint_config(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task done", 100, 50), - make_response("summary", 10, 5), - ] - mock_agent = MockAgent(responses) - phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) - - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - memory_prompt = mock_agent.send_calls[1] - assert memory_prompt == "Write a brief summary of what you accomplished in this phase." + assert (mock_agent.compact_call_count >= 1) == expect_compact # --------------------------------------------------------------------------- -# AgenticPhase.process_message — context compaction between tasks +# process_message — before_react / after_react hooks # --------------------------------------------------------------------------- -async def test_compact_between_tasks_when_above_threshold(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task1 done", 100, 50, context_pct=85), # above 80% threshold - make_response("task2 done", 200, 80), - make_response("summary", 10, 5), - ] - mock_agent = MockAgent(responses) - phase, mgr = make_agent_phase( - flow_dir, - mock_agent, - monkeypatch, - message_queue, - tasks=[ - TaskConfig(name="t1", prompt="First task."), - TaskConfig(name="t2", prompt="Second task."), - ], - ) - - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - checkpoint = mgr.read()["p1"] - assert checkpoint["status"] == "success" - assert checkpoint["memory_path"] - assert mock_agent.compact_call_count >= 1 - +@pytest.mark.parametrize("hook_name", ["before_react", "after_react"], ids=["before", "after"]) +async def test_react_hook_failure_fails_phase(flow_dir, monkeypatch, message_queue, hook_name): + mock_agent = MockAgent([make_response("done", 100, 50)]) + phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) + setattr(phase, hook_name, lambda: (_ for _ in ()).throw(RuntimeError("hook failed"))) -async def test_no_compact_when_below_threshold(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task1 done", 100, 50, context_pct=50), # below 80% threshold - make_response("task2 done", 200, 80), - make_response("summary", 10, 5), - ] - mock_agent = MockAgent(responses) - phase, mgr = make_agent_phase( - flow_dir, - mock_agent, - monkeypatch, - message_queue, - tasks=[ - TaskConfig(name="t1", prompt="First task."), - TaskConfig(name="t2", prompt="Second task."), - ], - ) + with pytest.raises(RuntimeError, match="hook failed"): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - checkpoint = mgr.read()["p1"] - assert checkpoint["status"] == "success" - assert checkpoint["memory_path"] - assert mock_agent.compact_call_count == 0 + assert mgr.read() == {} # --------------------------------------------------------------------------- -# AgenticPhase.process_message — template context +# process_message — template context # --------------------------------------------------------------------------- -async def test_flow_variables_in_system_prompt(flow_dir, monkeypatch, message_queue): +async def test_flow_variables_rendered_in_system_prompt(flow_dir, monkeypatch, message_queue): (flow_dir / "prompts" / "writer.md").write_text("Project: ${project}") mock_agent = MockAgent([make_response("done", 100, 50), make_response("summary", 10, 5)]) - captured_kwargs: dict = {} + captured: dict = {} phase, _ = make_agent_phase( flow_dir, mock_agent, monkeypatch, message_queue, flow_variables={"project": "myproj"}, - captured_agent_kwargs=captured_kwargs, + captured_agent_kwargs=captured, ) await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - assert captured_kwargs["system_prompt"] == "Project: myproj" + assert captured["system_prompt"] == "Project: myproj" async def test_runtime_variables_override_flow_variables(flow_dir, monkeypatch, message_queue): (flow_dir / "prompts" / "writer.md").write_text("Project: ${project}") mock_agent = MockAgent([make_response("done", 100, 50), make_response("summary", 10, 5)]) - captured_kwargs: dict = {} + captured: dict = {} phase, _ = make_agent_phase( flow_dir, mock_agent, monkeypatch, message_queue, - flow_variables={"project": "flow_default"}, - runtime_variables={"project": "runtime_override"}, - captured_agent_kwargs=captured_kwargs, + flow_variables={"project": "flow"}, + runtime_variables={"project": "runtime"}, + captured_agent_kwargs=captured, ) await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - assert captured_kwargs["system_prompt"] == "Project: runtime_override" - - -# --------------------------------------------------------------------------- -# AgenticPhase.process_message — before_react / after_react errors -# --------------------------------------------------------------------------- - - -async def test_before_react_raises_propagates(flow_dir, monkeypatch, message_queue): - mock_agent = MockAgent([]) - phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) - - def failing_hook(): - raise RuntimeError("setup failed") - - phase.before_react = failing_hook - - with pytest.raises(RuntimeError, match="setup failed"): - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - assert mgr.read() == {} - - -async def test_after_react_raises_propagates(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("done", 100, 50), - ] - mock_agent = MockAgent(responses) - phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) - - def failing_hook(): - raise RuntimeError("teardown failed") - - phase.after_react = failing_hook - - with pytest.raises(RuntimeError, match="teardown failed"): - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - - assert mgr.read() == {} - - -# --------------------------------------------------------------------------- -# AgenticPhase.process_message — resolver integration with memory files -# --------------------------------------------------------------------------- + assert captured["system_prompt"] == "Project: runtime" async def test_task_prompt_resolves_memory_variable(flow_dir, monkeypatch, message_queue): @@ -344,7 +239,6 @@ async def test_task_prompt_resolves_memory_variable(flow_dir, monkeypatch, messa phase_id="review", tasks=[TaskConfig(name="t1", prompt="Review: ${draft_memory}")], ) - mgr.write_phase_checkpoint("draft", {"status": "success"}) mgr.write_memory("draft", "Created file.py") await phase.process_message(PhaseTrigger(id="start", phase_id=None)) @@ -353,14 +247,15 @@ async def test_task_prompt_resolves_memory_variable(flow_dir, monkeypatch, messa # --------------------------------------------------------------------------- -# AgenticPhase.process_message — memory step failure behaviour +# process_message — failure modes # --------------------------------------------------------------------------- async def test_memory_api_failure_fails_phase(flow_dir, monkeypatch, message_queue): - responses = [make_response("task done", 100, 50)] - mock_agent = MockAgent(responses) - phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) + # Only one response — IndexError when memory step tries to call agent again + phase, mgr = make_agent_phase( + flow_dir, MockAgent([make_response("task done", 100, 50)]), monkeypatch, message_queue + ) with pytest.raises(IndexError): await phase.process_message(PhaseTrigger(id="start", phase_id=None)) @@ -368,45 +263,37 @@ async def test_memory_api_failure_fails_phase(flow_dir, monkeypatch, message_que assert mgr.read() == {} -async def test_memory_template_error_fails_phase(flow_dir, monkeypatch, message_queue): - responses = [make_response("task done", 100, 50)] - mock_agent = MockAgent(responses) +async def test_memory_template_render_failure_fails_phase(flow_dir, monkeypatch, message_queue): phase, mgr = make_agent_phase( flow_dir, - mock_agent, + MockAgent([make_response("task done", 100, 50)]), monkeypatch, message_queue, checkpoint=CheckpointConfig(memory_prompt="Summarize."), ) + monkeypatch.setattr( + "ddev.ai.phases.agentic_phase.render_memory_prompt", + lambda *a, **kw: (_ for _ in ()).throw(ValueError("bad template")), + ) - def raise_render_error(*args, **kwargs): - raise ValueError("template error") - - monkeypatch.setattr("ddev.ai.phases.agentic_phase.render_memory_prompt", raise_render_error) - - with pytest.raises(ValueError, match="template error"): + with pytest.raises(ValueError, match="bad template"): await phase.process_message(PhaseTrigger(id="start", phase_id=None)) assert mgr.read() == {} -async def test_successful_phase_writes_memory_path_into_checkpoint(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task done", 100, 50), - make_response("summary text", 10, 5), - ] - mock_agent = MockAgent(responses) +async def test_disk_failure_on_write_memory_fails_phase(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([make_response("task done", 100, 50), make_response("summary", 10, 5)]) phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) + monkeypatch.setattr( + "ddev.ai.phases.checkpoint.CheckpointManager.write_memory", + lambda *a, **kw: (_ for _ in ()).throw(PermissionError("read-only")), + ) - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + with pytest.raises(PermissionError, match="read-only"): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) - checkpoint = mgr.read()["p1"] - assert "memory_path" in checkpoint - memory_path = Path(checkpoint["memory_path"]) - assert memory_path.is_absolute() - assert memory_path.exists() - assert memory_path.name == "p1_memory.md" - assert memory_path.read_text() == "summary text" + assert mgr.read() == {} # --------------------------------------------------------------------------- @@ -415,19 +302,15 @@ async def test_successful_phase_writes_memory_path_into_checkpoint(flow_dir, mon @pytest.mark.parametrize( - "checkpoint, expected_build_arg", - [ - (None, None), - (CheckpointConfig(memory_prompt="anything"), "USER_ADDITIONS"), - ], + "checkpoint,expected_user_additions", + [(None, None), (CheckpointConfig(memory_prompt="anything"), "USER_ADDITIONS")], ids=["no_checkpoint", "with_checkpoint"], ) -async def test_run_memory_step_forwards_user_additions_to_build( - flow_dir, monkeypatch, message_queue, checkpoint, expected_build_arg +async def test_run_memory_step_passes_user_additions_to_build( + flow_dir, monkeypatch, message_queue, checkpoint, expected_user_additions ): mock_agent = MockAgent([make_response("ok", 0, 0)]) phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue, checkpoint=checkpoint) - monkeypatch.setattr("ddev.ai.phases.agentic_phase.render_memory_prompt", lambda *a, **kw: "USER_ADDITIONS") build_calls: list = [] monkeypatch.setattr( @@ -436,7 +319,7 @@ async def test_run_memory_step_forwards_user_additions_to_build( await phase._run_memory_step(mock_agent, {}) - assert build_calls == [expected_build_arg] + assert build_calls == [expected_user_additions] async def test_run_memory_step_sends_built_prompt_with_no_tools(flow_dir, monkeypatch, message_queue): @@ -444,13 +327,12 @@ async def test_run_memory_step_sends_built_prompt_with_no_tools(flow_dir, monkey class CapturingAgent(MockAgent): async def send(self, content, allowed_tools=None): - captured["content"] = content - captured["allowed_tools"] = allowed_tools + captured.update({"content": content, "allowed_tools": allowed_tools}) return await super().send(content, allowed_tools) agent = CapturingAgent([make_response("ok", 0, 0)]) phase, mgr = make_agent_phase(flow_dir, agent, monkeypatch, message_queue) - monkeypatch.setattr(mgr, "build_memory_prompt", lambda user_additions: "BUILT") + monkeypatch.setattr(mgr, "build_memory_prompt", lambda _: "BUILT") await phase._run_memory_step(agent, {}) @@ -479,24 +361,74 @@ async def _response(response, iteration): # --------------------------------------------------------------------------- -# AgenticPhase.process_message — disk failure regression +# AgenticPhase with spawn_subagent — wiring smoke test # --------------------------------------------------------------------------- -async def test_write_memory_disk_failure_fails_phase(flow_dir, monkeypatch, message_queue): - responses = [ - make_response("task done", 100, 50), - make_response("summary text", 10, 5), - ] - mock_agent = MockAgent(responses) - phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) +async def test_spawn_subagent_wiring(flow_dir, message_queue): + """Phase correctly passes subagent_builder + log_dir to the agent builder at execute time.""" - def raise_permission_error(*args, **kwargs): - raise PermissionError("disk is read-only") + def make_usage() -> TokenUsage: + return TokenUsage(input_tokens=100, output_tokens=50, cache_read_input_tokens=0, cache_creation_input_tokens=0) - monkeypatch.setattr("ddev.ai.phases.checkpoint.CheckpointManager.write_memory", raise_permission_error) + spawn_call = ToolCall( + id="tc1", + name="spawn_subagent", + input={"system_prompt": "you are a helper", "prompt": "answer 42", "tools": [], "name": "child"}, + ) + parent_agent = MockAgent( + [ + AgentResponse(stop_reason=StopReason.TOOL_USE, text="", tool_calls=[spawn_call], usage=make_usage()), + AgentResponse(stop_reason=StopReason.END_TURN, text="parent done", tool_calls=[], usage=make_usage()), + AgentResponse(stop_reason=StopReason.END_TURN, text="memory summary", tool_calls=[], usage=make_usage()), + ] + ) - with pytest.raises(PermissionError, match="disk is read-only"): - await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + subagent_calls: list = [] + + def mock_subagent_builder(system_prompt: str, owner_id: str, tool_names: list[str]): + subagent_calls.append(system_prompt) + return MockAgent( + [AgentResponse(stop_reason=StopReason.END_TURN, text="42", tool_calls=[], usage=make_usage())] + ), ToolRegistry([]) + + from ddev.ai.tools.agents.spawn_subagent import SpawnSubagentTool + + def agent_builder_fn(system_prompt: str, owner_id: str, subagent_builder=None, log_dir=None): + parent_agent.name = owner_id + return parent_agent, ToolRegistry( + [ + SpawnSubagentTool( + owner_id=owner_id, + subagent_builder=subagent_builder, + allowed_tools=[], + log_dir=log_dir, + ) + ] + ) + + checkpoint_manager = CheckpointManager(flow_dir / "checkpoints.yaml") + phase = AgenticPhase( + phase_id="p1", + dependencies=[], + config=PhaseConfig(agent="writer", tasks=[TaskConfig(name="t1", prompt="Do the work.")]), + agent_builder=agent_builder_fn, + checkpoint_manager=checkpoint_manager, + runtime_variables={}, + flow_variables={}, + config_dir=flow_dir, + file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), + subagent_builder=mock_subagent_builder, + ) + phase.queue = message_queue - assert mgr.read() == {} + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + submitted = [message_queue.get_nowait() for _ in range(message_queue.qsize())] + assert not any(isinstance(m, PhaseFailedMessage) for m in submitted) + assert subagent_calls == ["you are a helper"] + + log_file = checkpoint_manager.root / "subagents" / "p1" / "001-child.jsonl" + assert log_file.exists() + events = {e["event"] for e in read_jsonl(log_file)} + assert {"start", "finish"} <= events diff --git a/ddev/tests/ai/tools/agents/__init__.py b/ddev/tests/ai/tools/agents/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/tools/agents/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/tools/agents/test_spawn_subagent.py b/ddev/tests/ai/tools/agents/test_spawn_subagent.py new file mode 100644 index 0000000000000..95c7c3e2d581c --- /dev/null +++ b/ddev/tests/ai/tools/agents/test_spawn_subagent.py @@ -0,0 +1,310 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import asyncio +import json +from pathlib import Path + +import pytest + +from ddev.ai.agent.exceptions import AgentError +from ddev.ai.agent.types import AgentResponse, StopReason, TokenUsage, ToolCall, ToolResultMessage +from ddev.ai.tools.agents.spawn_subagent import SpawnSubagentInput, SpawnSubagentTool +from ddev.ai.tools.core.types import ToolResult + +# --------------------------------------------------------------------------- +# Mock helpers +# --------------------------------------------------------------------------- + + +class MockAgent: + def __init__(self, responses: list[AgentResponse]) -> None: + self._responses = list(responses) + self._index = 0 + self.name = "mock" + self._history: list = [] + + async def send( + self, content: str | list[ToolResultMessage], allowed_tools: list[str] | None = None + ) -> AgentResponse: + response = self._responses[self._index] + self._index += 1 + return response + + def reset(self) -> None: + self._history = [] + + async def compact(self) -> AgentResponse | None: + return None + + async def compact_preserving_last_turn(self) -> AgentResponse | None: + return None + + +class _RaisingAgent: + """Raises a fixed exception on every send() call.""" + + def __init__(self, exc: BaseException) -> None: + self._exc = exc + self.name = "raising" + self._history: list = [] + + async def send(self, content, allowed_tools=None) -> AgentResponse: + raise self._exc + + def reset(self) -> None: + self._history = [] + + async def compact(self) -> AgentResponse | None: + return None + + async def compact_preserving_last_turn(self) -> AgentResponse | None: + return None + + +class MockToolRegistry: + def __init__(self, result: ToolResult | None = None) -> None: + self._result = result or ToolResult(success=True, data="ok") + + @property + def definitions(self) -> list: + return [] + + async def run(self, name: str, raw: dict) -> ToolResult: + return self._result + + +def make_response( + text: str = "", + stop_reason: StopReason = StopReason.END_TURN, + tool_calls: list[ToolCall] | None = None, +) -> AgentResponse: + return AgentResponse( + stop_reason=stop_reason, + text=text, + tool_calls=tool_calls or [], + usage=TokenUsage(input_tokens=10, output_tokens=5, cache_read_input_tokens=0, cache_creation_input_tokens=0), + ) + + +def make_builder(responses: list[AgentResponse], tool_result: ToolResult | None = None): + """Return a builder closure that replays fixed responses.""" + tr = tool_result or ToolResult(success=True, data="ok") + + def builder(system_prompt: str, owner_id: str, tool_names: list[str]): + return MockAgent(list(responses)), MockToolRegistry(tr) + + return builder + + +def make_tool( + log_dir: Path, builder, allowed_tools: list[str] | None = None, owner_id: str = "parent" +) -> SpawnSubagentTool: + return SpawnSubagentTool( + owner_id=owner_id, + subagent_builder=builder, + allowed_tools=allowed_tools if allowed_tools is not None else ["read_file", "edit_file"], + log_dir=log_dir, + ) + + +def read_events(log_path: Path) -> list[dict]: + return [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +# --------------------------------------------------------------------------- +# Input validation — fails before any log file is opened +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "tools,allowed,error_fragment", + [ + (["spawn_subagent"], ["read_file"], "spawn further subagents"), + (["read_file", "edit_file"], ["read_file"], "edit_file"), + ], + ids=["recursive", "disallowed"], +) +async def test_input_validation_fails_before_logging(tmp_path, tools, allowed, error_fragment): + tool = make_tool(tmp_path, make_builder([make_response()]), allowed_tools=allowed) + result = await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=tools, name="x")) + + assert result.success is False + assert error_fragment in result.error + assert "x" in result.error + assert list(tmp_path.glob("*.jsonl")) == [] + assert tool._counter == 0 + + +# --------------------------------------------------------------------------- +# mkdir failure — after validation, before counter advances +# --------------------------------------------------------------------------- + + +async def test_mkdir_failure(tmp_path): + blocker = tmp_path / "blocked" + blocker.write_text("I am a file") + log_dir = blocker / "subagents" + + tool = make_tool(log_dir, make_builder([make_response()])) + result = await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="x")) + + assert result.success is False + assert "x" in result.error + assert str(log_dir) in result.error + assert tool._counter == 0 + + +# --------------------------------------------------------------------------- +# Happy path +# --------------------------------------------------------------------------- + + +async def test_happy_path(tmp_path): + tool = make_tool(tmp_path, make_builder([make_response(text="ok")])) + result = await tool(SpawnSubagentInput(system_prompt="sys", prompt="do it", tools=[], name="worker")) + + assert result.success is True + assert result.data == "ok" + + events = read_events(tmp_path / "001-worker.jsonl") + assert events[0]["event"] == "start" + assert events[-1]["event"] == "finish" + assert events[-1]["success"] is True + + +async def test_multi_iteration_wires_callbacks(tmp_path): + """Proves FileLogger callbacks are wired: a subagent tool call produces a tool_call log event.""" + tool_call = ToolCall(id="tc1", name="read_file", input={"path": "/f"}) + tool = make_tool( + tmp_path, + make_builder( + [make_response(stop_reason=StopReason.TOOL_USE, tool_calls=[tool_call]), make_response(text="done")], + tool_result=ToolResult(success=True, data="content"), + ), + allowed_tools=["read_file"], + ) + + result = await tool(SpawnSubagentInput(system_prompt="sys", prompt="go", tools=["read_file"])) + + assert result.success is True + assert result.data == "done" + assert "tool_call" in [e["event"] for e in read_events(tmp_path / "001-unnamed.jsonl")] + + +async def test_max_tokens_response_prefixed(tmp_path): + tool = make_tool(tmp_path, make_builder([make_response(text="partial", stop_reason=StopReason.MAX_TOKENS)])) + result = await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="mt")) + + assert result.success is True + assert result.data.startswith("[SUBAGENT HIT MAX_TOKENS — RESPONSE MAY BE TRUNCATED]") + assert "partial" in result.data + + finish = next(e for e in read_events(tmp_path / "001-mt.jsonl") if e["event"] == "finish") + assert finish["stop_reason"] == "max_tokens" + + +# --------------------------------------------------------------------------- +# Failure paths +# --------------------------------------------------------------------------- + + +async def test_builder_failure(tmp_path): + def failing_builder(sp, oid, tns): + raise ValueError("boom") + + tool = SpawnSubagentTool(owner_id="parent", subagent_builder=failing_builder, allowed_tools=[], log_dir=tmp_path) + result = await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="fail")) + + assert result.success is False + assert "fail" in result.error and "ValueError" in result.error and "boom" in result.error + + events = read_events(tmp_path / "001-fail.jsonl") + assert [e["event"] for e in events] == ["start", "finish"] + assert events[-1]["success"] is False + + +async def test_react_process_failure(tmp_path): + def builder(sp, oid, tns): + return _RaisingAgent(AgentError("rate limit")), MockToolRegistry() + + tool = make_tool(tmp_path, builder) + result = await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="rl")) + + assert result.success is False + assert "rl" in result.error and "AgentError" in result.error + + names = [e["event"] for e in read_events(tmp_path / "001-rl.jsonl")] + assert "error" in names and "finish" in names + assert names.index("error") < names.index("finish") + assert next(e for e in read_events(tmp_path / "001-rl.jsonl") if e["event"] == "finish")["success"] is False + + +async def test_finally_close_runs_on_base_exception(tmp_path): + """KeyboardInterrupt propagates but logger.close() still runs via finally.""" + + def builder(sp, oid, tns): + return _RaisingAgent(KeyboardInterrupt()), MockToolRegistry() + + tool = make_tool(tmp_path, builder) + + with pytest.raises(KeyboardInterrupt): + await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="ki")) + + names = [e["event"] for e in read_events(tmp_path / "001-ki.jsonl")] + assert "error" in names + assert "finish" not in names + + +# --------------------------------------------------------------------------- +# Counter and log file naming +# --------------------------------------------------------------------------- + + +async def test_counter_increments_per_invocation(tmp_path): + tool = make_tool(tmp_path, make_builder([make_response(text="r1"), make_response(text="r2")])) + + await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="a")) + await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="b")) + + assert (tmp_path / "001-a.jsonl").exists() + assert (tmp_path / "002-b.jsonl").exists() + + +async def test_parallel_spawns_get_distinct_counters(tmp_path): + owner_ids: list[str] = [] + + def recording_builder(sp: str, owner_id: str, tns: list[str]): + owner_ids.append(owner_id) + return MockAgent([make_response(text="ok")]), MockToolRegistry() + + tool = SpawnSubagentTool(owner_id="parent", subagent_builder=recording_builder, allowed_tools=[], log_dir=tmp_path) + results = await asyncio.gather( + *[tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name=n)) for n in ["x", "y", "z"]] + ) + + assert all(r.success for r in results) + assert len(list(tmp_path.glob("*.jsonl"))) == 3 + assert len(set(owner_ids)) == 3 + + +# --------------------------------------------------------------------------- +# Pydantic input validation (via BaseTool.run) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "raw", + [ + {"prompt": "x"}, + {"system_prompt": "s", "prompt": "p", "tools": [], "bad_field": True}, + ], + ids=["missing_field", "extra_field"], +) +async def test_pydantic_rejects_invalid_input(raw): + tool = SpawnSubagentTool( + owner_id="p", subagent_builder=lambda *a: (None, None), allowed_tools=[], log_dir=Path("/tmp") + ) + result = await tool.run(raw) + assert result.success is False diff --git a/ddev/tests/ai/tools/test_registry.py b/ddev/tests/ai/tools/test_registry.py index e307773f5b7b6..a8e3f4f40ea3e 100644 --- a/ddev/tests/ai/tools/test_registry.py +++ b/ddev/tests/ai/tools/test_registry.py @@ -153,6 +153,7 @@ def test_available_tool_names_returns_fresh_copy(): OWNER_ID = "test-agent" +TOOLS_WITHOUT_EXTRA_DEPS = [n for n in ToolRegistry.available_tool_names() if n != "spawn_subagent"] def test_from_names_empty(tmp_path): @@ -169,7 +170,7 @@ def test_from_names_unknown_raises(tmp_path): ) -@pytest.mark.parametrize("name", ToolRegistry.available_tool_names()) +@pytest.mark.parametrize("name", TOOLS_WITHOUT_EXTRA_DEPS) def test_from_names_each_known_tool(name, tmp_path): registry = ToolRegistry.from_names( [name], owner_id=OWNER_ID, file_registry=FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) @@ -179,7 +180,7 @@ def test_from_names_each_known_tool(name, tmp_path): def test_from_names_all_at_once(tmp_path): - all_names = ToolRegistry.available_tool_names() + all_names = TOOLS_WITHOUT_EXTRA_DEPS registry = ToolRegistry.from_names( all_names, owner_id=OWNER_ID, file_registry=FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) ) @@ -187,9 +188,18 @@ def test_from_names_all_at_once(tmp_path): assert built_names == set(all_names) +def test_from_names_spawn_subagent_without_deps_raises(tmp_path): + with pytest.raises(ValueError, match="requires both 'subagent_builder' and 'log_dir'"): + ToolRegistry.from_names( + ["spawn_subagent"], + owner_id=OWNER_ID, + file_registry=FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)), + ) + + def test_from_names_fs_tools_share_file_registry(tmp_path): """All tools that use the file registry in the same ToolRegistry share a single instance.""" - all_names = ToolRegistry.available_tool_names() + all_names = TOOLS_WITHOUT_EXTRA_DEPS registry = ToolRegistry.from_names( all_names, owner_id=OWNER_ID, file_registry=FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) ) From ffbe46fed59e523a3e2f4a426344d897c0e43c81 Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Fri, 22 May 2026 11:01:01 +0200 Subject: [PATCH 38/44] Move Filelogger to tools/agents/ and rename it to Agentlogger --- .../agents/agent_logger.py} | 2 +- ddev/src/ddev/ai/tools/agents/spawn_subagent.py | 4 ++-- .../agents/test_agent_logger.py} | 14 +++++++------- ddev/tests/ai/tools/agents/test_spawn_subagent.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) rename ddev/src/ddev/ai/{callbacks/file_logger.py => tools/agents/agent_logger.py} (99%) rename ddev/tests/ai/{callbacks/test_file_logger.py => tools/agents/test_agent_logger.py} (92%) diff --git a/ddev/src/ddev/ai/callbacks/file_logger.py b/ddev/src/ddev/ai/tools/agents/agent_logger.py similarity index 99% rename from ddev/src/ddev/ai/callbacks/file_logger.py rename to ddev/src/ddev/ai/tools/agents/agent_logger.py index 2e58e4057e915..14a20a43d567a 100644 --- a/ddev/src/ddev/ai/callbacks/file_logger.py +++ b/ddev/src/ddev/ai/tools/agents/agent_logger.py @@ -12,7 +12,7 @@ from ddev.ai.tools.core.types import ToolResult -class FileLogger: +class AgentLogger: """Append-only JSONL writer for ReAct events plus subagent start/finish bookkeeping. Owns the file handle. Call build_callbacks() to obtain a Callbacks object whose diff --git a/ddev/src/ddev/ai/tools/agents/spawn_subagent.py b/ddev/src/ddev/ai/tools/agents/spawn_subagent.py index ec9b03f66cd85..ce714d1350013 100644 --- a/ddev/src/ddev/ai/tools/agents/spawn_subagent.py +++ b/ddev/src/ddev/ai/tools/agents/spawn_subagent.py @@ -9,8 +9,8 @@ from ddev.ai.agent.build import SubagentBuilder from ddev.ai.agent.types import StopReason -from ddev.ai.callbacks.file_logger import FileLogger from ddev.ai.react.process import ReActProcess +from ddev.ai.tools.agents.agent_logger import AgentLogger from ddev.ai.tools.core.base import BaseTool, BaseToolInput from ddev.ai.tools.core.types import ToolResult @@ -105,7 +105,7 @@ async def __call__(self, tool_input: SpawnSubagentInput) -> ToolResult: subagent_id = f"{self._owner_id}.sub.{self._counter:03d}-{label}" log_path = self._log_dir / f"{self._counter:03d}-{label}.jsonl" - logger = FileLogger(log_path) + logger = AgentLogger(log_path) try: logger.log_start( system_prompt=tool_input.system_prompt, diff --git a/ddev/tests/ai/callbacks/test_file_logger.py b/ddev/tests/ai/tools/agents/test_agent_logger.py similarity index 92% rename from ddev/tests/ai/callbacks/test_file_logger.py rename to ddev/tests/ai/tools/agents/test_agent_logger.py index 1b4f52bf251cb..808e9a84e44da 100644 --- a/ddev/tests/ai/callbacks/test_file_logger.py +++ b/ddev/tests/ai/tools/agents/test_agent_logger.py @@ -7,7 +7,7 @@ import pytest from ddev.ai.agent.types import AgentResponse, StopReason, TokenUsage, ToolCall -from ddev.ai.callbacks.file_logger import FileLogger +from ddev.ai.tools.agents.agent_logger import AgentLogger from ddev.ai.tools.core.types import ToolResult @@ -31,7 +31,7 @@ def read_events(log_path) -> list[dict]: def test_log_entries_are_valid_jsonl_with_timestamp(tmp_path): log_path = tmp_path / "log.jsonl" - logger = FileLogger(log_path) + logger = AgentLogger(log_path) logger.log_start(system_prompt="sys", prompt="go", tools=["read_file"]) logger.log_finish(success=True, iterations=1) logger.close() @@ -45,7 +45,7 @@ def test_log_entries_are_valid_jsonl_with_timestamp(tmp_path): def test_flush_after_each_write(tmp_path): log_path = tmp_path / "log.jsonl" - logger = FileLogger(log_path) + logger = AgentLogger(log_path) logger.log_start(system_prompt="s", prompt="p", tools=[]) # A second file handle reads the line without closing the logger first assert log_path.read_text(encoding="utf-8").strip() != "" @@ -54,7 +54,7 @@ def test_flush_after_each_write(tmp_path): def test_close_is_idempotent_and_prevents_further_writes(tmp_path): log_path = tmp_path / "log.jsonl" - logger = FileLogger(log_path) + logger = AgentLogger(log_path) logger.log_start(system_prompt="s", prompt="p", tools=[]) logger.close() logger.close() # must not raise @@ -64,12 +64,12 @@ def test_close_is_idempotent_and_prevents_further_writes(tmp_path): def test_constructor_requires_existing_parent(tmp_path): with pytest.raises(OSError): - FileLogger(tmp_path / "doesnotexist" / "log.jsonl") + AgentLogger(tmp_path / "doesnotexist" / "log.jsonl") def test_non_serializable_values_use_str_repr(tmp_path): log_path = tmp_path / "log.jsonl" - logger = FileLogger(log_path) + logger = AgentLogger(log_path) class Unserializable: def __repr__(self): @@ -88,7 +88,7 @@ def __repr__(self): async def test_build_callbacks_fires_all_event_types(tmp_path): log_path = tmp_path / "log.jsonl" - logger = FileLogger(log_path) + logger = AgentLogger(log_path) callbacks = logger.build_callbacks() tool_call = ToolCall(id="tc1", name="read_file", input={"path": "/f"}) diff --git a/ddev/tests/ai/tools/agents/test_spawn_subagent.py b/ddev/tests/ai/tools/agents/test_spawn_subagent.py index 95c7c3e2d581c..f2fbca64f6d45 100644 --- a/ddev/tests/ai/tools/agents/test_spawn_subagent.py +++ b/ddev/tests/ai/tools/agents/test_spawn_subagent.py @@ -175,7 +175,7 @@ async def test_happy_path(tmp_path): async def test_multi_iteration_wires_callbacks(tmp_path): - """Proves FileLogger callbacks are wired: a subagent tool call produces a tool_call log event.""" + """Proves AgentLogger callbacks are wired: a subagent tool call produces a tool_call log event.""" tool_call = ToolCall(id="tc1", name="read_file", input={"path": "/f"}) tool = make_tool( tmp_path, From 2423dd19e22390168d16e85e6ccc1219f6f90d58 Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Fri, 22 May 2026 12:29:39 +0200 Subject: [PATCH 39/44] Few bugs fixed --- ddev/src/ddev/ai/phases/agentic_phase.py | 23 +++-- .../ddev/ai/tools/agents/spawn_subagent.py | 1 + ddev/src/ddev/ai/tools/registry.py | 9 +- ddev/tests/ai/phases/test_agentic_phase.py | 34 ++++++- .../ai/tools/agents/test_agent_logger.py | 22 ++++- .../ai/tools/agents/test_spawn_subagent.py | 90 ++++++++++++++----- 6 files changed, 145 insertions(+), 34 deletions(-) diff --git a/ddev/src/ddev/ai/phases/agentic_phase.py b/ddev/src/ddev/ai/phases/agentic_phase.py index 25431f7a44c20..ab98e38722b45 100644 --- a/ddev/src/ddev/ai/phases/agentic_phase.py +++ b/ddev/src/ddev/ai/phases/agentic_phase.py @@ -16,6 +16,7 @@ from ddev.ai.phases.template import render_inline, render_prompt from ddev.ai.react.process import ReActProcess from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.registry import TOOL_MANIFEST def render_task_prompt( @@ -77,6 +78,9 @@ def __init__( ) self._agent_builder = agent_builder self._subagent_builder = subagent_builder + self._subagent_log_dir = ( + checkpoint_manager.root / "subagents" / phase_id if subagent_builder is not None else None + ) @classmethod def validate_config( @@ -108,8 +112,12 @@ def extra_init_kwargs( # type: ignore[override] agent_config = agents[phase_config.agent] subagent_builder = None - # TODO: generalize this dispatch if more agent-meta tools appear in tools/agents/. - if "spawn_subagent" in agent_config.tools: + requires_subagent_builder = any( + spec.requires_subagent_builder + for name in agent_config.tools + if (spec := TOOL_MANIFEST.get(name)) is not None + ) + if requires_subagent_builder: subagent_builder = make_subagent_builder( parent_agent_config=agent_config, agent_clients=agent_clients, @@ -162,11 +170,12 @@ def _build_agent_and_process(self, context: dict[str, Any]) -> tuple[BaseAgent[A context, self._resolver, ) - log_dir = None - if self._subagent_builder is not None: - log_dir = self._checkpoint_manager.root / "subagents" / self._phase_id - - agent, tool_registry = self._agent_builder(system_prompt, self._phase_id, self._subagent_builder, log_dir) + agent, tool_registry = self._agent_builder( + system_prompt, + self._phase_id, + self._subagent_builder, + self._subagent_log_dir, + ) process = ReActProcess( agent=agent, tool_registry=tool_registry, diff --git a/ddev/src/ddev/ai/tools/agents/spawn_subagent.py b/ddev/src/ddev/ai/tools/agents/spawn_subagent.py index ce714d1350013..2e126da14ee4d 100644 --- a/ddev/src/ddev/ai/tools/agents/spawn_subagent.py +++ b/ddev/src/ddev/ai/tools/agents/spawn_subagent.py @@ -38,6 +38,7 @@ class SpawnSubagentInput(BaseToolInput): str | None, Field( description=("Optional short human-readable name for the subagent."), + pattern=r"^$|^[A-Za-z0-9._-]{1,64}$", ), ] = None diff --git a/ddev/src/ddev/ai/tools/registry.py b/ddev/src/ddev/ai/tools/registry.py index 828694eed6c11..48b99e8f065da 100644 --- a/ddev/src/ddev/ai/tools/registry.py +++ b/ddev/src/ddev/ai/tools/registry.py @@ -70,11 +70,13 @@ class ToolSpec: ``module`` is relative to the registry's package (e.g. ``"fs.read_file"``). ``factory`` receives the already-imported class and the shared ToolContext and returns a constructed tool instance. + ``requires_subagent_builder`` marks agentic tools that need subagent wiring. """ module: str cls: str factory: Callable[[type, ToolContext], ToolProtocol] = _plain_factory + requires_subagent_builder: bool = False TOOL_MANIFEST: dict[str, ToolSpec] = { @@ -94,7 +96,12 @@ class ToolSpec: "ddev_env_test": ToolSpec("shell.ddev.env_test", "DdevEnvTestTool"), "ddev_release_changelog": ToolSpec("shell.ddev.release_changelog", "DdevReleaseChangelogTool"), "ddev_validate": ToolSpec("shell.ddev.validate", "DdevValidateTool"), - "spawn_subagent": ToolSpec("agents.spawn_subagent", "SpawnSubagentTool", factory=_spawn_subagent_factory), + "spawn_subagent": ToolSpec( + "agents.spawn_subagent", + "SpawnSubagentTool", + factory=_spawn_subagent_factory, + requires_subagent_builder=True, + ), } diff --git a/ddev/tests/ai/phases/test_agentic_phase.py b/ddev/tests/ai/phases/test_agentic_phase.py index bc333f965aee9..36d4701b5b353 100644 --- a/ddev/tests/ai/phases/test_agentic_phase.py +++ b/ddev/tests/ai/phases/test_agentic_phase.py @@ -365,6 +365,27 @@ async def _response(response, iteration): # --------------------------------------------------------------------------- +@pytest.mark.parametrize( + ("tools", "expected"), + [(["spawn_subagent"], True), (["read_file"], False), ([], False)], + ids=["spawn", "regular_tool", "no_tools"], +) +def test_extra_init_kwargs_creates_subagent_builder_from_tool_metadata( + flow_dir: Path, + tools: list[str], + expected: bool, +) -> None: + kwargs = AgenticPhase.extra_init_kwargs( + phase_id="p1", + phase_config=PhaseConfig(agent="writer", tasks=[TaskConfig(name="t1", prompt="Do the work.")]), + agents={"writer": AgentConfig(tools=tools)}, + agent_clients={}, + file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), + ) + + assert (kwargs["subagent_builder"] is not None) is expected + + async def test_spawn_subagent_wiring(flow_dir, message_queue): """Phase correctly passes subagent_builder + log_dir to the agent builder at execute time.""" @@ -394,7 +415,17 @@ def mock_subagent_builder(system_prompt: str, owner_id: str, tool_names: list[st from ddev.ai.tools.agents.spawn_subagent import SpawnSubagentTool - def agent_builder_fn(system_prompt: str, owner_id: str, subagent_builder=None, log_dir=None): + captured_log_dirs: list[Path | None] = [] + + def agent_builder_fn( + system_prompt: str, + owner_id: str, + subagent_builder=None, + log_dir: Path | None = None, + ): + captured_log_dirs.append(log_dir) + assert subagent_builder is not None + assert log_dir is not None parent_agent.name = owner_id return parent_agent, ToolRegistry( [ @@ -427,6 +458,7 @@ def agent_builder_fn(system_prompt: str, owner_id: str, subagent_builder=None, l submitted = [message_queue.get_nowait() for _ in range(message_queue.qsize())] assert not any(isinstance(m, PhaseFailedMessage) for m in submitted) assert subagent_calls == ["you are a helper"] + assert captured_log_dirs == [checkpoint_manager.root / "subagents" / "p1"] log_file = checkpoint_manager.root / "subagents" / "p1" / "001-child.jsonl" assert log_file.exists() diff --git a/ddev/tests/ai/tools/agents/test_agent_logger.py b/ddev/tests/ai/tools/agents/test_agent_logger.py index 808e9a84e44da..7cbd58767c859 100644 --- a/ddev/tests/ai/tools/agents/test_agent_logger.py +++ b/ddev/tests/ai/tools/agents/test_agent_logger.py @@ -52,7 +52,7 @@ def test_flush_after_each_write(tmp_path): logger.close() -def test_close_is_idempotent_and_prevents_further_writes(tmp_path): +def test_close_is_idempotent_and_prevents_further_writes(tmp_path, caplog): log_path = tmp_path / "log.jsonl" logger = AgentLogger(log_path) logger.log_start(system_prompt="s", prompt="p", tools=[]) @@ -60,6 +60,26 @@ def test_close_is_idempotent_and_prevents_further_writes(tmp_path): logger.close() # must not raise logger.log_finish(success=False) # must not write assert len(read_events(log_path)) == 1 + assert "dropping event 'finish'" in caplog.text + + +def test_reopening_same_path_appends_start_run_delimiter(tmp_path): + log_path = tmp_path / "log.jsonl" + + logger = AgentLogger(log_path) + logger.log_start(system_prompt="s", prompt="first", tools=[]) + logger.log_finish(success=True) + logger.close() + + logger = AgentLogger(log_path) + logger.log_start(system_prompt="s", prompt="second", tools=[]) + logger.log_finish(success=True) + logger.close() + + events = read_events(log_path) + assert [event["event"] for event in events] == ["start", "finish", "start", "finish"] + assert events[0]["prompt"] == "first" + assert events[2]["prompt"] == "second" def test_constructor_requires_existing_parent(tmp_path): diff --git a/ddev/tests/ai/tools/agents/test_spawn_subagent.py b/ddev/tests/ai/tools/agents/test_spawn_subagent.py index f2fbca64f6d45..9856169722463 100644 --- a/ddev/tests/ai/tools/agents/test_spawn_subagent.py +++ b/ddev/tests/ai/tools/agents/test_spawn_subagent.py @@ -5,28 +5,34 @@ import asyncio import json from pathlib import Path +from typing import Any import pytest +from anthropic.types import ToolParam +from ddev.ai.agent.base import BaseAgent +from ddev.ai.agent.build import SubagentBuilder from ddev.ai.agent.exceptions import AgentError from ddev.ai.agent.types import AgentResponse, StopReason, TokenUsage, ToolCall, ToolResultMessage from ddev.ai.tools.agents.spawn_subagent import SpawnSubagentInput, SpawnSubagentTool from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.registry import ToolRegistry # --------------------------------------------------------------------------- # Mock helpers # --------------------------------------------------------------------------- -class MockAgent: +class MockAgent(BaseAgent[Any]): def __init__(self, responses: list[AgentResponse]) -> None: + super().__init__("mock", "", ToolRegistry([])) self._responses = list(responses) self._index = 0 - self.name = "mock" - self._history: list = [] async def send( - self, content: str | list[ToolResultMessage], allowed_tools: list[str] | None = None + self, + content: str | list[ToolResultMessage], + allowed_tools: list[str] | None = None, ) -> AgentResponse: response = self._responses[self._index] self._index += 1 @@ -42,15 +48,18 @@ async def compact_preserving_last_turn(self) -> AgentResponse | None: return None -class _RaisingAgent: +class _RaisingAgent(BaseAgent[Any]): """Raises a fixed exception on every send() call.""" def __init__(self, exc: BaseException) -> None: + super().__init__("raising", "", ToolRegistry([])) self._exc = exc - self.name = "raising" - self._history: list = [] - async def send(self, content, allowed_tools=None) -> AgentResponse: + async def send( + self, + content: str | list[ToolResultMessage], + allowed_tools: list[str] | None = None, + ) -> AgentResponse: raise self._exc def reset(self) -> None: @@ -63,15 +72,16 @@ async def compact_preserving_last_turn(self) -> AgentResponse | None: return None -class MockToolRegistry: +class MockToolRegistry(ToolRegistry): def __init__(self, result: ToolResult | None = None) -> None: + super().__init__([]) self._result = result or ToolResult(success=True, data="ok") @property - def definitions(self) -> list: + def definitions(self) -> list[ToolParam]: return [] - async def run(self, name: str, raw: dict) -> ToolResult: + async def run(self, name: str, raw: dict[str, object]) -> ToolResult: return self._result @@ -88,18 +98,21 @@ def make_response( ) -def make_builder(responses: list[AgentResponse], tool_result: ToolResult | None = None): +def make_builder(responses: list[AgentResponse], tool_result: ToolResult | None = None) -> SubagentBuilder: """Return a builder closure that replays fixed responses.""" tr = tool_result or ToolResult(success=True, data="ok") - def builder(system_prompt: str, owner_id: str, tool_names: list[str]): + def builder(system_prompt: str, owner_id: str, tool_names: list[str]) -> tuple[BaseAgent[Any], ToolRegistry]: return MockAgent(list(responses)), MockToolRegistry(tr) return builder def make_tool( - log_dir: Path, builder, allowed_tools: list[str] | None = None, owner_id: str = "parent" + log_dir: Path, + builder: SubagentBuilder, + allowed_tools: list[str] | None = None, + owner_id: str = "parent", ) -> SpawnSubagentTool: return SpawnSubagentTool( owner_id=owner_id, @@ -161,14 +174,23 @@ async def test_mkdir_failure(tmp_path): # --------------------------------------------------------------------------- -async def test_happy_path(tmp_path): +@pytest.mark.parametrize( + ("name", "expected_log_name"), + [ + ("worker", "001-worker.jsonl"), + (None, "001-unnamed.jsonl"), + ("", "001-unnamed.jsonl"), + ], + ids=["named", "none", "empty"], +) +async def test_happy_path(tmp_path: Path, name: str | None, expected_log_name: str) -> None: tool = make_tool(tmp_path, make_builder([make_response(text="ok")])) - result = await tool(SpawnSubagentInput(system_prompt="sys", prompt="do it", tools=[], name="worker")) + result = await tool(SpawnSubagentInput(system_prompt="sys", prompt="do it", tools=[], name=name)) assert result.success is True assert result.data == "ok" - events = read_events(tmp_path / "001-worker.jsonl") + events = read_events(tmp_path / expected_log_name) assert events[0]["event"] == "start" assert events[-1]["event"] == "finish" assert events[-1]["success"] is True @@ -211,7 +233,11 @@ async def test_max_tokens_response_prefixed(tmp_path): async def test_builder_failure(tmp_path): - def failing_builder(sp, oid, tns): + def failing_builder( + system_prompt: str, + owner_id: str, + tool_names: list[str], + ) -> tuple[BaseAgent[Any], ToolRegistry]: raise ValueError("boom") tool = SpawnSubagentTool(owner_id="parent", subagent_builder=failing_builder, allowed_tools=[], log_dir=tmp_path) @@ -226,7 +252,11 @@ def failing_builder(sp, oid, tns): async def test_react_process_failure(tmp_path): - def builder(sp, oid, tns): + def builder( + system_prompt: str, + owner_id: str, + tool_names: list[str], + ) -> tuple[BaseAgent[Any], ToolRegistry]: return _RaisingAgent(AgentError("rate limit")), MockToolRegistry() tool = make_tool(tmp_path, builder) @@ -244,7 +274,11 @@ def builder(sp, oid, tns): async def test_finally_close_runs_on_base_exception(tmp_path): """KeyboardInterrupt propagates but logger.close() still runs via finally.""" - def builder(sp, oid, tns): + def builder( + system_prompt: str, + owner_id: str, + tool_names: list[str], + ) -> tuple[BaseAgent[Any], ToolRegistry]: return _RaisingAgent(KeyboardInterrupt()), MockToolRegistry() tool = make_tool(tmp_path, builder) @@ -275,7 +309,11 @@ async def test_counter_increments_per_invocation(tmp_path): async def test_parallel_spawns_get_distinct_counters(tmp_path): owner_ids: list[str] = [] - def recording_builder(sp: str, owner_id: str, tns: list[str]): + def recording_builder( + system_prompt: str, + owner_id: str, + tool_names: list[str], + ) -> tuple[BaseAgent[Any], ToolRegistry]: owner_ids.append(owner_id) return MockAgent([make_response(text="ok")]), MockToolRegistry() @@ -299,12 +337,16 @@ def recording_builder(sp: str, owner_id: str, tns: list[str]): [ {"prompt": "x"}, {"system_prompt": "s", "prompt": "p", "tools": [], "bad_field": True}, + {"system_prompt": "s", "prompt": "p", "tools": [], "name": "../oops"}, ], - ids=["missing_field", "extra_field"], + ids=["missing_field", "extra_field", "unsafe_name"], ) -async def test_pydantic_rejects_invalid_input(raw): +async def test_pydantic_rejects_invalid_input(raw: dict[str, object]) -> None: tool = SpawnSubagentTool( - owner_id="p", subagent_builder=lambda *a: (None, None), allowed_tools=[], log_dir=Path("/tmp") + owner_id="p", + subagent_builder=make_builder([make_response()]), + allowed_tools=[], + log_dir=Path("/tmp"), ) result = await tool.run(raw) assert result.success is False From 1bdbad81aec5ae4a7d2e352a00d178076fcd499f Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Fri, 22 May 2026 12:53:39 +0200 Subject: [PATCH 40/44] Fix bug in test --- ddev/tests/ai/tools/agents/test_agent_logger.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ddev/tests/ai/tools/agents/test_agent_logger.py b/ddev/tests/ai/tools/agents/test_agent_logger.py index 7cbd58767c859..6f15343fad07c 100644 --- a/ddev/tests/ai/tools/agents/test_agent_logger.py +++ b/ddev/tests/ai/tools/agents/test_agent_logger.py @@ -52,7 +52,7 @@ def test_flush_after_each_write(tmp_path): logger.close() -def test_close_is_idempotent_and_prevents_further_writes(tmp_path, caplog): +def test_close_is_idempotent_and_prevents_further_writes(tmp_path): log_path = tmp_path / "log.jsonl" logger = AgentLogger(log_path) logger.log_start(system_prompt="s", prompt="p", tools=[]) @@ -60,7 +60,6 @@ def test_close_is_idempotent_and_prevents_further_writes(tmp_path, caplog): logger.close() # must not raise logger.log_finish(success=False) # must not write assert len(read_events(log_path)) == 1 - assert "dropping event 'finish'" in caplog.text def test_reopening_same_path_appends_start_run_delimiter(tmp_path): From 7375f938ebebcface4a0df99f21530d1c39c800a Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Fri, 22 May 2026 15:21:47 +0200 Subject: [PATCH 41/44] Make subagents share same Fileregistry as their parent --- ddev/src/ddev/ai/agent/build.py | 11 ++++---- ddev/src/ddev/ai/phases/agentic_phase.py | 2 +- ddev/tests/ai/agent/test_build.py | 34 +++++++++++++++++------- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/ddev/src/ddev/ai/agent/build.py b/ddev/src/ddev/ai/agent/build.py index 6ab2e30fb9428..28a56e620bbbc 100644 --- a/ddev/src/ddev/ai/agent/build.py +++ b/ddev/src/ddev/ai/agent/build.py @@ -10,7 +10,6 @@ from ddev.ai.agent.anthropic_client import AnthropicAgent from ddev.ai.agent.base import BaseAgent -from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy from ddev.ai.tools.fs.file_registry import FileRegistry from ddev.ai.tools.registry import ToolRegistry @@ -95,12 +94,12 @@ def build_agent( def build_subagent( parent_agent_config: AgentConfig, agent_clients: dict[str, Any], - file_access_policy: FileAccessPolicy, + file_registry: FileRegistry, system_prompt: str, owner_id: str, tool_names: list[str], ) -> tuple[BaseAgent[Any], ToolRegistry]: - """Build a subagent + ToolRegistry. Always uses a fresh FileRegistry. + """Build a subagent + ToolRegistry using the shared FileRegistry. Reuses the parent's provider/model/max_tokens. No subagent_builder or log_dir is forwarded, so the subagent cannot recursively spawn subagents — @@ -112,7 +111,7 @@ def build_subagent( system_prompt=system_prompt, owner_id=owner_id, tool_names=tool_names, - file_registry=FileRegistry(policy=file_access_policy), + file_registry=file_registry, ) @@ -145,7 +144,7 @@ def builder( def make_subagent_builder( parent_agent_config: AgentConfig, agent_clients: dict[str, Any], - file_access_policy: FileAccessPolicy, + file_registry: FileRegistry, ) -> SubagentBuilder: """Return a closure that builds a subagent+registry given (system_prompt, owner_id, tool_names).""" @@ -153,7 +152,7 @@ def builder(system_prompt: str, owner_id: str, tool_names: list[str]) -> tuple[B return build_subagent( parent_agent_config=parent_agent_config, agent_clients=agent_clients, - file_access_policy=file_access_policy, + file_registry=file_registry, system_prompt=system_prompt, owner_id=owner_id, tool_names=tool_names, diff --git a/ddev/src/ddev/ai/phases/agentic_phase.py b/ddev/src/ddev/ai/phases/agentic_phase.py index ab98e38722b45..948bcfb3e7728 100644 --- a/ddev/src/ddev/ai/phases/agentic_phase.py +++ b/ddev/src/ddev/ai/phases/agentic_phase.py @@ -121,7 +121,7 @@ def extra_init_kwargs( # type: ignore[override] subagent_builder = make_subagent_builder( parent_agent_config=agent_config, agent_clients=agent_clients, - file_access_policy=file_registry.policy, + file_registry=file_registry, ) return { diff --git a/ddev/tests/ai/agent/test_build.py b/ddev/tests/ai/agent/test_build.py index 9cff25a6f8881..2ee6a42762bf6 100644 --- a/ddev/tests/ai/agent/test_build.py +++ b/ddev/tests/ai/agent/test_build.py @@ -80,19 +80,33 @@ def test_build_agent_uses_config_tools(file_registry, clients): # --------------------------------------------------------------------------- -def test_build_subagent_creates_fresh_file_registry(policy, clients): +def test_build_subagent_reuses_shared_file_registry(file_registry, clients): config = AgentConfig(provider="anthropic", tools=[]) - caller_registry = FileRegistry(policy=policy) - _, reg_a = build_subagent(config, clients, policy, "sys", "a", []) - _, reg_b = build_subagent(config, clients, policy, "sys", "b", []) - assert reg_a is not reg_b - assert reg_a is not caller_registry + _, registry = build_subagent(config, clients, file_registry, "sys", "child", ["read_file", "edit_file"]) + for tool in registry._tools.values(): + assert tool._registry is file_registry + assert tool._owner_id == "child" -def test_build_subagent_recursion_guard(policy, clients): + +def test_build_subagent_recursion_guard(file_registry, clients): config = AgentConfig.model_construct(provider="anthropic", tools=[]) with pytest.raises(ValueError): - build_subagent(config, clients, policy, "sys", "sub", ["spawn_subagent"]) + build_subagent(config, clients, file_registry, "sys", "sub", ["spawn_subagent"]) + + +async def test_shared_registry_does_not_share_parent_read_authorization(file_registry, clients, tmp_path): + config = AgentConfig(provider="anthropic", tools=[]) + path = tmp_path / "file.txt" + path.write_text("before", encoding="utf-8") + file_registry.record("parent", str(path), "before") + + _, registry = build_subagent(config, clients, file_registry, "sys", "parent.sub.001-child", ["edit_file"]) + result = await registry.run("edit_file", {"path": str(path), "old_string": "before", "new_string": "after"}) + + assert result.success is False + assert "Not authorized" in result.error + assert path.read_text(encoding="utf-8") == "before" # --------------------------------------------------------------------------- @@ -108,9 +122,9 @@ def test_make_agent_builder(file_registry, clients): assert agent.name == "p1" -def test_make_subagent_builder(policy, clients): +def test_make_subagent_builder(file_registry, clients): config = AgentConfig(provider="anthropic", tools=[]) - builder = make_subagent_builder(config, clients, policy) + builder = make_subagent_builder(config, clients, file_registry) agent, registry = builder("sys", "sub-1", []) assert isinstance(agent, AnthropicAgent) assert agent.name == "sub-1" From 0a23136acc3091e2a62aae0e88fcfdf44188cbdd Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Fri, 22 May 2026 16:41:05 +0200 Subject: [PATCH 42/44] Keep track of subagents tokens --- ddev/src/ddev/ai/react/process.py | 2 ++ ddev/src/ddev/ai/tools/agents/spawn_subagent.py | 7 ++++++- ddev/src/ddev/ai/tools/core/types.py | 2 ++ ddev/tests/ai/react/test_process.py | 14 ++++++++++++++ ddev/tests/ai/tools/agents/test_spawn_subagent.py | 2 ++ 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/ddev/src/ddev/ai/react/process.py b/ddev/src/ddev/ai/react/process.py index 4ed620947fc63..c925af8e31e6a 100644 --- a/ddev/src/ddev/ai/react/process.py +++ b/ddev/src/ddev/ai/react/process.py @@ -113,6 +113,8 @@ async def start(self, prompt: str, allowed_tools: list[str] | None = None) -> Re r if isinstance(r, ToolResult) else ToolResult(success=False, error=f"{type(r).__name__}: {r}") for r in raw_results ] + total_input += sum(result.total_input_tokens for result in tool_results) + total_output += sum(result.total_output_tokens for result in tool_results) tool_call_results = list(zip(response.tool_calls, tool_results, strict=True)) diff --git a/ddev/src/ddev/ai/tools/agents/spawn_subagent.py b/ddev/src/ddev/ai/tools/agents/spawn_subagent.py index 2e126da14ee4d..d73b085d0055a 100644 --- a/ddev/src/ddev/ai/tools/agents/spawn_subagent.py +++ b/ddev/src/ddev/ai/tools/agents/spawn_subagent.py @@ -152,6 +152,11 @@ async def __call__(self, tool_input: SpawnSubagentInput) -> ToolResult: data = result.final_response.text if result.final_response.stop_reason == StopReason.MAX_TOKENS: data = "[SUBAGENT HIT MAX_TOKENS — RESPONSE MAY BE TRUNCATED]\n\n" + data - return ToolResult(success=True, data=data) + return ToolResult( + success=True, + data=data, + total_input_tokens=result.total_input_tokens, + total_output_tokens=result.total_output_tokens, + ) finally: logger.close() diff --git a/ddev/src/ddev/ai/tools/core/types.py b/ddev/src/ddev/ai/tools/core/types.py index 1e5c89c9929c7..b5a0fc413d492 100644 --- a/ddev/src/ddev/ai/tools/core/types.py +++ b/ddev/src/ddev/ai/tools/core/types.py @@ -15,3 +15,5 @@ class ToolResult(BaseModel): total_size: int | None = None shown_size: int | None = None hint: str | None = None + total_input_tokens: int = 0 + total_output_tokens: int = 0 diff --git a/ddev/tests/ai/react/test_process.py b/ddev/tests/ai/react/test_process.py index 898a983ab4074..e29e6af80b70a 100644 --- a/ddev/tests/ai/react/test_process.py +++ b/ddev/tests/ai/react/test_process.py @@ -494,6 +494,20 @@ async def test_total_tokens_summed_across_iterations() -> None: assert result.iterations == 2 +async def test_tool_result_tokens_included_in_total_tokens() -> None: + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[make_tool_call()], input_tokens=100, output_tokens=50), + make_response(StopReason.END_TURN, input_tokens=200, output_tokens=80), + ] + agent = MockAgent(responses) + registry = MockToolRegistry(ToolResult(success=True, data="ok", total_input_tokens=30, total_output_tokens=10)) + + result = await make_process(agent, registry=registry).start("Task") + + assert result.total_input_tokens == 330 + assert result.total_output_tokens == 140 + + # --------------------------------------------------------------------------- # Context usage propagation — parametrized None vs present # --------------------------------------------------------------------------- diff --git a/ddev/tests/ai/tools/agents/test_spawn_subagent.py b/ddev/tests/ai/tools/agents/test_spawn_subagent.py index 9856169722463..b1d3cecd0e6c0 100644 --- a/ddev/tests/ai/tools/agents/test_spawn_subagent.py +++ b/ddev/tests/ai/tools/agents/test_spawn_subagent.py @@ -189,6 +189,8 @@ async def test_happy_path(tmp_path: Path, name: str | None, expected_log_name: s assert result.success is True assert result.data == "ok" + assert result.total_input_tokens == 10 + assert result.total_output_tokens == 5 events = read_events(tmp_path / expected_log_name) assert events[0]["event"] == "start" From eb0d2c06350a19042934c7530862f0898432cc0b Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Fri, 22 May 2026 16:47:37 +0200 Subject: [PATCH 43/44] Add try catch in logger opening --- .../ddev/ai/tools/agents/spawn_subagent.py | 9 ++++++- .../ai/tools/agents/test_spawn_subagent.py | 24 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/ddev/src/ddev/ai/tools/agents/spawn_subagent.py b/ddev/src/ddev/ai/tools/agents/spawn_subagent.py index d73b085d0055a..6cc5139fea006 100644 --- a/ddev/src/ddev/ai/tools/agents/spawn_subagent.py +++ b/ddev/src/ddev/ai/tools/agents/spawn_subagent.py @@ -106,7 +106,14 @@ async def __call__(self, tool_input: SpawnSubagentInput) -> ToolResult: subagent_id = f"{self._owner_id}.sub.{self._counter:03d}-{label}" log_path = self._log_dir / f"{self._counter:03d}-{label}.jsonl" - logger = AgentLogger(log_path) + try: + logger = AgentLogger(log_path) + except OSError as e: + return ToolResult( + success=False, + error=f"Subagent {label!r} not spawned: cannot open log file {log_path}: {e}", + ) + try: logger.log_start( system_prompt=tool_input.system_prompt, diff --git a/ddev/tests/ai/tools/agents/test_spawn_subagent.py b/ddev/tests/ai/tools/agents/test_spawn_subagent.py index b1d3cecd0e6c0..f3881b539b805 100644 --- a/ddev/tests/ai/tools/agents/test_spawn_subagent.py +++ b/ddev/tests/ai/tools/agents/test_spawn_subagent.py @@ -169,6 +169,30 @@ async def test_mkdir_failure(tmp_path): assert tool._counter == 0 +async def test_logger_open_failure_returns_tool_result(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def builder( + system_prompt: str, + owner_id: str, + tool_names: list[str], + ) -> tuple[BaseAgent[Any], ToolRegistry]: + raise AssertionError("builder should not be called") + + def failing_logger(log_path: Path) -> None: + raise OSError("permission denied") + + monkeypatch.setattr("ddev.ai.tools.agents.spawn_subagent.AgentLogger", failing_logger) + + tool = make_tool(tmp_path, builder, allowed_tools=[]) + result = await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="x")) + + assert result.success is False + assert "x" in result.error + assert "cannot open log file" in result.error + assert str(tmp_path / "001-x.jsonl") in result.error + assert "permission denied" in result.error + assert list(tmp_path.glob("*.jsonl")) == [] + + # --------------------------------------------------------------------------- # Happy path # --------------------------------------------------------------------------- From a5757e5e7c25d8edd850f140cfd2da07e637500c Mon Sep 17 00:00:00 2001 From: Luis Orofino Date: Tue, 26 May 2026 17:47:52 +0200 Subject: [PATCH 44/44] Fix pyproject.toml --- ddev/pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/ddev/pyproject.toml b/ddev/pyproject.toml index f2baa037e9570..73928e956e9d4 100644 --- a/ddev/pyproject.toml +++ b/ddev/pyproject.toml @@ -140,6 +140,3 @@ ban-relative-imports = "parents" [tool.ruff.lint.per-file-ignores] #Tests can use assertions and relative imports "**/tests/**/*" = ["I252"] - -[tool.pytest.ini_options] -asyncio_mode = "auto"