diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index b54d570280199..0000000000000 --- a/.codecov.yml +++ /dev/null @@ -1,1895 +0,0 @@ -comment: - layout: flags - behavior: default - branches: null -coverage: - range: 50..100 - round: down - precision: 2 - status: - default_rules: - flag_coverage_not_uploaded_behavior: exclude - project: - .NET_CLR: - target: 75 - flags: - - dotnetclr - ASP.NET: - target: 75 - flags: - - aspdotnet - AWS_Neuron: - target: 75 - flags: - - aws_neuron - ActiveMQ_XML: - target: 75 - flags: - - activemq_xml - Active_Directory: - target: 75 - flags: - - active_directory - Aerospike: - target: 75 - flags: - - aerospike - Airflow: - target: 75 - flags: - - airflow - Amazon_ECS_Fargate: - target: 75 - flags: - - ecs_fargate - Amazon_Kafka: - target: 75 - flags: - - amazon_msk - Ambari: - target: 75 - flags: - - ambari - Apache: - target: 75 - flags: - - apache - Apache_NiFi: - target: 75 - flags: - - nifi - Appgate_SDP: - target: 75 - flags: - - appgate_sdp - ArangoDB: - target: 75 - flags: - - arangodb - ArgoCD: - target: 75 - flags: - - argocd - Argo_Rollouts: - target: 75 - flags: - - argo_rollouts - Argo_Workflows: - target: 75 - flags: - - argo_workflows - Avi_Vantage: - target: 75 - flags: - - avi_vantage - Azure_IoT_Edge: - target: 75 - flags: - - azure_iot_edge - BentoML: - target: 75 - flags: - - bentoml - Boundary: - target: 75 - flags: - - boundary - Btrfs: - target: 75 - flags: - - btrfs - CRI-O: - target: 75 - flags: - - crio - Cacti: - target: 75 - flags: - - cacti - Calico: - target: 75 - flags: - - calico - Cassandra_Nodetool: - target: 75 - flags: - - cassandra_nodetool - Celery: - target: 75 - flags: - - celery - Ceph: - target: 75 - flags: - - ceph - Cilium: - target: 75 - flags: - - cilium - Cisco_ACI: - target: 75 - flags: - - cisco_aci - Citrix_Hypervisor: - target: 75 - flags: - - citrix_hypervisor - ClickHouse: - target: 75 - flags: - - clickhouse - Cloud_Foundry_API: - target: 75 - flags: - - cloud_foundry_api - Cloudera: - target: 75 - flags: - - cloudera - CockroachDB: - target: 75 - flags: - - cockroachdb - Consul: - target: 75 - flags: - - consul - Control-M: - target: 75 - flags: - - control_m - CoreDNS: - target: 75 - flags: - - coredns - CouchDB: - target: 75 - flags: - - couch - Couchbase: - target: 75 - flags: - - couchbase - DNS: - target: 75 - flags: - - dns_check - DO_Query_Actions: - target: 75 - flags: - - do_query_actions - Datadog_Checks_Base: - target: 75 - flags: - - datadog_checks_base - Datadog_Checks_Dev: - target: 75 - flags: - - datadog_checks_dev - Datadog_Checks_Downloader: - target: 75 - flags: - - datadog_checks_downloader - Datadog_Cluster_Agent: - target: 75 - flags: - - datadog_cluster_agent - Directory: - target: 75 - flags: - - directory - Disk: - target: 75 - flags: - - disk - Druid: - target: 75 - flags: - - druid - DuckDB: - target: 75 - flags: - - duckdb - EKS_Fargate: - target: 75 - flags: - - eks_fargate - ESXi: - target: 75 - flags: - - esxi - Elasticsearch: - target: 75 - flags: - - elastic - Envoy: - target: 75 - flags: - - envoy - Exchange_Server: - target: 75 - flags: - - exchange_server - External_DNS: - target: 75 - flags: - - external_dns - Falco: - target: 75 - flags: - - falco - Fluentd: - target: 75 - flags: - - fluentd - Fly.io: - target: 75 - flags: - - fly_io - FoundationDB: - target: 75 - flags: - - foundationdb - Gearman: - target: 75 - flags: - - gearmand - Gitlab: - target: 75 - flags: - - gitlab - Gitlab_Runner: - target: 75 - flags: - - gitlab_runner - GlusterFS: - target: 75 - flags: - - glusterfs - Go_Expvar: - target: 75 - flags: - - go_expvar - GuardDog: - target: 75 - flags: - - guarddog - Gunicorn: - target: 75 - flags: - - gunicorn - HAProxy: - target: 75 - flags: - - haproxy - HDFS_Datanode: - target: 75 - flags: - - hdfs_datanode - HDFS_Namenode: - target: 75 - flags: - - hdfs_namenode - HTTP: - target: 75 - flags: - - http_check - Harbor: - target: 75 - flags: - - harbor - Hazelcast: - target: 75 - flags: - - hazelcast - Hugging_Face_TGI: - target: 75 - flags: - - hugging_face_tgi - IBM_ACE: - target: 75 - flags: - - ibm_ace - IBM_Db2: - target: 75 - flags: - - ibm_db2 - IBM_MQ: - target: 75 - flags: - - ibm_mq - IBM_Spectrum_LSF: - target: 75 - flags: - - ibm_spectrum_lsf - IBM_WAS: - target: 75 - flags: - - ibm_was - IBM_i: - target: 75 - flags: - - ibm_i - IIS: - target: 75 - flags: - - iis - Impala: - target: 75 - flags: - - impala - Infiniband: - target: 75 - flags: - - infiniband - Istio: - target: 75 - flags: - - istio - Kafka_Actions: - target: 75 - flags: - - kafka_actions - Kafka_Consumer: - target: 75 - flags: - - kafka_consumer - Karpenter: - target: 75 - flags: - - karpenter - Keda: - target: 75 - flags: - - keda - Kong: - target: 75 - flags: - - kong - KrakenD: - target: 75 - flags: - - krakend - KubeVirt_API: - target: 75 - flags: - - kubevirt_api - KubeVirt_Controller: - target: 75 - flags: - - kubevirt_controller - KubeVirt_Handler: - target: 75 - flags: - - kubevirt_handler - Kube_DNS: - target: 75 - flags: - - kube_dns - Kube_Proxy: - target: 75 - flags: - - kube_proxy - Kube_metrics_server: - target: 75 - flags: - - kube_metrics_server - Kubeflow: - target: 75 - flags: - - kubeflow - Kubelet: - target: 75 - flags: - - kubelet - Kubernetes_API_server_metrics: - target: 75 - flags: - - kube_apiserver_metrics - Kubernetes_Cluster_Autoscaler: - target: 75 - flags: - - kubernetes_cluster_autoscaler - Kubernetes_Controller_Manager: - target: 75 - flags: - - kube_controller_manager - Kubernetes_Scheduler: - target: 75 - flags: - - kube_scheduler - Kubernetes_State: - target: 75 - flags: - - kubernetes_state - Kuma: - target: 75 - flags: - - kuma - Kyoto_Tycoon: - target: 75 - flags: - - kyototycoon - LPARStats: - target: 75 - flags: - - lparstats - Lighttpd: - target: 75 - flags: - - lighttpd - Linkerd: - target: 75 - flags: - - linkerd - Linux_proc_extras: - target: 75 - flags: - - linux_proc_extras - LiteLLM: - target: 75 - flags: - - litellm - Lustre: - target: 75 - flags: - - lustre - Mac_Audit_Logs: - target: 75 - flags: - - mac_audit_logs - MapR: - target: 75 - flags: - - mapr - MapReduce: - target: 75 - flags: - - mapreduce - Marathon: - target: 75 - flags: - - marathon - MarkLogic: - target: 75 - flags: - - marklogic - Memcached: - target: 75 - flags: - - mcache - Mesos: - target: 75 - flags: - - mesos_slave - Mesos_Master: - target: 75 - flags: - - mesos_master - Milvus: - target: 75 - flags: - - milvus - MongoDB: - target: 75 - flags: - - mongo - MySQL: - target: 75 - flags: - - mysql - NFSstat: - target: 75 - flags: - - nfsstat - NGINX: - target: 75 - flags: - - nginx - NGINX_Ingress_Controller: - target: 75 - flags: - - nginx_ingress_controller - Nagios: - target: 75 - flags: - - nagios - Network: - target: 75 - flags: - - network - Nutanix: - target: 75 - flags: - - nutanix - Nvidia_Triton: - target: 75 - flags: - - nvidia_triton - Octopus_Deploy: - target: 75 - flags: - - octopus_deploy - OpenLDAP: - target: 75 - flags: - - openldap - OpenMetrics: - target: 75 - flags: - - openmetrics - OpenStack: - target: 50 - flags: - - openstack - OpenStack_Controller: - target: 75 - flags: - - openstack_controller - PDH: - target: 75 - flags: - - pdh_check - PGBouncer: - target: 75 - flags: - - pgbouncer - PHP-FPM: - target: 75 - flags: - - php_fpm - Postfix: - target: 75 - flags: - - postfix - Postgres: - target: 75 - flags: - - postgres - PowerDNS_Recursor: - target: 75 - flags: - - powerdns_recursor - Prefect: - target: 75 - flags: - - prefect - Process: - target: 75 - flags: - - process - Prometheus: - target: 75 - flags: - - prometheus - Proxmox: - target: 75 - flags: - - proxmox - ProxySQL: - target: 75 - flags: - - proxysql - Pulsar: - target: 75 - flags: - - pulsar - Quarkus: - target: 75 - flags: - - quarkus - RabbitMQ: - target: 75 - flags: - - rabbitmq - Ray: - target: 75 - flags: - - ray - Redis: - target: 75 - flags: - - redisdb - RethinkDB: - target: 75 - flags: - - rethinkdb - Riak: - target: 75 - flags: - - riak - RiakCS: - target: 75 - flags: - - riakcs - SAP_HANA: - target: 75 - flags: - - sap_hana - SNMP: - target: 30 - flags: - - snmp - SQL_Server: - target: 75 - flags: - - sqlserver - SSH: - target: 75 - flags: - - ssh_check - Scylla: - target: 75 - flags: - - scylla - Silk: - target: 75 - flags: - - silk - Silverstripe_CMS: - target: 75 - flags: - - silverstripe_cms - SingleStore: - target: 75 - flags: - - singlestore - Slurm: - target: 75 - flags: - - slurm - SonarQube: - target: 75 - flags: - - sonarqube - Spark: - target: 75 - flags: - - spark - Squid: - target: 75 - flags: - - squid - StatsD: - target: 75 - flags: - - statsd - Strimzi: - target: 75 - flags: - - strimzi - Supabase: - target: 75 - flags: - - supabase - Supervisord: - target: 75 - flags: - - supervisord - System_Core: - target: 75 - flags: - - system_core - System_Swap: - target: 75 - flags: - - system_swap - TCP: - target: 75 - flags: - - tcp_check - TLS: - target: 75 - flags: - - tls - TeamCity: - target: 75 - flags: - - teamcity - Tekton: - target: 75 - flags: - - tekton - Teleport: - target: 75 - flags: - - teleport - Temporal: - target: 75 - flags: - - temporal - Teradata: - target: 75 - flags: - - teradata - TokuMX: - target: 50 - flags: - - tokumx - TorchServe: - target: 75 - flags: - - torchserve - Traefik_Mesh: - target: 75 - flags: - - traefik_mesh - Traffic_Server: - target: 75 - flags: - - traffic_server - Twemproxy: - target: 75 - flags: - - twemproxy - Twistlock: - target: 75 - flags: - - twistlock - Varnish: - target: 75 - flags: - - varnish - Vault: - target: 75 - flags: - - vault - Velero: - target: 75 - flags: - - velero - Vertica: - target: 75 - flags: - - vertica - VoltDB: - target: 75 - flags: - - voltdb - WMI: - target: 75 - flags: - - wmi_check - Weaviate: - target: 75 - flags: - - weaviate - Windows_Event_Log: - target: 75 - flags: - - win32_event_log - Windows_Service: - target: 75 - flags: - - windows_service - Windows_performance_counters: - target: 75 - flags: - - windows_performance_counters - Yarn: - target: 75 - flags: - - yarn - ZooKeeper: - target: 75 - flags: - - zk - cert-manager: - target: 75 - flags: - - cert_manager - checkpoint_harmony_endpoint: - target: 75 - flags: - - checkpoint_harmony_endpoint - dcgm: - target: 75 - flags: - - dcgm - ddev: - target: 75 - flags: - - ddev - etcd: - target: 75 - flags: - - etcd - fluxcd: - target: 75 - flags: - - fluxcd - kyverno: - target: 75 - flags: - - kyverno - n8n: - target: 75 - flags: - - n8n - nvidia_nim: - target: 75 - flags: - - nvidia_nim - sonatype_nexus: - target: 75 - flags: - - sonatype_nexus - tibco_ems: - target: 75 - flags: - - tibco_ems - vLLM: - target: 75 - flags: - - vllm - vSphere: - target: 75 - flags: - - vsphere - patch: false -flags: - active_directory: - carryforward: true - paths: - - active_directory/datadog_checks/active_directory - - active_directory/tests - activemq_xml: - carryforward: true - paths: - - activemq_xml/datadog_checks/activemq_xml - - activemq_xml/tests - aerospike: - carryforward: true - paths: - - aerospike/datadog_checks/aerospike - - aerospike/tests - airflow: - carryforward: true - paths: - - airflow/datadog_checks/airflow - - airflow/tests - amazon_msk: - carryforward: true - paths: - - amazon_msk/datadog_checks/amazon_msk - - amazon_msk/tests - ambari: - carryforward: true - paths: - - ambari/datadog_checks/ambari - - ambari/tests - apache: - carryforward: true - paths: - - apache/datadog_checks/apache - - apache/tests - appgate_sdp: - carryforward: true - paths: - - appgate_sdp/datadog_checks/appgate_sdp - - appgate_sdp/tests - arangodb: - carryforward: true - paths: - - arangodb/datadog_checks/arangodb - - arangodb/tests - argo_rollouts: - carryforward: true - paths: - - argo_rollouts/datadog_checks/argo_rollouts - - argo_rollouts/tests - argo_workflows: - carryforward: true - paths: - - argo_workflows/datadog_checks/argo_workflows - - argo_workflows/tests - argocd: - carryforward: true - paths: - - argocd/datadog_checks/argocd - - argocd/tests - aspdotnet: - carryforward: true - paths: - - aspdotnet/datadog_checks/aspdotnet - - aspdotnet/tests - avi_vantage: - carryforward: true - paths: - - avi_vantage/datadog_checks/avi_vantage - - avi_vantage/tests - aws_neuron: - carryforward: true - paths: - - aws_neuron/datadog_checks/aws_neuron - - aws_neuron/tests - azure_iot_edge: - carryforward: true - paths: - - azure_iot_edge/datadog_checks/azure_iot_edge - - azure_iot_edge/tests - bentoml: - carryforward: true - paths: - - bentoml/datadog_checks/bentoml - - bentoml/tests - boundary: - carryforward: true - paths: - - boundary/datadog_checks/boundary - - boundary/tests - btrfs: - carryforward: true - paths: - - btrfs/datadog_checks/btrfs - - btrfs/tests - cacti: - carryforward: true - paths: - - cacti/datadog_checks/cacti - - cacti/tests - calico: - carryforward: true - paths: - - calico/datadog_checks/calico - - calico/tests - cassandra_nodetool: - carryforward: true - paths: - - cassandra_nodetool/datadog_checks/cassandra_nodetool - - cassandra_nodetool/tests - celery: - carryforward: true - paths: - - celery/datadog_checks/celery - - celery/tests - ceph: - carryforward: true - paths: - - ceph/datadog_checks/ceph - - ceph/tests - cert_manager: - carryforward: true - paths: - - cert_manager/datadog_checks/cert_manager - - cert_manager/tests - checkpoint_harmony_endpoint: - carryforward: true - paths: - - checkpoint_harmony_endpoint/datadog_checks/checkpoint_harmony_endpoint - - checkpoint_harmony_endpoint/tests - cilium: - carryforward: true - paths: - - cilium/datadog_checks/cilium - - cilium/tests - cisco_aci: - carryforward: true - paths: - - cisco_aci/datadog_checks/cisco_aci - - cisco_aci/tests - citrix_hypervisor: - carryforward: true - paths: - - citrix_hypervisor/datadog_checks/citrix_hypervisor - - citrix_hypervisor/tests - clickhouse: - carryforward: true - paths: - - clickhouse/datadog_checks/clickhouse - - clickhouse/tests - cloud_foundry_api: - carryforward: true - paths: - - cloud_foundry_api/datadog_checks/cloud_foundry_api - - cloud_foundry_api/tests - cloudera: - carryforward: true - paths: - - cloudera/datadog_checks/cloudera - - cloudera/tests - cockroachdb: - carryforward: true - paths: - - cockroachdb/datadog_checks/cockroachdb - - cockroachdb/tests - consul: - carryforward: true - paths: - - consul/datadog_checks/consul - - consul/tests - control_m: - carryforward: true - paths: - - control_m/datadog_checks/control_m - - control_m/tests - coredns: - carryforward: true - paths: - - coredns/datadog_checks/coredns - - coredns/tests - couch: - carryforward: true - paths: - - couch/datadog_checks/couch - - couch/tests - couchbase: - carryforward: true - paths: - - couchbase/datadog_checks/couchbase - - couchbase/tests - crio: - carryforward: true - paths: - - crio/datadog_checks/crio - - crio/tests - datadog_checks_base: - carryforward: true - paths: - - datadog_checks_base/datadog_checks/base - - datadog_checks_base/tests - datadog_checks_dev: - carryforward: true - paths: - - datadog_checks_dev/datadog_checks/dev - - datadog_checks_dev/tests - datadog_checks_downloader: - carryforward: true - paths: - - datadog_checks_downloader/datadog_checks/downloader - - datadog_checks_downloader/tests - datadog_cluster_agent: - carryforward: true - paths: - - datadog_cluster_agent/datadog_checks/datadog_cluster_agent - - datadog_cluster_agent/tests - dcgm: - carryforward: true - paths: - - dcgm/datadog_checks/dcgm - - dcgm/tests - ddev: - carryforward: true - paths: - - ddev/src/ddev - - ddev/tests - directory: - carryforward: true - paths: - - directory/datadog_checks/directory - - directory/tests - disk: - carryforward: true - paths: - - disk/datadog_checks/disk - - disk/tests - dns_check: - carryforward: true - paths: - - dns_check/datadog_checks/dns_check - - dns_check/tests - do_query_actions: - carryforward: true - paths: - - do_query_actions/datadog_checks/do_query_actions - - do_query_actions/tests - dotnetclr: - carryforward: true - paths: - - dotnetclr/datadog_checks/dotnetclr - - dotnetclr/tests - druid: - carryforward: true - paths: - - druid/datadog_checks/druid - - druid/tests - duckdb: - carryforward: true - paths: - - duckdb/datadog_checks/duckdb - - duckdb/tests - ecs_fargate: - carryforward: true - paths: - - ecs_fargate/datadog_checks/ecs_fargate - - ecs_fargate/tests - eks_fargate: - carryforward: true - paths: - - eks_fargate/datadog_checks/eks_fargate - - eks_fargate/tests - elastic: - carryforward: true - paths: - - elastic/datadog_checks/elastic - - elastic/tests - envoy: - carryforward: true - paths: - - envoy/datadog_checks/envoy - - envoy/tests - esxi: - carryforward: true - paths: - - esxi/datadog_checks/esxi - - esxi/tests - etcd: - carryforward: true - paths: - - etcd/datadog_checks/etcd - - etcd/tests - exchange_server: - carryforward: true - paths: - - exchange_server/datadog_checks/exchange_server - - exchange_server/tests - external_dns: - carryforward: true - paths: - - external_dns/datadog_checks/external_dns - - external_dns/tests - falco: - carryforward: true - paths: - - falco/datadog_checks/falco - - falco/tests - fluentd: - carryforward: true - paths: - - fluentd/datadog_checks/fluentd - - fluentd/tests - fluxcd: - carryforward: true - paths: - - fluxcd/datadog_checks/fluxcd - - fluxcd/tests - fly_io: - carryforward: true - paths: - - fly_io/datadog_checks/fly_io - - fly_io/tests - foundationdb: - carryforward: true - paths: - - foundationdb/datadog_checks/foundationdb - - foundationdb/tests - gearmand: - carryforward: true - paths: - - gearmand/datadog_checks/gearmand - - gearmand/tests - gitlab: - carryforward: true - paths: - - gitlab/datadog_checks/gitlab - - gitlab/tests - gitlab_runner: - carryforward: true - paths: - - gitlab_runner/datadog_checks/gitlab_runner - - gitlab_runner/tests - glusterfs: - carryforward: true - paths: - - glusterfs/datadog_checks/glusterfs - - glusterfs/tests - go_expvar: - carryforward: true - paths: - - go_expvar/datadog_checks/go_expvar - - go_expvar/tests - guarddog: - carryforward: true - paths: - - guarddog/datadog_checks/guarddog - - guarddog/tests - gunicorn: - carryforward: true - paths: - - gunicorn/datadog_checks/gunicorn - - gunicorn/tests - haproxy: - carryforward: true - paths: - - haproxy/datadog_checks/haproxy - - haproxy/tests - harbor: - carryforward: true - paths: - - harbor/datadog_checks/harbor - - harbor/tests - hazelcast: - carryforward: true - paths: - - hazelcast/datadog_checks/hazelcast - - hazelcast/tests - hdfs_datanode: - carryforward: true - paths: - - hdfs_datanode/datadog_checks/hdfs_datanode - - hdfs_datanode/tests - hdfs_namenode: - carryforward: true - paths: - - hdfs_namenode/datadog_checks/hdfs_namenode - - hdfs_namenode/tests - http_check: - carryforward: true - paths: - - http_check/datadog_checks/http_check - - http_check/tests - hugging_face_tgi: - carryforward: true - paths: - - hugging_face_tgi/datadog_checks/hugging_face_tgi - - hugging_face_tgi/tests - ibm_ace: - carryforward: true - paths: - - ibm_ace/datadog_checks/ibm_ace - - ibm_ace/tests - ibm_db2: - carryforward: true - paths: - - ibm_db2/datadog_checks/ibm_db2 - - ibm_db2/tests - ibm_i: - carryforward: true - paths: - - ibm_i/datadog_checks/ibm_i - - ibm_i/tests - ibm_mq: - carryforward: true - paths: - - ibm_mq/datadog_checks/ibm_mq - - ibm_mq/tests - ibm_spectrum_lsf: - carryforward: true - paths: - - ibm_spectrum_lsf/datadog_checks/ibm_spectrum_lsf - - ibm_spectrum_lsf/tests - ibm_was: - carryforward: true - paths: - - ibm_was/datadog_checks/ibm_was - - ibm_was/tests - iis: - carryforward: true - paths: - - iis/datadog_checks/iis - - iis/tests - impala: - carryforward: true - paths: - - impala/datadog_checks/impala - - impala/tests - infiniband: - carryforward: true - paths: - - infiniband/datadog_checks/infiniband - - infiniband/tests - istio: - carryforward: true - paths: - - istio/datadog_checks/istio - - istio/tests - kafka_actions: - carryforward: true - paths: - - kafka_actions/datadog_checks/kafka_actions - - kafka_actions/tests - kafka_consumer: - carryforward: true - paths: - - kafka_consumer/datadog_checks/kafka_consumer - - kafka_consumer/tests - karpenter: - carryforward: true - paths: - - karpenter/datadog_checks/karpenter - - karpenter/tests - keda: - carryforward: true - paths: - - keda/datadog_checks/keda - - keda/tests - kong: - carryforward: true - paths: - - kong/datadog_checks/kong - - kong/tests - krakend: - carryforward: true - paths: - - krakend/datadog_checks/krakend - - krakend/tests - kube_apiserver_metrics: - carryforward: true - paths: - - kube_apiserver_metrics/datadog_checks/kube_apiserver_metrics - - kube_apiserver_metrics/tests - kube_controller_manager: - carryforward: true - paths: - - kube_controller_manager/datadog_checks/kube_controller_manager - - kube_controller_manager/tests - kube_dns: - carryforward: true - paths: - - kube_dns/datadog_checks/kube_dns - - kube_dns/tests - kube_metrics_server: - carryforward: true - paths: - - kube_metrics_server/datadog_checks/kube_metrics_server - - kube_metrics_server/tests - kube_proxy: - carryforward: true - paths: - - kube_proxy/datadog_checks/kube_proxy - - kube_proxy/tests - kube_scheduler: - carryforward: true - paths: - - kube_scheduler/datadog_checks/kube_scheduler - - kube_scheduler/tests - kubeflow: - carryforward: true - paths: - - kubeflow/datadog_checks/kubeflow - - kubeflow/tests - kubelet: - carryforward: true - paths: - - kubelet/datadog_checks/kubelet - - kubelet/tests - kubernetes_cluster_autoscaler: - carryforward: true - paths: - - kubernetes_cluster_autoscaler/datadog_checks/kubernetes_cluster_autoscaler - - kubernetes_cluster_autoscaler/tests - kubernetes_state: - carryforward: true - paths: - - kubernetes_state/datadog_checks/kubernetes_state - - kubernetes_state/tests - kubevirt_api: - carryforward: true - paths: - - kubevirt_api/datadog_checks/kubevirt_api - - kubevirt_api/tests - kubevirt_controller: - carryforward: true - paths: - - kubevirt_controller/datadog_checks/kubevirt_controller - - kubevirt_controller/tests - kubevirt_handler: - carryforward: true - paths: - - kubevirt_handler/datadog_checks/kubevirt_handler - - kubevirt_handler/tests - kuma: - carryforward: true - paths: - - kuma/datadog_checks/kuma - - kuma/tests - kyototycoon: - carryforward: true - paths: - - kyototycoon/datadog_checks/kyototycoon - - kyototycoon/tests - kyverno: - carryforward: true - paths: - - kyverno/datadog_checks/kyverno - - kyverno/tests - lparstats: - carryforward: true - paths: - - lparstats/datadog_checks/lparstats - - lparstats/tests - lighttpd: - carryforward: true - paths: - - lighttpd/datadog_checks/lighttpd - - lighttpd/tests - linkerd: - carryforward: true - paths: - - linkerd/datadog_checks/linkerd - - linkerd/tests - linux_proc_extras: - carryforward: true - paths: - - linux_proc_extras/datadog_checks/linux_proc_extras - - linux_proc_extras/tests - litellm: - carryforward: true - paths: - - litellm/datadog_checks/litellm - - litellm/tests - lustre: - carryforward: true - paths: - - lustre/datadog_checks/lustre - - lustre/tests - mac_audit_logs: - carryforward: true - paths: - - mac_audit_logs/datadog_checks/mac_audit_logs - - mac_audit_logs/tests - mapr: - carryforward: true - paths: - - mapr/datadog_checks/mapr - - mapr/tests - mapreduce: - carryforward: true - paths: - - mapreduce/datadog_checks/mapreduce - - mapreduce/tests - marathon: - carryforward: true - paths: - - marathon/datadog_checks/marathon - - marathon/tests - marklogic: - carryforward: true - paths: - - marklogic/datadog_checks/marklogic - - marklogic/tests - mcache: - carryforward: true - paths: - - mcache/datadog_checks/mcache - - mcache/tests - mesos_master: - carryforward: true - paths: - - mesos_master/datadog_checks/mesos_master - - mesos_master/tests - mesos_slave: - carryforward: true - paths: - - mesos_slave/datadog_checks/mesos_slave - - mesos_slave/tests - milvus: - carryforward: true - paths: - - milvus/datadog_checks/milvus - - milvus/tests - mongo: - carryforward: true - paths: - - mongo/datadog_checks/mongo - - mongo/tests - mysql: - carryforward: true - paths: - - mysql/datadog_checks/mysql - - mysql/tests - n8n: - carryforward: true - paths: - - n8n/datadog_checks/n8n - - n8n/tests - nagios: - carryforward: true - paths: - - nagios/datadog_checks/nagios - - nagios/tests - network: - carryforward: true - paths: - - network/datadog_checks/network - - network/tests - nfsstat: - carryforward: true - paths: - - nfsstat/datadog_checks/nfsstat - - nfsstat/tests - nginx: - carryforward: true - paths: - - nginx/datadog_checks/nginx - - nginx/tests - nginx_ingress_controller: - carryforward: true - paths: - - nginx_ingress_controller/datadog_checks/nginx_ingress_controller - - nginx_ingress_controller/tests - nifi: - carryforward: true - paths: - - nifi/datadog_checks/nifi - - nifi/tests - nutanix: - carryforward: true - paths: - - nutanix/datadog_checks/nutanix - - nutanix/tests - nvidia_nim: - carryforward: true - paths: - - nvidia_nim/datadog_checks/nvidia_nim - - nvidia_nim/tests - nvidia_triton: - carryforward: true - paths: - - nvidia_triton/datadog_checks/nvidia_triton - - nvidia_triton/tests - octopus_deploy: - carryforward: true - paths: - - octopus_deploy/datadog_checks/octopus_deploy - - octopus_deploy/tests - openldap: - carryforward: true - paths: - - openldap/datadog_checks/openldap - - openldap/tests - openmetrics: - carryforward: true - paths: - - openmetrics/datadog_checks/openmetrics - - openmetrics/tests - openstack: - carryforward: true - paths: - - openstack/datadog_checks/openstack - - openstack/tests - openstack_controller: - carryforward: true - paths: - - openstack_controller/datadog_checks/openstack_controller - - openstack_controller/tests - pdh_check: - carryforward: true - paths: - - pdh_check/datadog_checks/pdh_check - - pdh_check/tests - pgbouncer: - carryforward: true - paths: - - pgbouncer/datadog_checks/pgbouncer - - pgbouncer/tests - php_fpm: - carryforward: true - paths: - - php_fpm/datadog_checks/php_fpm - - php_fpm/tests - postfix: - carryforward: true - paths: - - postfix/datadog_checks/postfix - - postfix/tests - postgres: - carryforward: true - paths: - - postgres/datadog_checks/postgres - - postgres/tests - powerdns_recursor: - carryforward: true - paths: - - powerdns_recursor/datadog_checks/powerdns_recursor - - powerdns_recursor/tests - prefect: - carryforward: true - paths: - - prefect/datadog_checks/prefect - - prefect/tests - process: - carryforward: true - paths: - - process/datadog_checks/process - - process/tests - prometheus: - carryforward: true - paths: - - prometheus/datadog_checks/prometheus - - prometheus/tests - proxmox: - carryforward: true - paths: - - proxmox/datadog_checks/proxmox - - proxmox/tests - proxysql: - carryforward: true - paths: - - proxysql/datadog_checks/proxysql - - proxysql/tests - pulsar: - carryforward: true - paths: - - pulsar/datadog_checks/pulsar - - pulsar/tests - quarkus: - carryforward: true - paths: - - quarkus/datadog_checks/quarkus - - quarkus/tests - rabbitmq: - carryforward: true - paths: - - rabbitmq/datadog_checks/rabbitmq - - rabbitmq/tests - ray: - carryforward: true - paths: - - ray/datadog_checks/ray - - ray/tests - redisdb: - carryforward: true - paths: - - redisdb/datadog_checks/redisdb - - redisdb/tests - rethinkdb: - carryforward: true - paths: - - rethinkdb/datadog_checks/rethinkdb - - rethinkdb/tests - riak: - carryforward: true - paths: - - riak/datadog_checks/riak - - riak/tests - riakcs: - carryforward: true - paths: - - riakcs/datadog_checks/riakcs - - riakcs/tests - sap_hana: - carryforward: true - paths: - - sap_hana/datadog_checks/sap_hana - - sap_hana/tests - scylla: - carryforward: true - paths: - - scylla/datadog_checks/scylla - - scylla/tests - silk: - carryforward: true - paths: - - silk/datadog_checks/silk - - silk/tests - silverstripe_cms: - carryforward: true - paths: - - silverstripe_cms/datadog_checks/silverstripe_cms - - silverstripe_cms/tests - singlestore: - carryforward: true - paths: - - singlestore/datadog_checks/singlestore - - singlestore/tests - slurm: - carryforward: true - paths: - - slurm/datadog_checks/slurm - - slurm/tests - snmp: - carryforward: true - paths: - - snmp/datadog_checks/snmp - - snmp/tests - sonarqube: - carryforward: true - paths: - - sonarqube/datadog_checks/sonarqube - - sonarqube/tests - sonatype_nexus: - carryforward: true - paths: - - sonatype_nexus/datadog_checks/sonatype_nexus - - sonatype_nexus/tests - spark: - carryforward: true - paths: - - spark/datadog_checks/spark - - spark/tests - sqlserver: - carryforward: true - paths: - - sqlserver/datadog_checks/sqlserver - - sqlserver/tests - squid: - carryforward: true - paths: - - squid/datadog_checks/squid - - squid/tests - ssh_check: - carryforward: true - paths: - - ssh_check/datadog_checks/ssh_check - - ssh_check/tests - statsd: - carryforward: true - paths: - - statsd/datadog_checks/statsd - - statsd/tests - strimzi: - carryforward: true - paths: - - strimzi/datadog_checks/strimzi - - strimzi/tests - supabase: - carryforward: true - paths: - - supabase/datadog_checks/supabase - - supabase/tests - supervisord: - carryforward: true - paths: - - supervisord/datadog_checks/supervisord - - supervisord/tests - system_core: - carryforward: true - paths: - - system_core/datadog_checks/system_core - - system_core/tests - system_swap: - carryforward: true - paths: - - system_swap/datadog_checks/system_swap - - system_swap/tests - tcp_check: - carryforward: true - paths: - - tcp_check/datadog_checks/tcp_check - - tcp_check/tests - teamcity: - carryforward: true - paths: - - teamcity/datadog_checks/teamcity - - teamcity/tests - tekton: - carryforward: true - paths: - - tekton/datadog_checks/tekton - - tekton/tests - teleport: - carryforward: true - paths: - - teleport/datadog_checks/teleport - - teleport/tests - temporal: - carryforward: true - paths: - - temporal/datadog_checks/temporal - - temporal/tests - teradata: - carryforward: true - paths: - - teradata/datadog_checks/teradata - - teradata/tests - tibco_ems: - carryforward: true - paths: - - tibco_ems/datadog_checks/tibco_ems - - tibco_ems/tests - tls: - carryforward: true - paths: - - tls/datadog_checks/tls - - tls/tests - tokumx: - carryforward: true - paths: - - tokumx/datadog_checks/tokumx - - tokumx/tests - torchserve: - carryforward: true - paths: - - torchserve/datadog_checks/torchserve - - torchserve/tests - traefik_mesh: - carryforward: true - paths: - - traefik_mesh/datadog_checks/traefik_mesh - - traefik_mesh/tests - traffic_server: - carryforward: true - paths: - - traffic_server/datadog_checks/traffic_server - - traffic_server/tests - twemproxy: - carryforward: true - paths: - - twemproxy/datadog_checks/twemproxy - - twemproxy/tests - twistlock: - carryforward: true - paths: - - twistlock/datadog_checks/twistlock - - twistlock/tests - varnish: - carryforward: true - paths: - - varnish/datadog_checks/varnish - - varnish/tests - vault: - carryforward: true - paths: - - vault/datadog_checks/vault - - vault/tests - velero: - carryforward: true - paths: - - velero/datadog_checks/velero - - velero/tests - vertica: - carryforward: true - paths: - - vertica/datadog_checks/vertica - - vertica/tests - vllm: - carryforward: true - paths: - - vllm/datadog_checks/vllm - - vllm/tests - voltdb: - carryforward: true - paths: - - voltdb/datadog_checks/voltdb - - voltdb/tests - vsphere: - carryforward: true - paths: - - vsphere/datadog_checks/vsphere - - vsphere/tests - weaviate: - carryforward: true - paths: - - weaviate/datadog_checks/weaviate - - weaviate/tests - win32_event_log: - carryforward: true - paths: - - win32_event_log/datadog_checks/win32_event_log - - win32_event_log/tests - windows_performance_counters: - carryforward: true - paths: - - windows_performance_counters/datadog_checks/windows_performance_counters - - windows_performance_counters/tests - windows_service: - carryforward: true - paths: - - windows_service/datadog_checks/windows_service - - windows_service/tests - wmi_check: - carryforward: true - paths: - - wmi_check/datadog_checks/wmi_check - - wmi_check/tests - yarn: - carryforward: true - paths: - - yarn/datadog_checks/yarn - - yarn/tests - zk: - carryforward: true - paths: - - zk/datadog_checks/zk - - zk/tests diff --git a/.ddev/config.toml b/.ddev/config.toml index eabffa06e2919..92c39bef62c2b 100644 --- a/.ddev/config.toml +++ b/.ddev/config.toml @@ -240,6 +240,24 @@ trace-captures = false ## Just in case __pycache__ is present in the root of the repo __pycache__ = false +# Integrations that were pinned in requirements-agent-release.txt but were not shipped +# in the listed Agent releases. Agent release generation uses these entries to skip +# false positives when building AGENT_INTEGRATIONS.md and Agent changelog data. +# Use by-integration for one-off skips: +# integration_name = ["7.78.0", "7.79.0"] +# Use by-agent-version-range for inclusive Agent version ranges: +# "7.74.0..7.78.0" = ["datadog-first-integration", "datadog-second-integration"] +[overrides.release.agent.unreleased-integrations.by-integration] + +[overrides.release.agent.unreleased-integrations.by-agent-version-range] +"7.74.0..7.78.0" = [ + "datadog-control-m", + "datadog-krakend", + "datadog-lustre", + "datadog-n8n", + "datadog-prefect", +] + # Explicitely add the platforms supported by an integration for those where the manifest has been # removed. # This is a temporary fix while we implement a metadata.json file that we can add to each integration diff --git a/.github/workflows/config/labeler.yml b/.github/workflows/config/labeler.yml index 0ea2917444c7a..0363017ec6cf1 100644 --- a/.github/workflows/config/labeler.yml +++ b/.github/workflows/config/labeler.yml @@ -22,7 +22,7 @@ dev/testing: - changed-files: - any-glob-to-any-file: - .github/workflows/** - - .codecov.yml + - code-coverage.datadog.yml dev/tooling: - changed-files: - any-glob-to-any-file: diff --git a/.github/workflows/dependency-wheel-promotion.yaml b/.github/workflows/dependency-wheel-promotion.yaml index 5e26cb45bef8a..e5b522f43586e 100644 --- a/.github/workflows/dependency-wheel-promotion.yaml +++ b/.github/workflows/dependency-wheel-promotion.yaml @@ -27,6 +27,25 @@ jobs: - name: Checkout trusted code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Find existing lifecycle comment + id: find_comment + uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 + with: + issue-number: ${{ inputs.pr_number }} + body-includes: "" + + - name: Post lifecycle comment (started) + id: started_comment + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 + with: + issue-number: ${{ inputs.pr_number }} + comment-id: ${{ steps.find_comment.outputs.comment-id }} + edit-mode: replace + body: | + + Wheel promotion started for commit `${{ inputs.head_sha }}` by @${{ github.actor }}. + Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + - name: Checkout PR lockfiles only uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -62,41 +81,57 @@ jobs: - name: Set dependency-wheel-promotion status to success uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + HEAD_SHA: ${{ inputs.head_sha }} with: script: | await github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, - sha: '${{ inputs.head_sha }}', + sha: process.env.HEAD_SHA, state: 'success', context: 'dependency-wheel-promotion', description: 'Wheels promoted to stable storage.', target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, }); - - name: Post success comment - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + - name: Update lifecycle comment (success) + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 with: - script: | - const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: ${{ inputs.pr_number }}, - body: `Wheels promoted to stable storage for commit ${{ inputs.head_sha }} by @${context.actor}. [Workflow run](${runUrl}).`, - }); + issue-number: ${{ inputs.pr_number }} + comment-id: ${{ steps.started_comment.outputs.comment-id }} + edit-mode: replace + body: | + + Wheels promoted to stable storage for commit `${{ inputs.head_sha }}` by @${{ github.actor }}. + Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - name: Set dependency-wheel-promotion status to error if: failure() uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + HEAD_SHA: ${{ inputs.head_sha }} with: script: | await github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, - sha: '${{ inputs.head_sha }}', + sha: process.env.HEAD_SHA, state: 'error', context: 'dependency-wheel-promotion', description: 'Wheel promotion failed. Check the Actions tab for details.', target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, }); + + - name: Update lifecycle comment (failure) + if: failure() + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 + with: + issue-number: ${{ inputs.pr_number }} + comment-id: ${{ steps.started_comment.outputs.comment-id }} + edit-mode: replace + body: | + + Wheel promotion failed for commit `${{ inputs.head_sha }}` by @${{ github.actor }}. + Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + Check the workflow logs before retrying. diff --git a/.github/workflows/master-windows.yml b/.github/workflows/master-windows.yml index 7f858efd91012..0e810f7e1b299 100644 --- a/.github/workflows/master-windows.yml +++ b/.github/workflows/master-windows.yml @@ -78,8 +78,6 @@ jobs: (success() || failure()) runs-on: ubuntu-latest permissions: - # needed for codecov, allows the action to get a JWT signed by Github - id-token: write contents: read steps: @@ -92,17 +90,10 @@ jobs: path: coverage-reports merge-multiple: false - - name: Upload coverage to Codecov - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - directory: coverage-reports - fail_ci_if_error: false - - name: Upload coverage to Datadog if: always() continue-on-error: true - uses: DataDog/coverage-upload-github-action@9bbbf86d16f7db1b14c5b885e61cf0d96053686a # v1.0.0 + uses: DataDog/coverage-upload-github-action@6c4bd935248daa6f0ef94e3e6ba71ad5ad079998 # v1.0.3 with: api_key: ${{ secrets.DD_API_KEY }} files: coverage-reports diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 7fcc2f903aaf0..374a0cc211ba5 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -3,27 +3,27 @@ name: Master on: push: branches: - - master + - master paths: # List of files/paths that should trigger the run. The intention is to avoid running all tests if the commit only includes changes on assets or README - - '*/datadog_checks/**' - - '*/tests/**' - - 'ddev/**' - - 'datadog_checks_base/**' - - 'datadog_checks_dev/**' - # Contains overrides for testing - - '.ddev/**' - # Want to ensure any change in workflows is validated - - '.github/workflows/**' - # Test matrices and dependencies - - '*/hatch.toml' - - '*/pyproject.toml' - # Some integrations might use this file to validate metrics emission - - '*/metadata.csv' - # In case some linting formatting config has changed - - 'pyproject.toml' + - "*/datadog_checks/**" + - "*/tests/**" + - "ddev/**" + - "datadog_checks_base/**" + - "datadog_checks_dev/**" + # Contains overrides for testing + - ".ddev/**" + # Want to ensure any change in workflows is validated + - ".github/workflows/**" + # Test matrices and dependencies + - "*/hatch.toml" + - "*/pyproject.toml" + # Some integrations might use this file to validate metrics emission + - "*/metadata.csv" + # In case some linting formatting config has changed + - "pyproject.toml" schedule: - - cron: '0 2 * * *' + - cron: "0 2 * * *" jobs: cache: @@ -31,7 +31,7 @@ jobs: test: needs: - - cache + - cache uses: ./.github/workflows/test-all.yml with: @@ -48,12 +48,12 @@ jobs: secrets: inherit permissions: - # needed for compute-matrix in test-target.yml - contents: read + # needed for compute-matrix in test-target.yml + contents: read publish-test-results: needs: - - test + - test if: success() || failure() concurrency: @@ -69,38 +69,29 @@ jobs: upload-coverage: needs: - - test + - test if: > !github.event.repository.private && (success() || failure()) runs-on: ubuntu-latest permissions: - # needed for codecov, allows the action to get a JWT signed by Github - id-token: write contents: read steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Download all coverage artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 - with: - pattern: coverage-* - path: coverage-reports - merge-multiple: false + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Upload coverage to Codecov - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - directory: coverage-reports - fail_ci_if_error: false + - name: Download all coverage artifacts + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + pattern: coverage-* + path: coverage-reports + merge-multiple: false - - name: Upload coverage to Datadog - if: always() - continue-on-error: true - uses: DataDog/coverage-upload-github-action@9bbbf86d16f7db1b14c5b885e61cf0d96053686a # v1.0.0 - with: - api_key: ${{ secrets.DD_API_KEY }} - files: coverage-reports - format: cobertura + - name: Upload coverage to Datadog + if: always() + continue-on-error: true + uses: DataDog/coverage-upload-github-action@6c4bd935248daa6f0ef94e3e6ba71ad5ad079998 # v1.0.3 + with: + api_key: ${{ secrets.DD_API_KEY }} + files: coverage-reports + format: cobertura diff --git a/.github/workflows/nightly-base-package-windows.yml b/.github/workflows/nightly-base-package-windows.yml index 12ba22fa2b292..53f1d4094f507 100644 --- a/.github/workflows/nightly-base-package-windows.yml +++ b/.github/workflows/nightly-base-package-windows.yml @@ -17,8 +17,6 @@ jobs: uses: ./.github/workflows/test-all-windows.yml permissions: - # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/.github/workflows/nightly-base-package.yml b/.github/workflows/nightly-base-package.yml index d28d529ebe0ea..1d3459ebc02e9 100644 --- a/.github/workflows/nightly-base-package.yml +++ b/.github/workflows/nightly-base-package.yml @@ -15,8 +15,6 @@ jobs: uses: ./.github/workflows/test-all.yml permissions: - # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/.github/workflows/pr-all-windows.yml b/.github/workflows/pr-all-windows.yml index 8f1d9c0e34268..e9bc38d991eb0 100644 --- a/.github/workflows/pr-all-windows.yml +++ b/.github/workflows/pr-all-windows.yml @@ -5,17 +5,17 @@ name: PR All Windows on: pull_request: paths: - - datadog_checks_base/datadog_checks/** - - datadog_checks_dev/datadog_checks/dev/*.py - - ddev/src/** - - "!agent_requirements.in" - # Also run if we modify the workflow files - - '.github/workflows/pr-all-windows.yml' - - '.github/workflows/test-target.yml' - - '.github/workflows/test-all-windows.yml' - # Also run in the action to install test-target scripts changes - - '.github/actions/setup-test-target-scripts/**' - - '.github/actions/setup-ddev/**' + - datadog_checks_base/datadog_checks/** + - datadog_checks_dev/datadog_checks/dev/*.py + - ddev/src/** + - "!agent_requirements.in" + # Also run if we modify the workflow files + - ".github/workflows/pr-all-windows.yml" + - ".github/workflows/test-target.yml" + - ".github/workflows/test-all-windows.yml" + # Also run in the action to install test-target scripts changes + - ".github/actions/setup-test-target-scripts/**" + - ".github/actions/setup-ddev/**" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.head_ref }} @@ -26,8 +26,8 @@ jobs: uses: ./.github/workflows/test-all-windows.yml permissions: - # needed for compute-matrix in test-target.yml - contents: read + # needed for compute-matrix in test-target.yml + contents: read with: repo: core @@ -39,45 +39,36 @@ jobs: save-event: needs: - - test + - test if: success() || failure() uses: ./.github/workflows/save-event.yml upload-coverage: needs: - - test + - test if: > !github.event.repository.private && (success() || failure()) runs-on: ubuntu-latest permissions: - # needed for codecov, allows the action to get a JWT signed by Github - id-token: write contents: read steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Download all coverage artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 - with: - pattern: coverage-* - path: coverage-reports - merge-multiple: false + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Upload coverage to Codecov - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - directory: coverage-reports - fail_ci_if_error: false + - name: Download all coverage artifacts + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + pattern: coverage-* + path: coverage-reports + merge-multiple: false - - name: Upload coverage to Datadog - if: always() - continue-on-error: true - uses: DataDog/coverage-upload-github-action@9bbbf86d16f7db1b14c5b885e61cf0d96053686a # v1.0.0 - with: - api_key: ${{ secrets.DD_API_KEY }} - files: coverage-reports - format: cobertura + - name: Upload coverage to Datadog + if: always() + continue-on-error: true + uses: DataDog/coverage-upload-github-action@6c4bd935248daa6f0ef94e3e6ba71ad5ad079998 # v1.0.3 + with: + api_key: ${{ secrets.DD_API_KEY }} + files: coverage-reports + format: cobertura diff --git a/.github/workflows/pr-all.yml b/.github/workflows/pr-all.yml index 9ea6dce99667e..0bb3bf1aafd48 100644 --- a/.github/workflows/pr-all.yml +++ b/.github/workflows/pr-all.yml @@ -3,20 +3,20 @@ name: PR All on: pull_request: paths: - - datadog_checks_base/datadog_checks/** - - datadog_checks_base/pyproject.toml - - datadog_checks_dev/datadog_checks/dev/*.py - - datadog_checks_dev/pyproject.toml - - ddev/src/** - - ddev/pyproject.toml - - "!agent_requirements.in" - # Also run if we modify the workflow files - - '.github/workflows/pr-all.yml' - - '.github/workflows/test-target.yml' - - '.github/workflows/test-all.yml' - # Also run if the action to install test-target scripts changes - - '.github/actions/setup-test-target-scripts/**' - - '.github/actions/setup-ddev/**' + - datadog_checks_base/datadog_checks/** + - datadog_checks_base/pyproject.toml + - datadog_checks_dev/datadog_checks/dev/*.py + - datadog_checks_dev/pyproject.toml + - ddev/src/** + - ddev/pyproject.toml + - "!agent_requirements.in" + # Also run if we modify the workflow files + - ".github/workflows/pr-all.yml" + - ".github/workflows/test-target.yml" + - ".github/workflows/test-all.yml" + # Also run if the action to install test-target scripts changes + - ".github/actions/setup-test-target-scripts/**" + - ".github/actions/setup-ddev/**" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.head_ref }} @@ -27,8 +27,8 @@ jobs: uses: ./.github/workflows/test-all.yml permissions: - # needed for compute-matrix in test-target.yml - contents: read + # needed for compute-matrix in test-target.yml + contents: read with: repo: core @@ -42,45 +42,36 @@ jobs: save-event: needs: - - test + - test if: success() || failure() uses: ./.github/workflows/save-event.yml upload-coverage: needs: - - test + - test if: > !github.event.repository.private && (success() || failure()) runs-on: ubuntu-latest permissions: - # needed for codecov, allows the action to get a JWT signed by Github - id-token: write contents: read steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Download all coverage artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 - with: - pattern: coverage-* - path: coverage-reports - merge-multiple: false + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Upload coverage to Codecov - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - directory: coverage-reports - fail_ci_if_error: false + - name: Download all coverage artifacts + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + pattern: coverage-* + path: coverage-reports + merge-multiple: false - - name: Upload coverage to Datadog - if: always() - continue-on-error: true - uses: DataDog/coverage-upload-github-action@9bbbf86d16f7db1b14c5b885e61cf0d96053686a # v1.0.0 - with: - api_key: ${{ secrets.DD_API_KEY }} - files: coverage-reports - format: cobertura + - name: Upload coverage to Datadog + if: always() + continue-on-error: true + uses: DataDog/coverage-upload-github-action@6c4bd935248daa6f0ef94e3e6ba71ad5ad079998 # v1.0.3 + with: + api_key: ${{ secrets.DD_API_KEY }} + files: coverage-reports + format: cobertura diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 8f3a14ece990e..759ea53b9edd6 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -33,7 +33,7 @@ jobs: test: needs: - - compute-matrix + - compute-matrix if: needs.compute-matrix.outputs.matrix != '[]' && github.event_name != 'merge_group' strategy: fail-fast: false @@ -64,7 +64,7 @@ jobs: test-minimum-base-package: needs: - - compute-matrix + - compute-matrix if: needs.compute-matrix.outputs.matrix != '[]' && github.event_name != 'merge_group' strategy: fail-fast: false @@ -96,56 +96,47 @@ jobs: save-event: needs: - - test - - test-minimum-base-package + - test + - test-minimum-base-package if: success() || failure() uses: ./.github/workflows/save-event.yml upload-coverage: needs: - - test - - test-minimum-base-package + - test + - test-minimum-base-package if: > !github.event.repository.private && (success() || failure()) && inputs.pytest-args != '-m flaky' runs-on: ubuntu-latest permissions: - # needed for codecov, allows the action to get a JWT signed by Github - id-token: write contents: read steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Download all coverage artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 - with: - pattern: coverage-* - path: coverage-reports - merge-multiple: false - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - directory: coverage-reports - fail_ci_if_error: false - - - name: Upload coverage to Datadog - if: always() - continue-on-error: true - uses: DataDog/coverage-upload-github-action@9bbbf86d16f7db1b14c5b885e61cf0d96053686a # v1.0.0 - with: - api_key: ${{ secrets.DD_API_KEY }} - files: coverage-reports - format: cobertura + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Download all coverage artifacts + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + pattern: coverage-* + path: coverage-reports + merge-multiple: false + + - name: Upload coverage to Datadog + if: always() + continue-on-error: true + uses: DataDog/coverage-upload-github-action@6c4bd935248daa6f0ef94e3e6ba71ad5ad079998 # v1.0.3 + with: + api_key: ${{ secrets.DD_API_KEY }} + files: coverage-reports + format: cobertura check: needs: - - test - - test-minimum-base-package + - test + - test-minimum-base-package # In integrations-core and integrations-extras repos the tests are flaky enough that # it would be a pain to merge PRs with the Merge Queue enabled. # While we work on the tests, we skip the job if it's triggered by Merge Queue. @@ -154,8 +145,8 @@ jobs: runs-on: ubuntu-latest steps: - - name: Check status of required jobs - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 - with: - jobs: ${{ toJSON(needs) }} - allowed-skips: test, test-minimum-base-package + - name: Check status of required jobs + uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 + with: + jobs: ${{ toJSON(needs) }} + allowed-skips: test, test-minimum-base-package diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a6810d189a3d0..7315836ddb7b4 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -24,7 +24,5 @@ jobs: secrets: inherit permissions: - # needed for codecov in pr-test.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/.github/workflows/release-dispatch.yml b/.github/workflows/release-dispatch.yml index ed5b1051575fa..42e8fcb02e954 100644 --- a/.github/workflows/release-dispatch.yml +++ b/.github/workflows/release-dispatch.yml @@ -20,6 +20,10 @@ on: description: "Commit SHA or ref to build from" required: false type: string + source-repo-branch: + description: "Branch that contains source-repo-ref, used to determine stable vs pre-release behavior" + required: false + type: string dry-run: description: >- When true, print what would be released and where without pushing tags @@ -56,20 +60,42 @@ jobs: batches: ${{ steps.release-dispatch.outputs.batches }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # Workflow tooling (composite actions + release scripts) always comes from + # integrations-core at the workflow's own commit. When called from another + # repo we don't have that commit available locally, so we fall back to + # master. This is decoupled from inputs.source-repo-ref on purpose so the + # release pipeline can build older refs without losing recent tooling. + - name: Checkout workflow tooling + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - ref: ${{ inputs.source-repo-ref || github.sha }} - fetch-depth: 0 # ddev needs full tag history + repository: DataDog/integrations-core + ref: ${{ github.repository == 'DataDog/integrations-core' && github.sha || 'master' }} + fetch-depth: 1 + sparse-checkout: .github + path: tooling - - name: Checkout integrations-core actions - if: github.repository != 'DataDog/integrations-core' + # Source tree to tag and validate. ddev release tag operates on HEAD of + # this checkout, so it must be at the ref the caller asked to release. + - name: Checkout source repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - repository: "DataDog/integrations-core" - ref: master - fetch-depth: 1 - sparse-checkout: .github/actions - path: core-temp + ref: ${{ inputs.source-repo-ref || github.sha }} + fetch-depth: 0 # ddev needs full tag history + path: source + + - name: Verify source ref is on release branch + if: inputs.source-repo-branch != '' + working-directory: source + env: + SOURCE_REF: ${{ inputs.source-repo-ref || github.sha }} + SOURCE_BRANCH: ${{ inputs.source-repo-branch }} + run: | + branch="${SOURCE_BRANCH#refs/heads/}" + git fetch --no-tags origin "+refs/heads/${branch}:refs/remotes/origin/${branch}" + if ! git merge-base --is-ancestor HEAD "refs/remotes/origin/${branch}"; then + echo "::error::source-repo-ref '${SOURCE_REF}' is not contained in source-repo-branch '${SOURCE_BRANCH}'" + exit 1 + fi - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -77,47 +103,42 @@ jobs: python-version: "${{ env.PYTHON_VERSION }}" - name: Install ddev - if: github.repository == 'DataDog/integrations-core' - uses: ./.github/actions/setup-ddev - with: - install-mode: pypi - ddev-version: "==${{ inputs.ddev-version || env.DEFAULT_DDEV_VERSION }}" - - - name: Install ddev - if: github.repository != 'DataDog/integrations-core' - uses: ./core-temp/.github/actions/setup-ddev + uses: ./tooling/.github/actions/setup-ddev with: install-mode: pypi ddev-version: "==${{ inputs.ddev-version || env.DEFAULT_DDEV_VERSION }}" - name: Configure ddev + working-directory: source env: SOURCE_REPO: ${{ inputs.source-repo || 'integrations-core' }} run: | REPO_SHORT="${SOURCE_REPO#integrations-}" ddev config set upgrade_check false - ddev config set repos.${REPO_SHORT} . + ddev config set repos.${REPO_SHORT} "$PWD" ddev config set repo ${REPO_SHORT} - name: Prepare dispatch id: prepare + working-directory: source env: DRY_RUN: ${{ inputs.dry-run }} SELECTED_PACKAGES: ${{ inputs.packages }} SOURCE_REPO: ${{ inputs.source-repo || 'integrations-core' }} REF: ${{ inputs.source-repo-ref || github.sha }} IS_STABLE_RELEASE: ${{ inputs.is-stable-release }} - run: python .github/workflows/scripts/release_prepare.py + run: python "$GITHUB_WORKSPACE/tooling/.github/workflows/scripts/release_prepare.py" - name: Build dispatch batches id: release-dispatch if: steps.prepare.outputs.has_packages == 'true' + working-directory: source env: PACKAGES: ${{ steps.prepare.outputs.packages }} SOURCE_REPO: ${{ inputs.source-repo || 'integrations-core' }} REF: ${{ inputs.source-repo-ref || github.sha }} DRY_RUN: ${{ inputs.dry-run }} - run: python .github/workflows/scripts/release_dispatch.py + run: python "$GITHUB_WORKSPACE/tooling/.github/workflows/scripts/release_dispatch.py" dispatch: name: Dispatch wheel builds (batch ${{ strategy.job-index }}) diff --git a/.github/workflows/release-trigger.yml b/.github/workflows/release-trigger.yml index d5d1c542d3c2c..060b3068fdf95 100644 --- a/.github/workflows/release-trigger.yml +++ b/.github/workflows/release-trigger.yml @@ -27,6 +27,10 @@ on: description: "Commit SHA or ref to build from" required: true type: string + source-repo-branch: + description: "Branch that contains source-repo-ref, used to determine stable vs pre-release behavior" + required: true + type: string dry-run: description: "Print what would be released without pushing tags or starting builds" required: false @@ -45,10 +49,14 @@ jobs: is-stable-release: ${{ steps.detect.outputs.is-stable-release }} steps: - id: detect - # Sets is-stable-release based on the branch: true for master/X.Y.x, false otherwise. - # Manual runs on master get "true", which blocks pre-release packages β€” conservative and intentional. + # Stable for master/X.Y.x, pre-release for alpha/beta/rc branches. + # Manual dispatches use source-repo-branch instead of GITHUB_REF so the + # release behavior follows the branch that contains source-repo-ref. + env: + RELEASE_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.source-repo-branch || github.ref_name }} run: | - if [[ "$GITHUB_REF" =~ ^refs/heads/(master|[0-9]+\.[0-9]+\.x)$ ]]; then + branch="${RELEASE_BRANCH#refs/heads/}" + if [[ "$branch" =~ ^(master|[0-9]+\.[0-9]+\.x)$ ]]; then echo "is-stable-release=true" >> "$GITHUB_OUTPUT" else echo "is-stable-release=false" >> "$GITHUB_OUTPUT" @@ -70,6 +78,7 @@ jobs: source-repo: integrations-core packages: ${{ inputs.packages || '' }} source-repo-ref: ${{ github.event_name == 'workflow_dispatch' && inputs.source-repo-ref || github.sha }} + source-repo-branch: ${{ github.event_name == 'workflow_dispatch' && inputs.source-repo-branch || github.ref_name }} dry-run: ${{ inputs.dry-run || false }} ddev-version: ${{ inputs.ddev-version || '' }} is-stable-release: ${{ needs.context.outputs.is-stable-release }} diff --git a/.github/workflows/test-agent-target.yml b/.github/workflows/test-agent-target.yml index 5b13976ce3381..a90be1218a566 100644 --- a/.github/workflows/test-agent-target.yml +++ b/.github/workflows/test-agent-target.yml @@ -63,7 +63,5 @@ jobs: context: "test-agent-target" secrets: inherit permissions: - # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/.github/workflows/test-agent-windows.yml b/.github/workflows/test-agent-windows.yml index dba54b173a1fe..5a1fde593115c 100644 --- a/.github/workflows/test-agent-windows.yml +++ b/.github/workflows/test-agent-windows.yml @@ -51,7 +51,5 @@ jobs: context: "test-agent" secrets: inherit permissions: - # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/.github/workflows/test-agent.yml b/.github/workflows/test-agent.yml index e04504ab428e8..c70461895d14d 100644 --- a/.github/workflows/test-agent.yml +++ b/.github/workflows/test-agent.yml @@ -51,7 +51,5 @@ jobs: context: "test-agent" secrets: inherit permissions: - # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/.github/workflows/test-fips-e2e.yml b/.github/workflows/test-fips-e2e.yml index 1035573ea9424..6b97beb9ffefd 100644 --- a/.github/workflows/test-fips-e2e.yml +++ b/.github/workflows/test-fips-e2e.yml @@ -17,10 +17,10 @@ on: type: string pull_request: paths: - - datadog_checks_base/datadog_checks/** - - datadog_checks_base/pyproject.toml + - datadog_checks_base/datadog_checks/** + - datadog_checks_base/pyproject.toml schedule: - - cron: '0 0,8,16 * * *' + - cron: "0 0,8,16 * * *" defaults: run: @@ -43,103 +43,92 @@ jobs: DD_TRACE_ANALYTICS_ENABLED: "true" permissions: - # needed for dd-sts and codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write - # needed for compute-matrix in test-target.yml - contents: read + # needed for dd-sts + id-token: write + # needed for compute-matrix in test-target.yml + contents: read steps: - - - name: Set environment variables with sanitized paths - run: | - JOB_NAME="test-fips-e2e" - - echo "TEST_RESULTS_DIR=$TEST_RESULTS_BASE_DIR/$JOB_NAME" >> $GITHUB_ENV - echo "TRACE_CAPTURE_FILE=$TRACE_CAPTURE_BASE_DIR/$JOB_NAME" >> $GITHUB_ENV - - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: "${{ env.PYTHON_VERSION }}" - - - name: Get Datadog credentials - id: dd-sts - uses: DataDog/dd-sts-action@2e8187910199bd93129520183c093e19aa585c75 # v1.0.0 - with: - policy: integrations-core-api-key - - - name: Install ddev from local folder - uses: ./.github/actions/setup-ddev - with: - install-mode: local - cache-profile: local-ddev-base - - - name: Configure ddev - run: |- - ddev config set upgrade_check false - ddev config set repos.core . - ddev config set repo core - - - name: Prepare for testing - env: - PYTHONUNBUFFERED: "1" - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} - ORACLE_DOCKER_USERNAME: ${{ secrets.ORACLE_DOCKER_USERNAME }} - ORACLE_DOCKER_PASSWORD: ${{ secrets.ORACLE_DOCKER_PASSWORD }} - DD_GITHUB_USER: ${{ github.actor }} - DD_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: ddev ci setup ${{ inputs.target || 'tls' }} - - - name: Run E2E tests with FIPS disabled - env: - DDEV_E2E_AGENT: "${{ inputs.agent-image || 'registry.datadoghq.com/agent-dev:master-py3' }}" - DD_API_KEY: "${{ steps.dd-sts.outputs.api_key }}" - run: | - ddev env test --base --new-env --junit ${{ inputs.target || 'tls' }} -- all -m "fips_off" - - - name: Run E2E tests with FIPS enabled - env: - DDEV_E2E_AGENT: "${{ inputs.agent-image-fips || 'registry.datadoghq.com/agent-dev:master-fips' }}" - DD_API_KEY: "${{ steps.dd-sts.outputs.api_key }}" - run: | - ddev env test --base --new-env --junit ${{ inputs.target || 'tls' }} -- all -k "fips_on" - - - name: Finalize test results - if: always() - run: |- - mkdir -p "${{ env.TEST_RESULTS_DIR }}" - if [[ -d ${{ inputs.target || 'tls' }}/junit ]]; then - mv ${{ inputs.target || 'tls' }}/junit/*.xml "${{ env.TEST_RESULTS_DIR }}" - fi - - - name: Upload test results - if: always() - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: "test-results-${{ inputs.target || 'tls' }}" - path: "${{ env.TEST_RESULTS_BASE_DIR }}" - - - name: Upload coverage data - if: > - !github.event.repository.private && - always() - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de - with: - use_oidc: true - files: "${{ inputs.target || 'tls' }}/coverage.xml" - flags: "${{ inputs.target || 'tls' }}" - - - name: Upload coverage to Datadog - if: > - !github.event.repository.private && - always() - continue-on-error: true - uses: DataDog/coverage-upload-github-action@9bbbf86d16f7db1b14c5b885e61cf0d96053686a # v1.0.0 - with: - api_key: ${{ secrets.DD_API_KEY }} - files: "${{ inputs.target || 'tls' }}/coverage.xml" - format: cobertura - flags: "${{ inputs.target || 'tls' }}" + - name: Set environment variables with sanitized paths + run: | + JOB_NAME="test-fips-e2e" + + echo "TEST_RESULTS_DIR=$TEST_RESULTS_BASE_DIR/$JOB_NAME" >> $GITHUB_ENV + echo "TRACE_CAPTURE_FILE=$TRACE_CAPTURE_BASE_DIR/$JOB_NAME" >> $GITHUB_ENV + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Get Datadog credentials + id: dd-sts + uses: DataDog/dd-sts-action@2e8187910199bd93129520183c093e19aa585c75 # v1.0.0 + with: + policy: integrations-core-api-key + + - name: Install ddev from local folder + uses: ./.github/actions/setup-ddev + with: + install-mode: local + cache-profile: local-ddev-base + + - name: Configure ddev + run: |- + ddev config set upgrade_check false + ddev config set repos.core . + ddev config set repo core + + - name: Prepare for testing + env: + PYTHONUNBUFFERED: "1" + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} + ORACLE_DOCKER_USERNAME: ${{ secrets.ORACLE_DOCKER_USERNAME }} + ORACLE_DOCKER_PASSWORD: ${{ secrets.ORACLE_DOCKER_PASSWORD }} + DD_GITHUB_USER: ${{ github.actor }} + DD_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: ddev ci setup ${{ inputs.target || 'tls' }} + + - name: Run E2E tests with FIPS disabled + env: + DDEV_E2E_AGENT: "${{ inputs.agent-image || 'registry.datadoghq.com/agent-dev:master-py3' }}" + DD_API_KEY: "${{ steps.dd-sts.outputs.api_key }}" + run: | + ddev env test --base --new-env --junit ${{ inputs.target || 'tls' }} -- all -m "fips_off" + + - name: Run E2E tests with FIPS enabled + env: + DDEV_E2E_AGENT: "${{ inputs.agent-image-fips || 'registry.datadoghq.com/agent-dev:master-fips' }}" + DD_API_KEY: "${{ steps.dd-sts.outputs.api_key }}" + run: | + ddev env test --base --new-env --junit ${{ inputs.target || 'tls' }} -- all -k "fips_on" + + - name: Finalize test results + if: always() + run: |- + mkdir -p "${{ env.TEST_RESULTS_DIR }}" + if [[ -d ${{ inputs.target || 'tls' }}/junit ]]; then + mv ${{ inputs.target || 'tls' }}/junit/*.xml "${{ env.TEST_RESULTS_DIR }}" + fi + + - name: Upload test results + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: "test-results-${{ inputs.target || 'tls' }}" + path: "${{ env.TEST_RESULTS_BASE_DIR }}" + + - name: Upload coverage to Datadog + if: > + !github.event.repository.private && + always() + continue-on-error: true + uses: DataDog/coverage-upload-github-action@6c4bd935248daa6f0ef94e3e6ba71ad5ad079998 # v1.0.3 + with: + api_key: ${{ secrets.DD_API_KEY }} + files: "${{ inputs.target || 'tls' }}/coverage.xml" + format: cobertura + flags: "${{ inputs.target || 'tls' }}" diff --git a/.github/workflows/weekly-latest-windows.yml b/.github/workflows/weekly-latest-windows.yml index 435bc089c4a6c..032e60eca0019 100644 --- a/.github/workflows/weekly-latest-windows.yml +++ b/.github/workflows/weekly-latest-windows.yml @@ -18,7 +18,5 @@ jobs: context: "weekly-latest" secrets: inherit permissions: - # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/.github/workflows/weekly-latest.yml b/.github/workflows/weekly-latest.yml index 1fe57dbc0fe3a..5b8b3747a7ada 100644 --- a/.github/workflows/weekly-latest.yml +++ b/.github/workflows/weekly-latest.yml @@ -16,7 +16,5 @@ jobs: context: "weekly-latest" secrets: inherit permissions: - # needed for codecov in test-target.yml, allows the action to get a JWT signed by Github - id-token: write # needed for compute-matrix in test-target.yml contents: read diff --git a/README.md b/README.md index fd081156a41a1..bf828e8b81e7d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ | | | | --- | --- | -| CI/CD | [![CI - Test][1]][2] [![CI - Coverage][17]][18] | +| CI/CD | [![CI - Test][1]][2] | | Docs | [![Docs - Release][19]][20] | | Meta | [![Hatch project][26]][27] [![Linting - Ruff][24]][25] [![Code style - black][21]][22] [![Typing - Mypy][28]][29] [![License - BSD-3-Clause][30]][31] | @@ -41,8 +41,6 @@ For more information on integrations, please reference our [documentation][11] a [13]: https://docs.datadoghq.com/help/ [15]: https://github.com/DataDog/integrations-core/blob/6.2.1/requirements-integration-core.txt [16]: https://github.com/DataDog/integrations-core/blob/ea2dfbf1e8859333af4c8db50553eb72a3b466f9/requirements-agent-release.txt -[17]: https://codecov.io/github/DataDog/integrations-core/coverage.svg?branch=master -[18]: https://codecov.io/github/DataDog/integrations-core?branch=master [19]: https://github.com/DataDog/integrations-core/workflows/docs/badge.svg [20]: https://github.com/DataDog/integrations-core/actions?workflow=docs [21]: https://img.shields.io/badge/code%20style-black-000000.svg diff --git a/anthropic_compliance_logs/README.md b/anthropic_compliance_logs/README.md index 0883bbbf73b36..eed55df5de67a 100644 --- a/anthropic_compliance_logs/README.md +++ b/anthropic_compliance_logs/README.md @@ -41,7 +41,7 @@ The Compliance API is available to Anthropic Enterprise plan customers with the 1. Wait up to 5 minutes for the first crawl. 2. Open [Log Explorer][3] and filter on `source:claude-compliance-logs`. -3. Confirm logs appear with `evt.name` values such as `claude_chat_viewed`, `admin_api_key_created`, or `user_signed_in_sso`. +3. Confirm logs appear with `evt.name` values such as `claude_chat_viewed`, `admin_api_key_created`, or `sso_login_succeeded`. ## Data Collected @@ -51,7 +51,7 @@ The integration collects audit activity logs from `GET /v1/compliance/activities - A timestamp (`created_at`) with microsecond precision - An actor (user, API key, SCIM, or system) with email, user ID, IP address, and User-Agent when applicable -- An activity `type` such as `user_signed_in_sso`, `admin_api_key_created`, `org_user_invite_accepted`, or `claude_chat_viewed` (150+ activity types across 35+ categories) +- An activity `type` such as `sso_login_succeeded`, `admin_api_key_created`, `org_user_invite_accepted`, or `claude_chat_viewed` (150+ activity types across 35+ categories) - Organization and workspace context Logs are tagged `source:claude-compliance-logs` and processed by a Datadog log pipeline that flattens the actor object into standard `usr.*` and `network.client.*` attributes and enriches the source IP with GeoIP and the User-Agent string. diff --git a/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs.yaml b/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs.yaml index 975da722e2bbf..d8065b01a1f84 100644 --- a/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs.yaml +++ b/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs.yaml @@ -88,6 +88,104 @@ facets: name: User ID path: usr.id source: log + - groups: + - OCSF + name: Activity ID + path: ocsf.activity_id + source: log + type: integer + - groups: + - OCSF + name: Activity Name + path: ocsf.activity_name + source: log + - groups: + - OCSF + name: Category + path: ocsf.category_name + source: log + - groups: + - OCSF + name: Category ID + path: ocsf.category_uid + source: log + type: integer + - groups: + - OCSF + name: Class + path: ocsf.class_name + source: log + - groups: + - OCSF + name: Class ID + path: ocsf.class_uid + source: log + type: integer + - groups: + - OCSF + name: Type ID + path: ocsf.type_uid + source: log + type: integer + - groups: + - OCSF + name: Severity ID + path: ocsf.severity_id + source: log + type: integer + - groups: + - OCSF + name: Status + path: ocsf.status + source: log + - groups: + - OCSF + name: Status ID + path: ocsf.status_id + source: log + type: integer + - groups: + - OCSF + name: Event Code + path: ocsf.metadata.event_code + source: log + - groups: + - OCSF + name: Product Name + path: ocsf.metadata.product.name + source: log + - groups: + - OCSF + name: Vendor Name + path: ocsf.metadata.product.vendor_name + source: log + - groups: + - OCSF + name: Email Address + path: ocsf.actor.user.email_addr + source: log + - groups: + - OCSF + name: Unique ID + path: ocsf.actor.user.uid + source: log + - groups: + - OCSF + name: Source IP Address + path: ocsf.src_endpoint.ip + source: log + - groups: + - OCSF + name: Auth Protocol ID + path: ocsf.auth_protocol_id + source: log + type: integer + - groups: + - OCSF + name: Multi Factor Authentication + path: ocsf.is_mfa + source: log + type: boolean pipeline: type: pipeline name: Claude Compliance Logs @@ -108,7 +206,7 @@ pipeline: sourceType: attribute target: evt.name targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: true - type: attribute-remapper name: Map `actor.email_address` to `usr.email` @@ -118,7 +216,7 @@ pipeline: sourceType: attribute target: usr.email targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: true - type: attribute-remapper name: Map `actor.user_id` to `usr.id` @@ -128,7 +226,7 @@ pipeline: sourceType: attribute target: usr.id targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: true - type: attribute-remapper name: Map `actor.ip_address` to `network.client.ip` @@ -138,7 +236,7 @@ pipeline: sourceType: attribute target: network.client.ip targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: true - type: attribute-remapper name: Map `actor.user_agent` to `http.useragent` @@ -148,7 +246,7 @@ pipeline: sourceType: attribute target: http.useragent targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: true - type: geo-ip-parser name: GeoIP parser on `network.client.ip` @@ -163,3 +261,1539 @@ pipeline: - http.useragent target: http.useragent_details encoded: false + - type: pipeline + name: OCSF pre transformations + enabled: true + ocsf: + isOcsf: true + filter: + query: "*" + processors: + - type: string-builder-processor + name: Add product name + enabled: true + template: "Claude" + target: ocsf.metadata.product.name + replaceMissing: false + - type: string-builder-processor + name: Add product vendor + enabled: true + template: "Anthropic" + target: ocsf.metadata.product.vendor_name + replaceMissing: false + - type: attribute-remapper + name: Map `type` to `ocsf.metadata.event_code` + enabled: true + sources: + - type + sourceType: attribute + target: ocsf.metadata.event_code + targetType: attribute + preserveSource: true + overrideOnConflict: true + - type: attribute-remapper + name: Map `created_at` to `ocsf.metadata.original_time` + enabled: true + sources: + - created_at + sourceType: attribute + target: ocsf.metadata.original_time + targetType: attribute + preserveSource: true + overrideOnConflict: true + - type: grok-parser + name: Parse `created_at` to `ocsf.time` + enabled: true + source: created_at + samples: + - "2026-05-22T15:21:54.358426Z" + grok: + supportRules: "" + matchRules: | + parsing_time %{date("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"):ocsf.time} + parsing_time_ms %{date("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"):ocsf.time} + parsing_time_s %{date("yyyy-MM-dd'T'HH:mm:ss'Z'"):ocsf.time} + - type: pipeline + name: OCSF sub pipeline for class Account Change [3001] - target events + enabled: true + ocsf: + isOcsf: true + filter: + query: "@ocsf.metadata.event_code:(org_user_deleted OR org_user_invite_sent)" + processors: + - type: schema-processor + name: Apply OCSF schema for 3001 + enabled: true + mappers: + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` + sources: + - ocsf.metadata.event_code + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id` to `ocsf.metadata.uid` + sources: + - id + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.original_time` to `ocsf.metadata.original_time` + sources: + - ocsf.metadata.original_time + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address` to `ocsf.actor.user.email_addr` + sources: + - actor.email_address + target: ocsf.actor.user.email_addr + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_id`, `actor.admin_api_key_id` to `ocsf.actor.user.uid` + sources: + - actor.user_id + - actor.admin_api_key_id + target: ocsf.actor.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.ip_address` to `ocsf.src_endpoint.ip` + sources: + - actor.ip_address + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_agent` to `ocsf.http_request.user_agent` + sources: + - actor.user_agent + target: ocsf.http_request.user_agent + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `organization_id`, `organization_uuid` to `ocsf.actor.user.org.uid` + sources: + - organization_id + - organization_uuid + target: ocsf.actor.user.org.uid + preserveSource: true + overrideOnConflict: false + - type: schema-remapper + name: Map `deleted_user_id` to `ocsf.user.uid` + sources: + - deleted_user_id + target: ocsf.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `deleted_user_email`, `invited_email` to `ocsf.user.email_addr` + sources: + - deleted_user_email + - invited_email + target: ocsf.user.email_addr + preserveSource: true + overrideOnConflict: false + - type: schema-remapper + name: Map `deleted_user_email`, `invited_email` to `ocsf.user.name` + sources: + - deleted_user_email + - invited_email + target: ocsf.user.name + preserveSource: true + overrideOnConflict: false + - type: schema-category-mapper + name: ocsf.actor.user.type_id + categories: + - filter: + query: "@actor.type:user_actor" + name: User + id: 1 + - filter: + query: "@actor.type:admin*" + name: Admin + id: 2 + - filter: + query: "-@actor.type:*" + name: Unknown + id: 0 + - filter: + query: "@actor.type:*" + name: Other + id: 99 + targets: + name: ocsf.actor.user.type + id: ocsf.actor.user.type_id + fallback: + values: + ocsf.actor.user.type: Other + ocsf.actor.user.type_id: "99" + sources: + ocsf.actor.user.type: + - actor.type + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@ocsf.metadata.event_code:org_user_invite_sent" + name: Create + id: 1 + - filter: + query: "@ocsf.metadata.event_code:org_user_deleted" + name: Delete + id: 6 + - filter: + query: "@ocsf.metadata.event_code:*" + name: Other + id: 99 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - type + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "*" + name: Success + id: 1 + targets: + name: ocsf.status + id: ocsf.status_id + schema: + schemaType: ocsf + version: 1.5.0 + className: Account Change + classUid: 3001 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Account Change [3001] - self events + enabled: true + ocsf: + isOcsf: true + filter: + query: "@ocsf.metadata.event_code:(org_user_invite_accepted OR claude_user_settings_updated OR *_api_key_*)" + processors: + - type: schema-processor + name: Apply OCSF schema for 3001 + enabled: true + mappers: + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` + sources: + - ocsf.metadata.event_code + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id` to `ocsf.metadata.uid` + sources: + - id + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.original_time` to `ocsf.metadata.original_time` + sources: + - ocsf.metadata.original_time + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address` to `ocsf.actor.user.email_addr` + sources: + - actor.email_address + target: ocsf.actor.user.email_addr + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_id`, `actor.admin_api_key_id` to `ocsf.actor.user.uid` + sources: + - actor.user_id + - actor.admin_api_key_id + target: ocsf.actor.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.ip_address` to `ocsf.src_endpoint.ip` + sources: + - actor.ip_address + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_agent` to `ocsf.http_request.user_agent` + sources: + - actor.user_agent + target: ocsf.http_request.user_agent + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `organization_id`, `organization_uuid` to `ocsf.actor.user.org.uid` + sources: + - organization_id + - organization_uuid + target: ocsf.actor.user.org.uid + preserveSource: true + overrideOnConflict: false + - type: schema-remapper + name: Map `actor.user_id`, `actor.admin_api_key_id` to `ocsf.user.uid` + sources: + - actor.user_id + - actor.admin_api_key_id + target: ocsf.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address` to `ocsf.user.email_addr` + sources: + - actor.email_address + target: ocsf.user.email_addr + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.actor.user.type_id + categories: + - filter: + query: "@actor.type:user_actor" + name: User + id: 1 + - filter: + query: "@actor.type:admin*" + name: Admin + id: 2 + - filter: + query: "-@actor.type:*" + name: Unknown + id: 0 + - filter: + query: "@actor.type:*" + name: Other + id: 99 + targets: + name: ocsf.actor.user.type + id: ocsf.actor.user.type_id + fallback: + values: + ocsf.actor.user.type: Other + ocsf.actor.user.type_id: "99" + sources: + ocsf.actor.user.type: + - actor.type + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@ocsf.metadata.event_code:(*_api_key_created OR org_user_invite_accepted)" + name: Create + id: 1 + - filter: + query: "@ocsf.metadata.event_code:*_api_key_updated AND @updates.current_value:active" + name: Enable + id: 2 + - filter: + query: "@ocsf.metadata.event_code:*_api_key_updated AND @updates.current_value:archived" + name: Disable + id: 5 + - filter: + query: "@ocsf.metadata.event_code:*_api_key_deleted" + name: Delete + id: 6 + - filter: + query: "@ocsf.metadata.event_code:*" + name: Other + id: 99 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - type + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "*" + name: Success + id: 1 + targets: + name: ocsf.status + id: ocsf.status_id + schema: + schemaType: ocsf + version: 1.5.0 + className: Account Change + classUid: 3001 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Authentication [3002] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@ocsf.metadata.event_code:(sso_* OR magic_link_* OR social_login_* OR anonymous_mobile_login_* OR user_logged_out)" + processors: + - type: string-builder-processor + name: Add service name + enabled: true + template: "Claude" + target: ocsf.service.name + replaceMissing: false + - type: schema-processor + name: Apply OCSF schema for 3002 + enabled: true + mappers: + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` + sources: + - ocsf.metadata.event_code + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id` to `ocsf.metadata.uid` + sources: + - id + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.original_time` to `ocsf.metadata.original_time` + sources: + - ocsf.metadata.original_time + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address`, `actor.unauthenticated_email_address` to `ocsf.actor.user.email_addr` + sources: + - actor.email_address + - actor.unauthenticated_email_address + target: ocsf.actor.user.email_addr + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_id` to `ocsf.actor.user.uid` + sources: + - actor.user_id + target: ocsf.actor.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address`, `actor.unauthenticated_email_address` to `ocsf.actor.user.name` + sources: + - actor.email_address + - actor.unauthenticated_email_address + target: ocsf.actor.user.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address`, `actor.unauthenticated_email_address` to `ocsf.user.email_addr` + sources: + - actor.email_address + - actor.unauthenticated_email_address + target: ocsf.user.email_addr + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_id` to `ocsf.user.uid` + sources: + - actor.user_id + target: ocsf.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address`, `actor.unauthenticated_email_address` to `ocsf.user.name` + sources: + - actor.email_address + - actor.unauthenticated_email_address + target: ocsf.user.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.ip_address` to `ocsf.src_endpoint.ip` + sources: + - actor.ip_address + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_agent` to `ocsf.http_request.user_agent` + sources: + - actor.user_agent + target: ocsf.http_request.user_agent + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `organization_id`, `organization_uuid` to `ocsf.actor.user.org.uid` + sources: + - organization_id + - organization_uuid + target: ocsf.actor.user.org.uid + preserveSource: true + overrideOnConflict: false + - type: schema-remapper + name: Map `ocsf.service.name` to `ocsf.service.name` + sources: + - ocsf.service.name + target: ocsf.service.name + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.actor.user.type_id + categories: + - filter: + query: "@actor.type:user_actor" + name: User + id: 1 + - filter: + query: "@actor.type:admin*" + name: Admin + id: 2 + - filter: + query: "-@actor.type:*" + name: Unknown + id: 0 + - filter: + query: "@actor.type:*" + name: Other + id: 99 + targets: + name: ocsf.actor.user.type + id: ocsf.actor.user.type_id + fallback: + values: + ocsf.actor.user.type: Other + ocsf.actor.user.type_id: "99" + sources: + ocsf.actor.user.type: + - actor.type + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@ocsf.metadata.event_code:(sso_* OR magic_link_* OR social_login_* OR anonymous_mobile_login_*)" + name: Logon + id: 1 + - filter: + query: "@ocsf.metadata.event_code:user_logged_out" + name: Logoff + id: 2 + - filter: + query: "@ocsf.metadata.event_code:*" + name: Other + id: 99 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - type + - type: schema-remapper + name: Map `provider` to `ocsf.actor.idp.name` + sources: + - provider + target: ocsf.actor.idp.name + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.auth_protocol_id + categories: + - filter: + query: "@auth_method:sso OR @ocsf.metadata.event_code:sso_*" + name: SAML + id: 5 + - filter: + query: "@auth_method:social OR @ocsf.metadata.event_code:social_login_succeeded" + name: OpenID + id: 4 + - filter: + query: "@ocsf.metadata.event_code:*" + name: Other + id: 99 + targets: + name: ocsf.auth_protocol + id: ocsf.auth_protocol_id + fallback: + values: + ocsf.auth_protocol: Other + ocsf.auth_protocol_id: "99" + sources: + ocsf.auth_protocol: + - auth_method + - type + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "@ocsf.metadata.event_code:(sso_login_succeeded OR magic_link_login_succeeded OR social_login_succeeded OR sso_second_factor_magic_link OR user_logged_out)" + name: Success + id: 1 + - filter: + query: "@ocsf.metadata.event_code:(sso_login_failed OR magic_link_login_failed)" + name: Failure + id: 2 + - filter: + query: "@ocsf.metadata.event_code:(sso_login_initiated OR magic_link_login_initiated OR anonymous_mobile_login_attempted)" + name: Unknown + id: 0 + - filter: + query: "@ocsf.metadata.event_code:*" + name: Other + id: 99 + targets: + name: ocsf.status + id: ocsf.status_id + fallback: + values: + ocsf.status: Other + ocsf.status_id: "99" + sources: + ocsf.status: + - type + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + schema: + schemaType: ocsf + version: 1.5.0 + className: Authentication + classUid: 3002 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class User Access Management [3005] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@ocsf.metadata.event_code:(role_assignment_granted OR role_assignment_revoked)" + processors: + - type: grok-parser + name: Parse `created_at` to `ocsf.time` + enabled: true + source: created_at + samples: + - "2026-05-22T15:21:54.358426Z" + grok: + supportRules: "" + matchRules: | + parsing_time %{date("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"):ocsf.time} + parsing_time_ms %{date("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"):ocsf.time} + parsing_time_s %{date("yyyy-MM-dd'T'HH:mm:ss'Z'"):ocsf.time} + - type: attribute-remapper + name: Map `role` to `ocsf.privilege` + enabled: true + sources: + - role + sourceType: attribute + target: ocsf.privilege + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: array-processor + name: Move privilege into privileges array + enabled: true + operation: + source: ocsf.privilege + target: ocsf.privileges + preserveSource: false + type: append + - type: attribute-remapper + name: Map `resource_id` to `ocsf.resource.uid` + enabled: true + sources: + - resource_id + sourceType: attribute + target: ocsf.resource.uid + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: Map `resource_type` to `ocsf.resource.type` + enabled: true + sources: + - resource_type + sourceType: attribute + target: ocsf.resource.type + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: array-processor + name: Move resource into resources array + enabled: true + operation: + source: ocsf.resource + target: ocsf.resources + preserveSource: false + type: append + - type: schema-processor + name: Apply OCSF schema for 3005 + enabled: true + mappers: + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` + sources: + - ocsf.metadata.event_code + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id` to `ocsf.metadata.uid` + sources: + - id + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.original_time` to `ocsf.metadata.original_time` + sources: + - ocsf.metadata.original_time + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address` to `ocsf.actor.user.email_addr` + sources: + - actor.email_address + target: ocsf.actor.user.email_addr + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_id` to `ocsf.actor.user.uid` + sources: + - actor.user_id + target: ocsf.actor.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.ip_address` to `ocsf.src_endpoint.ip` + sources: + - actor.ip_address + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_agent` to `ocsf.http_request.user_agent` + sources: + - actor.user_agent + target: ocsf.http_request.user_agent + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `organization_id`, `organization_uuid` to `ocsf.user.org.uid` + sources: + - organization_id + - organization_uuid + target: ocsf.user.org.uid + preserveSource: true + overrideOnConflict: false + - type: schema-remapper + name: Map `target_id` to `ocsf.user.uid` + sources: + - target_id + target: ocsf.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.privileges` to `ocsf.privileges` + sources: + - ocsf.privileges + target: ocsf.privileges + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.resources` to `ocsf.resources` + sources: + - ocsf.resources + target: ocsf.resources + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.actor.user.type_id + categories: + - filter: + query: "@actor.type:user_actor" + name: User + id: 1 + - filter: + query: "@actor.type:admin*" + name: Admin + id: 2 + - filter: + query: "-@actor.type:*" + name: Unknown + id: 0 + - filter: + query: "@actor.type:*" + name: Other + id: 99 + targets: + name: ocsf.actor.user.type + id: ocsf.actor.user.type_id + fallback: + values: + ocsf.actor.user.type: Other + ocsf.actor.user.type_id: "99" + sources: + ocsf.actor.user.type: + - actor.type + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@ocsf.metadata.event_code:role_assignment_granted" + name: Assign Privileges + id: 1 + - filter: + query: "@ocsf.metadata.event_code:role_assignment_revoked" + name: Revoke Privileges + id: 2 + - filter: + query: "@ocsf.metadata.event_code:*" + name: Other + id: 99 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - type + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "*" + name: Success + id: 1 + targets: + name: ocsf.status + id: ocsf.status_id + schema: + schemaType: ocsf + version: 1.5.0 + className: User Access Management + classUid: 3005 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Web Resources Activity [6001] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@ocsf.metadata.event_code:(claude_chat_* OR claude_project_* OR claude_file_* OR claude_artifact_* OR claude_skill_*)" + processors: + - type: attribute-remapper + name: Map `claude_chat_id`, `claude_file_id`, `claude_project_document_id`, `claude_artifact_id`, `skill_id`, `claude_project_id` to `ocsf.web_resource.uid` + enabled: true + sources: + - claude_chat_id + - claude_file_id + - claude_project_document_id + - claude_artifact_id + - skill_id + - claude_project_id + sourceType: attribute + target: ocsf.web_resource.uid + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: Map `filename`, `skill_name` to `ocsf.web_resource.name` + enabled: true + sources: + - filename + - skill_name + sourceType: attribute + target: ocsf.web_resource.name + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: array-processor + name: Move web_resource into web_resources array + enabled: true + operation: + source: ocsf.web_resource + target: ocsf.web_resources + preserveSource: false + type: append + - type: schema-processor + name: Apply OCSF schema for 6001 + enabled: true + mappers: + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` + sources: + - ocsf.metadata.event_code + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id` to `ocsf.metadata.uid` + sources: + - id + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.original_time` to `ocsf.metadata.original_time` + sources: + - ocsf.metadata.original_time + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.email_address` to `ocsf.src_endpoint.owner.email_addr` + sources: + - actor.email_address + target: ocsf.src_endpoint.owner.email_addr + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_id`, `actor.admin_api_key_id` to `ocsf.src_endpoint.owner.uid` + sources: + - actor.user_id + - actor.admin_api_key_id + target: ocsf.src_endpoint.owner.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.ip_address` to `ocsf.src_endpoint.ip` + sources: + - actor.ip_address + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_agent` to `ocsf.http_request.user_agent` + sources: + - actor.user_agent + target: ocsf.http_request.user_agent + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `organization_id`, `organization_uuid` to `ocsf.src_endpoint.owner.org.uid` + sources: + - organization_id + - organization_uuid + target: ocsf.src_endpoint.owner.org.uid + preserveSource: true + overrideOnConflict: false + - type: schema-remapper + name: Map `ocsf.web_resources` to `ocsf.web_resources` + sources: + - ocsf.web_resources + target: ocsf.web_resources + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.src_endpoint.owner.type_id + categories: + - filter: + query: "@actor.type:user_actor" + name: User + id: 1 + - filter: + query: "@actor.type:admin*" + name: Admin + id: 2 + - filter: + query: "-@actor.type:*" + name: Unknown + id: 0 + - filter: + query: "@actor.type:*" + name: Other + id: 99 + targets: + name: ocsf.src_endpoint.owner.type + id: ocsf.src_endpoint.owner.type_id + fallback: + values: + ocsf.src_endpoint.owner.type: Other + ocsf.src_endpoint.owner.type_id: "99" + sources: + ocsf.src_endpoint.owner.type: + - actor.type + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@ocsf.metadata.event_code:(*_created OR *_uploaded)" + name: Create + id: 1 + - filter: + query: "@ocsf.metadata.event_code:*_viewed" + name: Read + id: 2 + - filter: + query: "@ocsf.metadata.event_code:(*_updated OR *_replaced)" + name: Update + id: 3 + - filter: + query: "@ocsf.metadata.event_code:*_deleted" + name: Delete + id: 4 + - filter: + query: "@ocsf.metadata.event_code:*" + name: Other + id: 99 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - type + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "*" + name: Success + id: 1 + targets: + name: ocsf.status + id: ocsf.status_id + schema: + schemaType: ocsf + version: 1.5.0 + className: Web Resources Activity + classUid: 6001 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class API Activity [6003] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@ocsf.metadata.event_code:compliance_api_accessed" + processors: + - type: schema-processor + name: Apply OCSF schema for 6003 + enabled: true + mappers: + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` + sources: + - ocsf.metadata.event_code + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id` to `ocsf.metadata.uid` + sources: + - id + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.original_time` to `ocsf.metadata.original_time` + sources: + - ocsf.metadata.original_time + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.api_key_id` to `ocsf.actor.user.uid` + sources: + - actor.api_key_id + target: ocsf.actor.user.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.api_key_id` to `ocsf.actor.app_uid` + sources: + - actor.api_key_id + target: ocsf.actor.app_uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.ip_address` to `ocsf.src_endpoint.ip` + sources: + - actor.ip_address + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `actor.user_agent` to `ocsf.http_request.user_agent` + sources: + - actor.user_agent + target: ocsf.http_request.user_agent + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `request_method` to `ocsf.http_request.http_method` + sources: + - request_method + target: ocsf.http_request.http_method + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `url` to `ocsf.http_request.url.url_string` + sources: + - url + target: ocsf.http_request.url.url_string + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `request_id` to `ocsf.http_request.uid` + sources: + - request_id + target: ocsf.http_request.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `status_code` to `ocsf.http_response.code` + sources: + - status_code + target: ocsf.http_response.code + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `request_method` to `ocsf.api.operation` + sources: + - request_method + target: ocsf.api.operation + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `request_id` to `ocsf.api.request.uid` + sources: + - request_id + target: ocsf.api.request.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `status_code` to `ocsf.api.response.code` + sources: + - status_code + target: ocsf.api.response.code + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-category-mapper + name: ocsf.actor.user.type_id + categories: + - filter: + query: "@actor.type:user_actor" + name: User + id: 1 + - filter: + query: "@actor.type:admin*" + name: Admin + id: 2 + - filter: + query: "-@actor.type:*" + name: Unknown + id: 0 + - filter: + query: "@actor.type:*" + name: Other + id: 99 + targets: + name: ocsf.actor.user.type + id: ocsf.actor.user.type_id + fallback: + values: + ocsf.actor.user.type: Other + ocsf.actor.user.type_id: "99" + sources: + ocsf.actor.user.type: + - actor.type + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@request_method:POST" + name: Create + id: 1 + - filter: + query: "@request_method:GET" + name: Read + id: 2 + - filter: + query: "@request_method:(PUT OR PATCH)" + name: Update + id: 3 + - filter: + query: "@request_method:DELETE" + name: Delete + id: 4 + - filter: + query: "@request_method:*" + name: Other + id: 99 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - request_method + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "@status_code:[200 TO 299]" + name: Success + id: 1 + - filter: + query: "@status_code:[400 TO 599]" + name: Failure + id: 2 + - filter: + query: "-@status_code:*" + name: Unknown + id: 0 + - filter: + query: "@status_code:*" + name: Other + id: 99 + targets: + name: ocsf.status + id: ocsf.status_id + fallback: + values: + ocsf.status: Other + ocsf.status_id: "99" + sources: + ocsf.status: + - status_code + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "@status_code:[200 TO 399]" + name: Informational + id: 1 + - filter: + query: "@status_code:[400 TO 499]" + name: Medium + id: 3 + - filter: + query: "@status_code:[500 TO 599]" + name: High + id: 4 + - filter: + query: "-@status_code:*" + name: Unknown + id: 0 + - filter: + query: "@status_code:*" + name: Other + id: 99 + targets: + name: ocsf.severity + id: ocsf.severity_id + fallback: + values: + ocsf.severity: Other + ocsf.severity_id: "99" + sources: + ocsf.severity: + - status_code + schema: + schemaType: ocsf + version: 1.5.0 + className: API Activity + classUid: 6003 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Base Event [0] + enabled: true + ocsf: + isOcsf: true + filter: + query: "-@ocsf.class_uid:*" + processors: + - type: schema-processor + name: Apply OCSF schema for 0 + enabled: true + mappers: + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` + sources: + - ocsf.metadata.event_code + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id` to `ocsf.metadata.uid` + sources: + - id + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.original_time` to `ocsf.metadata.original_time` + sources: + - ocsf.metadata.original_time + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.status + id: ocsf.status_id + schema: + schemaType: ocsf + version: 1.5.0 + className: Base Event + classUid: 0 + extensions: [] + profiles: [] diff --git a/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs_tests.yaml b/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs_tests.yaml index 17d3d7c1f3948..a3fb1fd394044 100644 --- a/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs_tests.yaml +++ b/anthropic_compliance_logs/assets/logs/anthropic-compliance-logs_tests.yaml @@ -1,72 +1,2745 @@ id: "anthropic-compliance-logs" tests: - - sample: |- + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36" + }, + "claude_artifact_id" : "claude_artifact_01EXAMPLEARTIFACT0", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:25:13.701734Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_artifact_viewed" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_artifact_id: "claude_artifact_01EXAMPLEARTIFACT0" + created_at: "2026-05-22T15:25:13.701734Z" + evt: + name: "claude_artifact_viewed" + http: + useragent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36" + useragent_details: + browser: + family: "Chrome" + major: "148" + minor: "0" + patch: "0" + patch_minor: "0" + device: + brand: "Apple" + category: "Desktop" + family: "Mac" + model: "Mac" + os: + family: "Mac OS X" + major: "10" + minor: "15" + patch: "7" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 2 + activity_name: "Read" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36" + metadata: + event_code: "claude_artifact_viewed" + original_time: "2026-05-22T15:25:13.701734Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463513701 + web_resources: + - uid: "claude_artifact_01EXAMPLEARTIFACT0" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_artifact_viewed" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- { "actor" : { "email_address" : "user@example.com", - "user_id" : "user_01FBY4qyk7SdPxJCAd4EfPbT", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36" + }, + "claude_artifact_id" : "claude_artifact_01EXAMPLEARTIFACT0", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:25:13.701734Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_artifact_viewed" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463513701 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.3883.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:54.358426Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_chat_created", + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0 Claude/1.3883.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_chat_id: "claude_chat_01EXAMPLECHATID000000" + created_at: "2026-05-22T15:21:54.358426Z" + evt: + name: "claude_chat_created" + http: + useragent: "Mozilla/5.0 Claude/1.3883.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 1 + activity_name: "Create" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0 Claude/1.3883.0" + metadata: + event_code: "claude_chat_created" + original_time: "2026-05-22T15:21:54.358426Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463314358 + web_resources: + - uid: "claude_chat_01EXAMPLECHATID000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_chat_created" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", "ip_address" : "192.0.2.1", "type" : "user_actor", - "user_agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" + "user_agent" : "Mozilla/5.0 Claude/1.3883.0" }, - "organization_id" : "org_01GuSHHxdWNCcTtk6Wr5arBM", - "organization_uuid" : "80cb55fa-462c-4bc0-82d6-07ebb1a6f004", - "created_at" : "2026-05-05T16:04:57.150724Z", - "id" : "activity_01R1sBnxj7yvtdZnt8DsfpRL", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:54.358426Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_chat_created", + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463314358 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.5354.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:03.415347Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_chat_deleted", + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0 Claude/1.5354.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_chat_id: "claude_chat_01EXAMPLECHATID000000" + created_at: "2026-05-22T15:21:03.415347Z" + evt: + name: "claude_chat_deleted" + http: + useragent: "Mozilla/5.0 Claude/1.5354.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 4 + activity_name: "Delete" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0 Claude/1.5354.0" + metadata: + event_code: "claude_chat_deleted" + original_time: "2026-05-22T15:21:03.415347Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463263415 + web_resources: + - uid: "claude_chat_01EXAMPLECHATID000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_chat_deleted" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.5354.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:03.415347Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_chat_deleted", + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463263415 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:44.621308Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_chat_updated", + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_chat_id: "claude_chat_01EXAMPLECHATID000000" + claude_project_id: "claude_proj_01EXAMPLEPROJECT00000" + created_at: "2026-05-22T15:21:44.621308Z" + evt: + name: "claude_chat_updated" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 3 + activity_name: "Update" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_chat_updated" + original_time: "2026-05-22T15:21:44.621308Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463304621 + web_resources: + - uid: "claude_chat_01EXAMPLECHATID000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_chat_updated" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:44.621308Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_chat_updated", + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463304621 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:53.556370Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_chat_viewed", + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_chat_id: "claude_chat_01EXAMPLECHATID000000" + created_at: "2026-05-22T15:21:53.556370Z" + evt: + name: "claude_chat_viewed" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 2 + activity_name: "Read" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_chat_viewed" + original_time: "2026-05-22T15:21:53.556370Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463313556 + web_resources: + - uid: "claude_chat_01EXAMPLECHATID000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_chat_viewed" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:53.556370Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", "type" : "claude_chat_viewed", - "claude_chat_id" : "claude_chat_01AxWT9aH4swoDJ8u6dShxMV" + "claude_chat_id" : "claude_chat_01EXAMPLECHATID000000" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463313556 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "filename" : "example-image.png", + "claude_file_id" : "claude_file_01EXAMPLEFILEID000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:32.702968Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_file_uploaded" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_file_id: "claude_file_01EXAMPLEFILEID000000" + created_at: "2026-05-22T15:21:32.702968Z" + evt: + name: "claude_file_uploaded" + filename: "example-image.png" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 1 + activity_name: "Create" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_file_uploaded" + original_time: "2026-05-22T15:21:32.702968Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463292702 + web_resources: + - name: "example-image.png" + uid: "claude_file_01EXAMPLEFILEID000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_file_uploaded" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "filename" : "example-image.png", + "claude_file_id" : "claude_file_01EXAMPLEFILEID000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:32.702968Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_file_uploaded" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463292702 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.3883.0" + }, + "filename" : "example-screenshot.png", + "claude_file_id" : "claude_file_01EXAMPLEFILEID000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:50.616332Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_file_viewed" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0 Claude/1.3883.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_file_id: "claude_file_01EXAMPLEFILEID000000" + created_at: "2026-05-22T15:21:50.616332Z" + evt: + name: "claude_file_viewed" + filename: "example-screenshot.png" + http: + useragent: "Mozilla/5.0 Claude/1.3883.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 2 + activity_name: "Read" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0 Claude/1.3883.0" + metadata: + event_code: "claude_file_viewed" + original_time: "2026-05-22T15:21:50.616332Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463310616 + web_resources: + - name: "example-screenshot.png" + uid: "claude_file_01EXAMPLEFILEID000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_file_viewed" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.3883.0" + }, + "filename" : "example-screenshot.png", + "claude_file_id" : "claude_file_01EXAMPLEFILEID000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:50.616332Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_file_viewed" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463310616 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:10.594873Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_created" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_project_id: "claude_proj_01EXAMPLEPROJECT00000" + created_at: "2026-05-22T15:21:10.594873Z" + evt: + name: "claude_project_created" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 1 + activity_name: "Create" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_project_created" + original_time: "2026-05-22T15:21:10.594873Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463270594 + web_resources: + - uid: "claude_proj_01EXAMPLEPROJECT00000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_project_created" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:10.594873Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_created" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463270594 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "filename" : "example-document.pdf", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:23:05.845058Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_document_uploaded", + "claude_project_document_id" : "claude_proj_doc_01EXAMPLEDOC0000000000" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_project_document_id: "claude_proj_doc_01EXAMPLEDOC0000000000" + claude_project_id: "claude_proj_01EXAMPLEPROJECT00000" + created_at: "2026-05-22T15:23:05.845058Z" + evt: + name: "claude_project_document_uploaded" + filename: "example-document.pdf" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 1 + activity_name: "Create" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_project_document_uploaded" + original_time: "2026-05-22T15:23:05.845058Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463385845 + web_resources: + - name: "example-document.pdf" + uid: "claude_proj_doc_01EXAMPLEDOC0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_project_document_uploaded" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "filename" : "example-document.pdf", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:23:05.845058Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_document_uploaded", + "claude_project_document_id" : "claude_proj_doc_01EXAMPLEDOC0000000000" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463385845 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "filename" : "example-file.txt", + "claude_file_id" : "claude_file_01EXAMPLEFILEID000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:20:23.462825Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_file_uploaded" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_file_id: "claude_file_01EXAMPLEFILEID000000" + claude_project_id: "claude_proj_01EXAMPLEPROJECT00000" + created_at: "2026-05-22T15:20:23.462825Z" + evt: + name: "claude_project_file_uploaded" + filename: "example-file.txt" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 1 + activity_name: "Create" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_project_file_uploaded" + original_time: "2026-05-22T15:20:23.462825Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463223462 + web_resources: + - name: "example-file.txt" + uid: "claude_file_01EXAMPLEFILEID000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "claude_project_file_uploaded" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "filename" : "example-file.txt", + "claude_file_id" : "claude_file_01EXAMPLEFILEID000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:20:23.462825Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_file_uploaded" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463223462 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "preview_only" : false, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:48.506301Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_viewed" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + claude_project_id: "claude_proj_01EXAMPLEPROJECT00000" + created_at: "2026-05-22T15:21:48.506301Z" + evt: + name: "claude_project_viewed" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 2 + activity_name: "Read" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_project_viewed" + original_time: "2026-05-22T15:21:48.506301Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463308506 + web_resources: + - uid: "claude_proj_01EXAMPLEPROJECT00000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + preview_only: false + type: "claude_project_viewed" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "claude_project_id" : "claude_proj_01EXAMPLEPROJECT00000", + "preview_only" : false, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:48.506301Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_project_viewed" } - result: - custom: + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463308506 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.3883.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "skill_name" : "example-skill-name", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:44.267747Z", + "skill_id" : "skill_01EXAMPLESKILLID00000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_skill_created" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0 Claude/1.3883.0" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-05-22T15:21:44.267747Z" + evt: + name: "claude_skill_created" + http: + useragent: "Mozilla/5.0 Claude/1.3883.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 1 + activity_name: "Create" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0 Claude/1.3883.0" + metadata: + event_code: "claude_skill_created" + original_time: "2026-05-22T15:21:44.267747Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463304267 + web_resources: + - name: "example-skill-name" + uid: "skill_01EXAMPLESKILLID00000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + skill_id: "skill_01EXAMPLESKILLID00000000" + skill_name: "example-skill-name" + type: "claude_skill_created" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.3883.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "skill_name" : "example-skill-name", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:44.267747Z", + "skill_id" : "skill_01EXAMPLESKILLID00000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_skill_created" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463304267 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.7196.1" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "skill_name" : "example-skill-name", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:17.287525Z", + "skill_id" : "skill_01EXAMPLESKILLID00000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_skill_replaced" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0 Claude/1.7196.1" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-05-22T15:21:17.287525Z" + evt: + name: "claude_skill_replaced" + http: + useragent: "Mozilla/5.0 Claude/1.7196.1" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 3 + activity_name: "Update" + category_name: "Application Activity" + category_uid: 6 + class_name: "Web Resources Activity" + class_uid: 6001 + http_request: + user_agent: "Mozilla/5.0 Claude/1.7196.1" + metadata: + event_code: "claude_skill_replaced" + original_time: "2026-05-22T15:21:17.287525Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + owner: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + status: "Success" + status_id: 1 + time: 1779463277287 + web_resources: + - name: "example-skill-name" + uid: "skill_01EXAMPLESKILLID00000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + skill_id: "skill_01EXAMPLESKILLID00000000" + skill_name: "example-skill-name" + type: "claude_skill_replaced" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.7196.1" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "skill_name" : "example-skill-name", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:17.287525Z", + "skill_id" : "skill_01EXAMPLESKILLID00000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_skill_replaced" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463277287 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "created_at" : "2026-05-22T15:21:50.371418Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_user_settings_updated", + "updates" : [ { + "previous_value" : { + "example_tool" : false + }, + "type" : "mcp_tools_enabled", + "current_value" : { + "example_tool" : true + } + } ] + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-05-22T15:21:50.371418Z" + evt: + name: "claude_user_settings_updated" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 99 + activity_name: "claude_user_settings_updated" actor: - type: "user_actor" - claude_chat_id: "claude_chat_01AxWT9aH4swoDJ8u6dShxMV" - created_at: "2026-05-05T16:04:57.150724Z" - evt: - name: "claude_chat_viewed" - http: - useragent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" - useragent_details: - browser: - family: "Safari" - major: "17" - minor: "0" - device: - brand: "Apple" - category: "Desktop" - family: "Mac" - model: "Mac" - os: - family: "Mac OS X" - major: "10" - minor: "15" - patch: "7" - id: "activity_01R1sBnxj7yvtdZnt8DsfpRL" - network: - client: - geoip: {} - ip: "192.0.2.1" - organization_id: "org_01GuSHHxdWNCcTtk6Wr5arBM" - organization_uuid: "80cb55fa-462c-4bc0-82d6-07ebb1a6f004" - usr: - email: "user@example.com" - id: "user_01FBY4qyk7SdPxJCAd4EfPbT" - message: |- - { - "actor" : { - "email_address" : "user@example.com", - "user_id" : "user_01FBY4qyk7SdPxJCAd4EfPbT", - "ip_address" : "192.0.2.1", - "type" : "user_actor", - "user_agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" + user: + email_addr: "user@example.com" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Account Change" + class_uid: 3001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "claude_user_settings_updated" + original_time: "2026-05-22T15:21:50.371418Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Success" + status_id: 1 + time: 1779463310371 + user: + email_addr: "user@example.com" + uid: "user_01EXAMPLEUSERID0000000000" + type: "claude_user_settings_updated" + updates: + - current_value: + example_tool: true + previous_value: + example_tool: false + type: "mcp_tools_enabled" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "created_at" : "2026-05-22T15:21:50.371418Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "claude_user_settings_updated", + "updates" : [ { + "previous_value" : { + "example_tool" : false }, - "organization_id" : "org_01GuSHHxdWNCcTtk6Wr5arBM", - "organization_uuid" : "80cb55fa-462c-4bc0-82d6-07ebb1a6f004", - "created_at" : "2026-05-05T16:04:57.150724Z", - "id" : "activity_01R1sBnxj7yvtdZnt8DsfpRL", - "type" : "claude_chat_viewed", - "claude_chat_id" : "claude_chat_01AxWT9aH4swoDJ8u6dShxMV" - } - tags: - - "source:LOGS_SOURCE" - timestamp: 1777997097150 + "type" : "mcp_tools_enabled", + "current_value" : { + "example_tool" : true + } + } ] + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463310371 + - + sample: |- + { + "actor" : { + "ip_address" : "192.0.2.1", + "type" : "api_actor", + "api_key_id" : "apikey_01EXAMPLEAPIKEY000000000", + "user_agent" : "example-client/1.0" + }, + "status_code" : 200, + "created_at" : "2026-05-22T15:21:38.920308Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "request_method" : "GET", + "type" : "compliance_api_accessed", + "request_id" : "req_011CbEXAMPLEREQUEST00000", + "url" : "https://api.anthropic.com/v1/compliance/activities?" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + api_key_id: "apikey_01EXAMPLEAPIKEY000000000" + ip_address: "192.0.2.1" + type: "api_actor" + user_agent: "example-client/1.0" + created_at: "2026-05-22T15:21:38.920308Z" + evt: + name: "compliance_api_accessed" + http: + useragent: "example-client/1.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 2 + activity_name: "Read" + actor: + app_uid: "apikey_01EXAMPLEAPIKEY000000000" + user: + type: "api_actor" + type_id: 99 + uid: "apikey_01EXAMPLEAPIKEY000000000" + api: + operation: "GET" + request: + uid: "req_011CbEXAMPLEREQUEST00000" + response: + code: 200 + category_name: "Application Activity" + category_uid: 6 + class_name: "API Activity" + class_uid: 6003 + http_request: + http_method: "GET" + uid: "req_011CbEXAMPLEREQUEST00000" + url: + url_string: "https://api.anthropic.com/v1/compliance/activities?" + user_agent: "example-client/1.0" + http_response: + code: 200 + metadata: + event_code: "compliance_api_accessed" + original_time: "2026-05-22T15:21:38.920308Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Success" + status_id: 1 + time: 1779463298920 + request_id: "req_011CbEXAMPLEREQUEST00000" + request_method: "GET" + status_code: 200 + type: "compliance_api_accessed" + url: "https://api.anthropic.com/v1/compliance/activities?" + message: |- + { + "actor" : { + "ip_address" : "192.0.2.1", + "type" : "api_actor", + "api_key_id" : "apikey_01EXAMPLEAPIKEY000000000", + "user_agent" : "example-client/1.0" + }, + "status_code" : 200, + "created_at" : "2026-05-22T15:21:38.920308Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "request_method" : "GET", + "type" : "compliance_api_accessed", + "request_id" : "req_011CbEXAMPLEREQUEST00000", + "url" : "https://api.anthropic.com/v1/compliance/activities?" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463298920 + - + sample: |- + { + "actor" : { + "admin_api_key_id" : "admin_api_key_01EXAMPLEADMIN0000", + "ip_address" : "192.0.2.1", + "type" : "admin_api_key_actor", + "user_agent" : "python-requests/2.32.5" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:07:03.419782Z", + "deleted_user_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "org_user_deleted" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + admin_api_key_id: "admin_api_key_01EXAMPLEADMIN0000" + ip_address: "192.0.2.1" + type: "admin_api_key_actor" + user_agent: "python-requests/2.32.5" + created_at: "2026-05-22T15:07:03.419782Z" + deleted_user_id: "user_01EXAMPLEUSERID0000000000" + evt: + name: "org_user_deleted" + http: + useragent: "python-requests/2.32.5" + useragent_details: + browser: + family: "Python Requests" + major: "2" + minor: "32" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 6 + activity_name: "Delete" + actor: + user: + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "Admin" + type_id: 2 + uid: "admin_api_key_01EXAMPLEADMIN0000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Account Change" + class_uid: 3001 + http_request: + user_agent: "python-requests/2.32.5" + metadata: + event_code: "org_user_deleted" + original_time: "2026-05-22T15:07:03.419782Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Success" + status_id: 1 + time: 1779462423419 + user: + uid: "user_01EXAMPLEUSERID0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "org_user_deleted" + message: |- + { + "actor" : { + "admin_api_key_id" : "admin_api_key_01EXAMPLEADMIN0000", + "ip_address" : "192.0.2.1", + "type" : "admin_api_key_actor", + "user_agent" : "python-requests/2.32.5" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:07:03.419782Z", + "deleted_user_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "org_user_deleted" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779462423419 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "deleted_user_email" : "user@example.com", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-04-22T12:38:58.658822Z", + "deleted_user_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "org_user_deleted" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-04-22T12:38:58.658822Z" + deleted_user_email: "user@example.com" + deleted_user_id: "user_01EXAMPLEUSERID0000000000" + evt: + name: "org_user_deleted" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 6 + activity_name: "Delete" + actor: + user: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Account Change" + class_uid: 3001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "org_user_deleted" + original_time: "2026-04-22T12:38:58.658822Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Success" + status_id: 1 + time: 1776861538658 + user: + email_addr: "user@example.com" + name: "user@example.com" + uid: "user_01EXAMPLEUSERID0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "org_user_deleted" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "deleted_user_email" : "user@example.com", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-04-22T12:38:58.658822Z", + "deleted_user_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "org_user_deleted" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1776861538658 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "invite_id" : "invite_01EXAMPLEINVITE0000000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-15T15:22:37.490878Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "org_user_invite_accepted" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-05-15T15:22:37.490878Z" + evt: + name: "org_user_invite_accepted" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + invite_id: "invite_01EXAMPLEINVITE0000000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 1 + activity_name: "Create" + actor: + user: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Account Change" + class_uid: 3001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "org_user_invite_accepted" + original_time: "2026-05-15T15:22:37.490878Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Success" + status_id: 1 + time: 1778858557490 + user: + email_addr: "user@example.com" + uid: "user_01EXAMPLEUSERID0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "org_user_invite_accepted" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "invite_id" : "invite_01EXAMPLEINVITE0000000000", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-15T15:22:37.490878Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "org_user_invite_accepted" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1778858557490 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "invited_role" : "owner", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-15T15:22:11.738584Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "invited_email" : "invitee@example.com", + "type" : "org_user_invite_sent" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-05-15T15:22:11.738584Z" + evt: + name: "org_user_invite_sent" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + invited_email: "invitee@example.com" + invited_role: "owner" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 1 + activity_name: "Create" + actor: + user: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Account Change" + class_uid: 3001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "org_user_invite_sent" + original_time: "2026-05-15T15:22:11.738584Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + status: "Success" + status_id: 1 + time: 1778858531738 + user: + email_addr: "invitee@example.com" + name: "invitee@example.com" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "org_user_invite_sent" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "invited_role" : "owner", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-15T15:22:11.738584Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "invited_email" : "invitee@example.com", + "type" : "org_user_invite_sent" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1778858531738 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:23.115384Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "platform_api_key_created", + "api_key_id" : "apikey_01EXAMPLEAPIKEY000000000" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + api_key_id: "apikey_01EXAMPLEAPIKEY000000000" + created_at: "2026-05-22T15:21:23.115384Z" + evt: + name: "platform_api_key_created" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 1 + activity_name: "Create" + actor: + user: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Account Change" + class_uid: 3001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "platform_api_key_created" + original_time: "2026-05-22T15:21:23.115384Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + status: "Success" + status_id: 1 + time: 1779463283115 + user: + email_addr: "user@example.com" + uid: "user_01EXAMPLEUSERID0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "platform_api_key_created" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:21:23.115384Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "platform_api_key_created", + "api_key_id" : "apikey_01EXAMPLEAPIKEY000000000" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463283115 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:23:11.707169Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "platform_api_key_updated", + "updates" : [ { + "previous_value" : "active", + "type" : "status", + "current_value" : "archived" + } ], + "api_key_id" : "apikey_01EXAMPLEAPIKEY000000000" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + api_key_id: "apikey_01EXAMPLEAPIKEY000000000" + created_at: "2026-05-22T15:23:11.707169Z" + evt: + name: "platform_api_key_updated" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 5 + activity_name: "Disable" + actor: + user: + email_addr: "user@example.com" + org: + uid: "org_01EXAMPLEORGID00000000000" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Account Change" + class_uid: 3001 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "platform_api_key_updated" + original_time: "2026-05-22T15:23:11.707169Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + status: "Success" + status_id: 1 + time: 1779463391707 + user: + email_addr: "user@example.com" + uid: "user_01EXAMPLEUSERID0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + type: "platform_api_key_updated" + updates: + - current_value: "archived" + previous_value: "active" + type: "status" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "created_at" : "2026-05-22T15:23:11.707169Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "platform_api_key_updated", + "updates" : [ { + "previous_value" : "active", + "type" : "status", + "current_value" : "archived" + } ], + "api_key_id" : "apikey_01EXAMPLEAPIKEY000000000" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463391707 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "role" : "chat_project:owner", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "resource_type" : "chat_project", + "target_type" : "organization_member", + "created_at" : "2026-05-22T15:21:10.596119Z", + "resource_id" : "claude_proj_01EXAMPLEPROJECT00000", + "target_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "role_assignment_granted" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "2001:db8::1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-05-22T15:21:10.596119Z" + evt: + name: "role_assignment_granted" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "2001:db8::1" + ocsf: + activity_id: 1 + activity_name: "Assign Privileges" + actor: + user: + email_addr: "user@example.com" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "User Access Management" + class_uid: 3005 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "role_assignment_granted" + original_time: "2026-05-22T15:21:10.596119Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + privileges: + - "chat_project:owner" + resources: + - type: "chat_project" + uid: "claude_proj_01EXAMPLEPROJECT00000" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "2001:db8::1" + status: "Success" + status_id: 1 + time: 1779463270596 + user: + org: + uid: "org_01EXAMPLEORGID00000000000" + uid: "user_01EXAMPLEUSERID0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + resource_id: "claude_proj_01EXAMPLEPROJECT00000" + resource_type: "chat_project" + role: "chat_project:owner" + target_id: "user_01EXAMPLEUSERID0000000000" + target_type: "organization_member" + type: "role_assignment_granted" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "2001:db8::1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "role" : "chat_project:owner", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "resource_type" : "chat_project", + "target_type" : "organization_member", + "created_at" : "2026-05-22T15:21:10.596119Z", + "resource_id" : "claude_proj_01EXAMPLEPROJECT00000", + "target_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "role_assignment_granted" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463270596 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.6259.0" + }, + "role" : "skill:viewer", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "resource_type" : "skill", + "target_type" : "organization_member", + "created_at" : "2026-05-21T20:47:11.194821Z", + "resource_id" : "skill_01EXAMPLESKILLID00000000", + "target_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "role_assignment_revoked" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0 Claude/1.6259.0" + user_id: "user_01EXAMPLEUSERID0000000000" + created_at: "2026-05-21T20:47:11.194821Z" + evt: + name: "role_assignment_revoked" + http: + useragent: "Mozilla/5.0 Claude/1.6259.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 2 + activity_name: "Revoke Privileges" + actor: + user: + email_addr: "user@example.com" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "User Access Management" + class_uid: 3005 + http_request: + user_agent: "Mozilla/5.0 Claude/1.6259.0" + metadata: + event_code: "role_assignment_revoked" + original_time: "2026-05-21T20:47:11.194821Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + privileges: + - "skill:viewer" + resources: + - type: "skill" + uid: "skill_01EXAMPLESKILLID00000000" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Success" + status_id: 1 + time: 1779396431194 + user: + org: + uid: "org_01EXAMPLEORGID00000000000" + uid: "user_01EXAMPLEUSERID0000000000" + organization_id: "org_01EXAMPLEORGID00000000000" + organization_uuid: "00000000-0000-0000-0000-000000000000" + resource_id: "skill_01EXAMPLESKILLID00000000" + resource_type: "skill" + role: "skill:viewer" + target_id: "user_01EXAMPLEUSERID0000000000" + target_type: "organization_member" + type: "role_assignment_revoked" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0 Claude/1.6259.0" + }, + "role" : "skill:viewer", + "organization_id" : "org_01EXAMPLEORGID00000000000", + "organization_uuid" : "00000000-0000-0000-0000-000000000000", + "resource_type" : "skill", + "target_type" : "organization_member", + "created_at" : "2026-05-21T20:47:11.194821Z", + "resource_id" : "skill_01EXAMPLESKILLID00000000", + "target_id" : "user_01EXAMPLEUSERID0000000000", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "role_assignment_revoked" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779396431194 + - + sample: |- + { + "actor" : { + "unauthenticated_email_address" : "user@example.com", + "ip_address" : "192.0.2.1", + "type" : "unauthenticated_user_actor", + "user_agent" : "Mozilla/5.0" + }, + "created_at" : "2026-05-22T15:19:09.946100Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "sso_login_initiated" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + ip_address: "192.0.2.1" + type: "unauthenticated_user_actor" + unauthenticated_email_address: "user@example.com" + user_agent: "Mozilla/5.0" + created_at: "2026-05-22T15:19:09.946100Z" + evt: + name: "sso_login_initiated" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 1 + activity_name: "Logon" + actor: + user: + email_addr: "user@example.com" + name: "user@example.com" + type: "unauthenticated_user_actor" + type_id: 99 + auth_protocol: "SAML" + auth_protocol_id: 5 + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Authentication" + class_uid: 3002 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "sso_login_initiated" + original_time: "2026-05-22T15:19:09.946100Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + service: + name: "Claude" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Unknown" + status_id: 0 + time: 1779463149946 + user: + email_addr: "user@example.com" + name: "user@example.com" + type: "sso_login_initiated" + message: |- + { + "actor" : { + "unauthenticated_email_address" : "user@example.com", + "ip_address" : "192.0.2.1", + "type" : "unauthenticated_user_actor", + "user_agent" : "Mozilla/5.0" + }, + "created_at" : "2026-05-22T15:19:09.946100Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "sso_login_initiated" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463149946 + - + sample: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "auth_method" : "sso", + "created_at" : "2026-05-22T15:19:14.445010Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "sso_login_succeeded" + } + tags: + - "source:LOGS_SOURCE" + result: + custom: + actor: + email_address: "user@example.com" + ip_address: "192.0.2.1" + type: "user_actor" + user_agent: "Mozilla/5.0" + user_id: "user_01EXAMPLEUSERID0000000000" + auth_method: "sso" + created_at: "2026-05-22T15:19:14.445010Z" + evt: + name: "sso_login_succeeded" + http: + useragent: "Mozilla/5.0" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + id: "activity_01EXAMPLEACTIVITY0000000" + network: + client: + geoip: {} + ip: "192.0.2.1" + ocsf: + activity_id: 1 + activity_name: "Logon" + actor: + user: + email_addr: "user@example.com" + name: "user@example.com" + type: "User" + type_id: 1 + uid: "user_01EXAMPLEUSERID0000000000" + auth_protocol: "SAML" + auth_protocol_id: 5 + category_name: "Identity & Access Management" + category_uid: 3 + class_name: "Authentication" + class_uid: 3002 + http_request: + user_agent: "Mozilla/5.0" + metadata: + event_code: "sso_login_succeeded" + original_time: "2026-05-22T15:19:14.445010Z" + product: + name: "Claude" + vendor_name: "Anthropic" + uid: "activity_01EXAMPLEACTIVITY0000000" + version: "1.5.0" + service: + name: "Claude" + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "192.0.2.1" + status: "Success" + status_id: 1 + time: 1779463154445 + user: + email_addr: "user@example.com" + name: "user@example.com" + uid: "user_01EXAMPLEUSERID0000000000" + type: "sso_login_succeeded" + usr: + email: "user@example.com" + id: "user_01EXAMPLEUSERID0000000000" + message: |- + { + "actor" : { + "email_address" : "user@example.com", + "user_id" : "user_01EXAMPLEUSERID0000000000", + "ip_address" : "192.0.2.1", + "type" : "user_actor", + "user_agent" : "Mozilla/5.0" + }, + "auth_method" : "sso", + "created_at" : "2026-05-22T15:19:14.445010Z", + "id" : "activity_01EXAMPLEACTIVITY0000000", + "type" : "sso_login_succeeded" + } + tags: + - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" + timestamp: 1779463154445 diff --git a/anthropic_usage_and_costs/README.md b/anthropic_usage_and_costs/README.md index 0b8a57878c0e7..240b76fd9c356 100644 --- a/anthropic_usage_and_costs/README.md +++ b/anthropic_usage_and_costs/README.md @@ -2,37 +2,54 @@ ## Overview -Datadog's Anthropic Usage and Costs integration allows you to get visibility into your Anthropic usage and associated costs. By ingesting data from Anthropic's newly released Admin usage and cost API, this integration enables your teams to: +Datadog's Anthropic Usage and Costs integration allows you to get visibility into your Anthropic usage and associated costs. By ingesting data from Anthropic's Admin and Analytics usage and cost APIs, this integration enables your teams to: - **Monitor LLM token consumption** (input, output, cache usage) in near real-time - **Track costs** by model, workspace, and service tier, supporting accurate attribution and budgeting +- **Attribute usage and spend to individual users** (Enterprise plans): break down token consumption and dollar cost by user, product (API, Claude Code, Claude.ai), context window, inference geography, and speed - **Understand usage trends** across teams, API keys, or users to optimize model usage - **Set up alerting and dashboards** that highlight anomalies in usage or unexpected cost spikes This integration is especially valuable for teams using Anthropic at scale who want to manage spend, understand product adoption, and ensure efficient use of AI resources-all within Datadog. With this data you will be able to introduce and validate optimization strategies to get the best out of Anthropic. -You can also see your Anthropic costs in Datadog [Cloud Cost Management][6], allowing you to answer key questions: Which models or workspaces are generating the most cost? Are workloads using the right service tier (Standard, Batch, or Priority)? Are teams effectively using caching or ephemeral sessions? What's the cost breakdown between Claude Opus and Claude Sonnet? +You can also see your Anthropic costs in Datadog [Cloud Cost Management][6], allowing you to answer key questions: Which models or workspaces are generating the most cost? Which users or teams are driving the most spend? Are workloads using the right service tier (Standard, Batch, or Priority)? Are teams effectively using caching or ephemeral sessions? What's the cost breakdown between Claude Opus and Claude Sonnet? **Minimum Agent version:** 7.69.0 ## Setup -To get started with the Anthropic Admin API integration in Datadog, follow the steps below: +To get started with the Anthropic integration in Datadog, follow the steps below: -### 1. Generate an Admin API Key +### 1. Identify your Anthropic plan and the key type you'll use -You will need an [Admin API key][5] from Anthropic. This key allows access to usage and cost reports across your organization. +Your Anthropic organization is on either a **Platform** plan or an **Enterprise** plan, and the plan determines which API key type you use to authenticate this integration: -1. Navigate to your organization's settings or reach out to your Anthropic account admin to create a new Admin API key. -2. Copy the API key to a secure location. +| Anthropic plan | API key type | Data ingested | +| --- | --- | --- | +| **Platform** | **Admin API key** | Organization-wide usage and cost, broken down by model, workspace, API key, and service tier. | +| **Enterprise** | **Analytics API key** | Per-user usage and cost, broken down by user, product (API, Claude Code, Claude.ai), model, context window, inference geography, and speed. Usage metrics are emitted in **1-hour buckets** after the hour closes on Anthropic's side, with a typical end-to-end latency of **1-3 hours**. Per-user cost data covers usage from **January 1, 2026** onward. | -### 2. Configure the Datadog Integration +Each Anthropic organization issues only one of these key types based on its plan, so most customers will configure exactly one key. If your company operates more than one Anthropic organization (for example, a Platform org and an Enterprise org), configure each one as a separate Datadog account. + +All metrics and Cloud Cost Management line items emitted from an Enterprise (Analytics) account carry the `org_type:enterprise` tag. Platform (Admin) data is emitted without an `org_type` tag, so you can separate the two data sources in dashboards, monitors, and Cloud Cost Management filters using the presence (or absence) of `org_type:enterprise`. + +### 2. Generate your Anthropic API key + +Follow the path that matches your Anthropic plan: + +- **Platform plan**: Generate an [Admin API key][5] from your Anthropic organization's settings, or ask your Anthropic account admin to create one for you. +- **Enterprise plan**: Generate an Analytics API key (with the `read:analytics` scope) at [claude.ai/analytics/api-keys][7]. Analytics API keys must be provisioned by your organization's **Primary Owner**. + +Copy the API key to a secure location after you generate it. + +### 3. Configure the Datadog integration 1. In Datadog, go to [**Integrations > Anthropic Usage and Costs**](https://app.datadoghq.com/integrations?integrationId=anthropic-usage-and-costs). -2. In the configuration panel, provide the **Admin API Key** by pasting the key you generated from Anthropic. -3. Click **Save Configuration**. +2. In the configuration panel, paste your **Admin API Key** or **Analytics API Key** into the API key field. Datadog automatically detects the key type and ingests the appropriate usage and cost data. +3. (Optional) Enable **Cost data ingestion** to send cost data to [Cloud Cost Management][6]. This requires Cloud Cost Management to be enabled on your Datadog account. +4. Click **Save Configuration**. -Once saved, Datadog will begin polling Anthropic usage and cost endpoints using this key and populate metrics in your environment. +After you save the configuration, Datadog begins polling the appropriate Anthropic usage and cost endpoints and populates metrics in your environment. Usage metrics typically appear within 10 minutes, and cost data appears in Cloud Cost Management within 24 hours. ## Data Collected @@ -58,3 +75,4 @@ Need help? Contact [Datadog support][3]. [4]: https://github.com/DataDog/integrations-core/blob/master/anthropic_usage_and_costs/metadata.csv [5]: https://docs.anthropic.com/en/api/administration-api [6]: /cost +[7]: https://claude.ai/analytics/api-keys diff --git a/apache/assets/logs/apache.yaml b/apache/assets/logs/apache.yaml index b572eb435b2b9..9c77406452236 100644 --- a/apache/assets/logs/apache.yaml +++ b/apache/assets/logs/apache.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: apache metric_id: apache backend_only: false diff --git a/apache/assets/logs/apache_tests.yaml b/apache/assets/logs/apache_tests.yaml index c51d2753cac14..7c3e9ed25ae39 100644 --- a/apache/assets/logs/apache_tests.yaml +++ b/apache/assets/logs/apache_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "apache" tests: - diff --git a/arctic_wolf_aurora_endpoint_security/assets/logs/arctic-wolf-aurora-endpoint-security_tests.yaml b/arctic_wolf_aurora_endpoint_security/assets/logs/arctic-wolf-aurora-endpoint-security_tests.yaml index 058935fd711c8..bbc06821640a4 100644 --- a/arctic_wolf_aurora_endpoint_security/assets/logs/arctic-wolf-aurora-endpoint-security_tests.yaml +++ b/arctic_wolf_aurora_endpoint_security/assets/logs/arctic-wolf-aurora-endpoint-security_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "arctic-wolf-aurora-endpoint-security" tests: - diff --git a/argocd/assets/logs/argocd_tests.yaml b/argocd/assets/logs/argocd_tests.yaml index ce2e5cebc9798..fb0ca3bc756bd 100644 --- a/argocd/assets/logs/argocd_tests.yaml +++ b/argocd/assets/logs/argocd_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "argocd" tests: diff --git a/azure_active_directory/assets/logs/azure.activedirectory.yaml b/azure_active_directory/assets/logs/azure.activedirectory.yaml index e8050796da873..c39db0e784b51 100644 --- a/azure_active_directory/assets/logs/azure.activedirectory.yaml +++ b/azure_active_directory/assets/logs/azure.activedirectory.yaml @@ -83,6 +83,11 @@ facets: name: Client IP path: network.client.ip source: log + - groups: + - Web Access + name: Client Port + path: network.client.port + source: log - groups: - User name: User Email @@ -200,6 +205,13 @@ facets: name: Source IP Address path: ocsf.src_endpoint.ip source: log + - facetType: range + groups: + - OCSF + name: Src Endpoint Port + path: ocsf.src_endpoint.port + source: log + type: integer - groups: - OCSF name: Event Code @@ -281,6 +293,29 @@ pipeline: overrideOnConflict: false sourceType: attribute targetType: attribute + - type: grok-parser + name: Parse `network.client.ip` to `network.client.ip`, `network.client.port` + enabled: true + source: network.client.ip + grok: + supportRules: | + matchRules: | + ipv4_rule %{ipv4:network.client.ip}(:%{port:network.client.port})? + ipv6_rule \[?%{ipv6:network.client.ip}\]?(:%{port:network.client.port})? + samples: + - 15.113.255.209 + - 15.113.255.209:21341 + - type: attribute-remapper + name: Map `network.client.port` to `network.client.port` + enabled: true + sources: + - network.client.port + sourceType: attribute + target: network.client.port + targetType: attribute + targetFormat: integer + preserveSource: false + overrideOnConflict: false - type: arithmetic-processor name: Compute duration in nanoseconds from durationMs in miliseconds enabled: true @@ -548,6 +583,18 @@ pipeline: targetType: attribute preserveSource: true overrideOnConflict: false + - type: grok-parser + name: Parse `ocsf.src_endpoint.ip` to `ocsf.src_endpoint.ip`, `ocsf.src_endpoint.port` + enabled: true + source: ocsf.src_endpoint.ip + grok: + supportRules: | + matchRules: | + ipv4_rule %{ipv4:ocsf.src_endpoint.ip}(:%{port:ocsf.src_endpoint.port})? + ipv6_rule \[?%{ipv6:ocsf.src_endpoint.ip}\]?(:%{port:ocsf.src_endpoint.port})? + samples: + - 15.113.255.209 + - 15.113.255.209:21341 - type: attribute-remapper name: Map `properties.resultReason` to `ocsf.status_code` enabled: true @@ -1019,6 +1066,18 @@ pipeline: targetType: attribute preserveSource: true overrideOnConflict: false + - type: grok-parser + name: Parse `ocsf.src_endpoint.ip` to `ocsf.src_endpoint.ip`, `ocsf.src_endpoint.port` + enabled: true + source: ocsf.src_endpoint.ip + grok: + supportRules: | + matchRules: | + ipv4_rule %{ipv4:ocsf.src_endpoint.ip}(:%{port:ocsf.src_endpoint.port})? + ipv6_rule \[?%{ipv6:ocsf.src_endpoint.ip}\]?(:%{port:ocsf.src_endpoint.port})? + samples: + - 15.113.255.209 + - 15.113.255.209:21341 - type: attribute-remapper name: Map `properties.deviceDetail.operatingSystem` to `ocsf.src_endpoint.os.name` enabled: true @@ -1877,6 +1936,18 @@ pipeline: targetType: attribute preserveSource: true overrideOnConflict: false + - type: grok-parser + name: Parse `ocsf.src_endpoint.ip` to `ocsf.src_endpoint.ip`, `ocsf.src_endpoint.port` + enabled: true + source: ocsf.src_endpoint.ip + grok: + supportRules: | + matchRules: | + ipv4_rule %{ipv4:ocsf.src_endpoint.ip}(:%{port:ocsf.src_endpoint.port})? + ipv6_rule \[?%{ipv6:ocsf.src_endpoint.ip}\]?(:%{port:ocsf.src_endpoint.port})? + samples: + - 15.113.255.209 + - 15.113.255.209:21341 - type: string-builder-processor name: Add dst_endpoint.hostname enabled: true @@ -2194,6 +2265,16 @@ pipeline: targetType: attribute preserveSource: false overrideOnConflict: false + - type: attribute-remapper + name: Map `ocsf.src_endpoint.port` to `network.client.port` + enabled: true + sources: + - ocsf.src_endpoint.port + sourceType: attribute + target: callerIpAddress + targetType: attribute + preserveSource: false + overrideOnConflict: false - type: pipeline name: OCSF post transformations enabled: true @@ -2296,3 +2377,14 @@ pipeline: targetFormat: integer preserveSource: false overrideOnConflict: false + - type: attribute-remapper + name: Map `ocsf.src_endpoint.port` to `ocsf.src_endpoint.port` + enabled: true + sources: + - ocsf.src_endpoint.port + sourceType: attribute + target: ocsf.src_endpoint.port + targetType: attribute + targetFormat: integer + preserveSource: false + overrideOnConflict: false diff --git a/azure_active_directory/assets/logs/azure.activedirectory_tests.yaml b/azure_active_directory/assets/logs/azure.activedirectory_tests.yaml index 64cebba79dbcd..5d8e13a038638 100644 --- a/azure_active_directory/assets/logs/azure.activedirectory_tests.yaml +++ b/azure_active_directory/assets/logs/azure.activedirectory_tests.yaml @@ -328,7 +328,7 @@ tests: "tenantId": "4d3bac44-0230-4732-9e70-cc00736f0a97", "resultSignature": "None", "durationMs": 0, - "callerIpAddress": "192.182.149.21", + "callerIpAddress": "192.182.149.21:43210", "correlationId": "a13bd0fa-70d0-4e60-ae23-b687377b4695", "Level": 4, "properties": { @@ -353,7 +353,7 @@ tests: "id": "018af091-5465-4aed-9d6f-8c40981b2375", "displayName": null, "userPrincipalName": "test.test@datadoghq.com", - "ipAddress": "192.182.149.21", + "ipAddress": "192.182.149.21:43210", "roles": [] } }, @@ -384,7 +384,7 @@ tests: result: custom: Level: 4 - callerIpAddress: "192.182.149.21" + callerIpAddress: "192.182.149.21:43210" category: "AuditLogs" correlationId: "a13bd0fa-70d0-4e60-ae23-b687377b4695" duration: 0.0 @@ -397,6 +397,7 @@ tests: client: geoip: {} ip: "192.182.149.21" + port: 43210 ocsf: activity_id: 6 activity_name: "Delete" @@ -427,6 +428,7 @@ tests: severity_id: 1 src_endpoint: ip: "192.182.149.21" + port: 43210 status: "Success" status_code: "" status_id: 1 @@ -453,7 +455,7 @@ tests: initiatedBy: user: id: "018af091-5465-4aed-9d6f-8c40981b2375" - ipAddress: "192.182.149.21" + ipAddress: "192.182.149.21:43210" userPrincipalName: "test.test@datadoghq.com" loggedByService: "Core Directory" operationName: "Delete user" @@ -481,7 +483,7 @@ tests: name: "test.test@datadoghq.com" message: |- { - "callerIpAddress" : "192.182.149.21", + "callerIpAddress" : "192.182.149.21:43210", "resourceId" : "/tenants/4d3bac44-0230-4732-9e70-cc00736f0a97/providers/Microsoft.aadiam", "operationVersion" : "1.0", "tenantId" : "4d3bac44-0230-4732-9e70-cc00736f0a97", @@ -522,7 +524,7 @@ tests: "resultType" : "", "initiatedBy" : { "user" : { - "ipAddress" : "192.182.149.21", + "ipAddress" : "192.182.149.21:43210", "id" : "018af091-5465-4aed-9d6f-8c40981b2375", "userPrincipalName" : "test.test@datadoghq.com" } diff --git a/barracuda_secure_edge/assets/logs/barracuda_secure_edge_tests.yaml b/barracuda_secure_edge/assets/logs/barracuda_secure_edge_tests.yaml index 42c70d4625c66..10b793e9900fd 100644 --- a/barracuda_secure_edge/assets/logs/barracuda_secure_edge_tests.yaml +++ b/barracuda_secure_edge/assets/logs/barracuda_secure_edge_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: barracuda_secure_edge tests: - diff --git a/checkpoint_quantum_firewall/assets/logs/checkpoint-quantum-firewall_tests.yaml b/checkpoint_quantum_firewall/assets/logs/checkpoint-quantum-firewall_tests.yaml index 45c057af1c9b5..1620ca125123a 100644 --- a/checkpoint_quantum_firewall/assets/logs/checkpoint-quantum-firewall_tests.yaml +++ b/checkpoint_quantum_firewall/assets/logs/checkpoint-quantum-firewall_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "checkpoint-quantum-firewall" tests: diff --git a/clickhouse/AGENTS.md b/clickhouse/AGENTS.md new file mode 100644 index 0000000000000..334a4b5138124 --- /dev/null +++ b/clickhouse/AGENTS.md @@ -0,0 +1,42 @@ +# ClickHouse integration agent notes + +A small orientation guide. Read this before touching `advanced_queries/` or +`scripts/generate_metrics.py`. + +## Bulk match queries live in JSON, not Python + +Three of the four advanced queries (`SystemEvents`, `SystemMetrics`, +`SystemAsynchronousMetrics`) are *bulk match queries*: one SQL that returns +`(value, metric_name)` rows and dispatches to per-name metric definitions +through a large lookup table (over 1,000 entries for `SystemEvents`). Those +lookup tables ship as compact JSON files under +`datadog_checks/clickhouse/data/system_*.json` and are reassembled into the +`QueryManager` shape at load time. + +Before changing anything in this area, read: + +- `datadog_checks/clickhouse/advanced_queries/__init__.py`: the loader + (`load_match_query`, `_expand_match_items`, `warm_cache`, `__getattr__`) and + the JSON-schema docstring at the top of the file. +- `scripts/generate_metrics.py`: the script that parses ClickHouse's C++ + source files and writes the three JSON files. + +The fourth query, `SystemErrors`, is a plain Python literal in the same +`__init__.py`. Its shape (one metric plus tag columns, no per-row lookup) +doesn't fit the bulk-match pattern, so the JSON compression has nothing to +compress for it. Don't move it into JSON; the dual format was deliberately +removed during the JSON migration. + +## Don't hand-edit the JSON files + +The three `data/system_*.json` files are autogenerated. JSON has no comment +syntax, so there's no "do not edit" header inside the files themselves; the +warning lives here. Any hand-edit is overwritten on the next run of: + +```shell +cd clickhouse && VERSIONS=24.8,25.3,25.8 hatch run metrics:generate +``` + +If you need to add a metric type or a new scale, edit `generate_metrics.py` +(specifically the `DD_VALUE_TYPES` mapping and the `generate_queries` +function). The script then writes the JSON. diff --git a/clickhouse/CLAUDE.md b/clickhouse/CLAUDE.md new file mode 100644 index 0000000000000..43c994c2d3617 --- /dev/null +++ b/clickhouse/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/clickhouse/changelog.d/23829.fixed b/clickhouse/changelog.d/23829.fixed new file mode 100644 index 0000000000000..5b90ecb3d5942 --- /dev/null +++ b/clickhouse/changelog.d/23829.fixed @@ -0,0 +1 @@ +Store advanced-queries metric definitions as JSON loaded on first check run. diff --git a/clickhouse/datadog_checks/clickhouse/advanced_queries/__init__.py b/clickhouse/datadog_checks/clickhouse/advanced_queries/__init__.py index 939110bd80c08..91dfd419a147f 100644 --- a/clickhouse/datadog_checks/clickhouse/advanced_queries/__init__.py +++ b/clickhouse/datadog_checks/clickhouse/advanced_queries/__init__.py @@ -1,10 +1,136 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +"""Advanced ClickHouse query definitions. -from .system_async_metrics import SystemAsynchronousMetrics -from .system_errors import SystemErrors -from .system_events import SystemEvents -from .system_metrics import SystemMetrics +This package exposes four ``QueryManager`` query dicts: + +- ``SystemEvents``, ``SystemMetrics``, and ``SystemAsynchronousMetrics`` are + *bulk match queries*. Each one runs a single SQL that returns + ``(value, metric_name)`` rows, then routes the metric-name column through a + per-name lookup table to emit hundreds of Datadog metrics from one statement. + Their lookup tables are large (over 1,000 entries for ``SystemEvents``), so + the data ships as compact JSON under ``data/system_*.json`` and is + reassembled into the ``QueryManager`` shape at load time. + +- ``SystemErrors`` does not follow the bulk-match pattern. It runs a SQL that + emits one ``errors.raised`` metric tagged by name/code/remote, with no + per-row metric lookup. It lives below as a plain Python literal because the + compression that justifies JSON has nothing to compress for a one-metric, + three-tag query. + +The compact JSON schema for a bulk match query is:: + + { + "name": "", + "query": "", + "value_column": "", + "match_column": "", + "prefix": "", + "items": { + "": ["", ...], # gauge / monotonic_gauge + "temporal_percent": {"": "", ...} # carries scale + } + } + +At load time, ``load_match_query`` synthesises the two-column scaffold +(``[{value_column source}, {match_column match}]``) and expands ``items`` into +the per-entry shape ``QueryManager`` consumes:: + + items[""] = {"name": f"{prefix}.{key}", "type": "" + [, "scale": ""]} + +The three ``data/system_*.json`` files are autogenerated from ClickHouse's C++ +source by ``clickhouse/scripts/generate_metrics.py``. To update them (typically +when supporting a new ClickHouse version), run from the ``clickhouse`` +directory:: + + VERSIONS=24.8,25.3,25.8 hatch run metrics:generate + +Don't edit those JSON files by hand; the generator overwrites them on the next +run. If you need a new metric type or a schema change, edit +``generate_metrics.py`` (the ``generate_queries`` function and the +``QUERY_SPECS`` table). +""" + +from __future__ import annotations + +import json +import os +from typing import Any __all__ = ['SystemAsynchronousMetrics', 'SystemErrors', 'SystemEvents', 'SystemMetrics'] + +DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'data') + +MATCH_QUERIES = { + 'SystemEvents': 'system_events', + 'SystemMetrics': 'system_metrics', + 'SystemAsynchronousMetrics': 'system_async_metrics', +} + +_match_query_cache: dict[str, dict[str, Any]] = {} + +SystemErrors: dict[str, Any] = { + 'name': 'system.errors', + 'query': 'SELECT value, name, code, remote FROM system.errors WHERE value > 0', + 'columns': [ + {'name': 'errors.raised', 'type': 'monotonic_count'}, + {'name': 'error_name', 'type': 'tag'}, + {'name': 'error_code', 'type': 'tag'}, + {'name': 'remote', 'type': 'tag', 'boolean': True}, + ], +} + + +def load_match_query(name: str) -> dict[str, Any]: + """Read ``data/.json`` and reconstitute the QueryManager-shaped dict.""" + try: + with open(os.path.join(DATA_DIR, f'{name}.json'), encoding='utf-8') as f: + spec = json.load(f) + items = _expand_match_items(spec['items'], spec['prefix']) + return { + 'name': spec['name'], + 'query': spec['query'], + 'columns': [ + {'name': spec['value_column'], 'type': 'source'}, + { + 'name': spec['match_column'], + 'type': 'match', + 'source': spec['value_column'], + 'items': items, + }, + ], + } + except (OSError, json.JSONDecodeError, KeyError, TypeError, AttributeError) as exc: + raise RuntimeError(f'failed to load advanced query {name!r}') from exc + + +def _expand_match_items( + compact: dict[str, list[str] | dict[str, str]], prefix: str +) -> dict[str, dict[str, Any]]: + """Expand the compact ``{type: keys | {key: scale}}`` map to the per-entry dict shape.""" + merged: dict[str, dict[str, Any]] = {} + for type_name, group in compact.items(): + if isinstance(group, dict): + for key, scale in group.items(): + merged[key] = {'name': f'{prefix}.{key}', 'type': type_name, 'scale': scale} + else: + for key in group: + merged[key] = {'name': f'{prefix}.{key}', 'type': type_name} + return dict(sorted(merged.items())) + + +def warm_cache() -> None: + """Populate the match-query cache for every known name. Idempotent.""" + for attr, file in MATCH_QUERIES.items(): + if attr not in _match_query_cache: + _match_query_cache[attr] = load_match_query(file) + + +def __getattr__(name: str) -> dict[str, Any]: + if name not in MATCH_QUERIES: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + if name not in _match_query_cache: + _match_query_cache[name] = load_match_query(MATCH_QUERIES[name]) + return _match_query_cache[name] diff --git a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_async_metrics.py b/clickhouse/datadog_checks/clickhouse/advanced_queries/system_async_metrics.py deleted file mode 100644 index c203ad9a785d9..0000000000000 --- a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_async_metrics.py +++ /dev/null @@ -1,273 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# This file is autogenerated. -# To change this file you should edit scripts/templates/system_async_metrics.tpl and then run the following command: -# hatch run metrics:generate - -# https://clickhouse.com/docs/operations/system-tables/asynchronous_metrics -SystemAsynchronousMetrics = { - 'name': 'system_asynchronous_metrics', - 'query': 'SELECT value, metric FROM system.asynchronous_metrics', - 'columns': [ - {'name': 'metric_value', 'type': 'source'}, - { - 'name': 'metric_name', - 'type': 'match', - 'source': 'metric_value', - 'items': { - 'AsynchronousHeavyMetricsCalculationTimeSpent': { - 'name': 'asynchronous_metrics.AsynchronousHeavyMetricsCalculationTimeSpent', - 'type': 'gauge', - }, - 'AsynchronousHeavyMetricsUpdateInterval': { - 'name': 'asynchronous_metrics.AsynchronousHeavyMetricsUpdateInterval', - 'type': 'gauge', - }, - 'AsynchronousMetricsCalculationTimeSpent': { - 'name': 'asynchronous_metrics.AsynchronousMetricsCalculationTimeSpent', - 'type': 'gauge', - }, - 'AsynchronousMetricsUpdateInterval': { - 'name': 'asynchronous_metrics.AsynchronousMetricsUpdateInterval', - 'type': 'gauge', - }, - 'CGroupMaxCPU': {'name': 'asynchronous_metrics.CGroupMaxCPU', 'type': 'gauge'}, - 'CGroupMemoryTotal': {'name': 'asynchronous_metrics.CGroupMemoryTotal', 'type': 'gauge'}, - 'CGroupMemoryUsed': {'name': 'asynchronous_metrics.CGroupMemoryUsed', 'type': 'gauge'}, - 'CGroupSystemTime': {'name': 'asynchronous_metrics.CGroupSystemTime', 'type': 'gauge'}, - 'CGroupSystemTimeNormalized': { - 'name': 'asynchronous_metrics.CGroupSystemTimeNormalized', - 'type': 'gauge', - }, - 'CGroupUserTime': {'name': 'asynchronous_metrics.CGroupUserTime', 'type': 'gauge'}, - 'CGroupUserTimeNormalized': {'name': 'asynchronous_metrics.CGroupUserTimeNormalized', 'type': 'gauge'}, - 'CompiledExpressionCacheBytes': { - 'name': 'asynchronous_metrics.CompiledExpressionCacheBytes', - 'type': 'gauge', - }, - 'CompiledExpressionCacheCount': { - 'name': 'asynchronous_metrics.CompiledExpressionCacheCount', - 'type': 'gauge', - }, - 'DictionaryTotalFailedUpdates': { - 'name': 'asynchronous_metrics.DictionaryTotalFailedUpdates', - 'type': 'gauge', - }, - 'FilesystemCacheBytes': {'name': 'asynchronous_metrics.FilesystemCacheBytes', 'type': 'gauge'}, - 'FilesystemCacheCapacity': {'name': 'asynchronous_metrics.FilesystemCacheCapacity', 'type': 'gauge'}, - 'FilesystemCacheFiles': {'name': 'asynchronous_metrics.FilesystemCacheFiles', 'type': 'gauge'}, - 'FilesystemLogsPathAvailableBytes': { - 'name': 'asynchronous_metrics.FilesystemLogsPathAvailableBytes', - 'type': 'gauge', - }, - 'FilesystemLogsPathAvailableINodes': { - 'name': 'asynchronous_metrics.FilesystemLogsPathAvailableINodes', - 'type': 'gauge', - }, - 'FilesystemLogsPathTotalBytes': { - 'name': 'asynchronous_metrics.FilesystemLogsPathTotalBytes', - 'type': 'gauge', - }, - 'FilesystemLogsPathTotalINodes': { - 'name': 'asynchronous_metrics.FilesystemLogsPathTotalINodes', - 'type': 'gauge', - }, - 'FilesystemLogsPathUsedBytes': { - 'name': 'asynchronous_metrics.FilesystemLogsPathUsedBytes', - 'type': 'gauge', - }, - 'FilesystemLogsPathUsedINodes': { - 'name': 'asynchronous_metrics.FilesystemLogsPathUsedINodes', - 'type': 'gauge', - }, - 'FilesystemMainPathAvailableBytes': { - 'name': 'asynchronous_metrics.FilesystemMainPathAvailableBytes', - 'type': 'gauge', - }, - 'FilesystemMainPathAvailableINodes': { - 'name': 'asynchronous_metrics.FilesystemMainPathAvailableINodes', - 'type': 'gauge', - }, - 'FilesystemMainPathTotalBytes': { - 'name': 'asynchronous_metrics.FilesystemMainPathTotalBytes', - 'type': 'gauge', - }, - 'FilesystemMainPathTotalINodes': { - 'name': 'asynchronous_metrics.FilesystemMainPathTotalINodes', - 'type': 'gauge', - }, - 'FilesystemMainPathUsedBytes': { - 'name': 'asynchronous_metrics.FilesystemMainPathUsedBytes', - 'type': 'gauge', - }, - 'FilesystemMainPathUsedINodes': { - 'name': 'asynchronous_metrics.FilesystemMainPathUsedINodes', - 'type': 'gauge', - }, - 'HashTableStatsCacheEntries': { - 'name': 'asynchronous_metrics.HashTableStatsCacheEntries', - 'type': 'gauge', - }, - 'HashTableStatsCacheHits': {'name': 'asynchronous_metrics.HashTableStatsCacheHits', 'type': 'gauge'}, - 'HashTableStatsCacheMisses': { - 'name': 'asynchronous_metrics.HashTableStatsCacheMisses', - 'type': 'gauge', - }, - 'IndexMarkCacheBytes': {'name': 'asynchronous_metrics.IndexMarkCacheBytes', 'type': 'gauge'}, - 'IndexMarkCacheFiles': {'name': 'asynchronous_metrics.IndexMarkCacheFiles', 'type': 'gauge'}, - 'IndexUncompressedCacheBytes': { - 'name': 'asynchronous_metrics.IndexUncompressedCacheBytes', - 'type': 'gauge', - }, - 'IndexUncompressedCacheCells': { - 'name': 'asynchronous_metrics.IndexUncompressedCacheCells', - 'type': 'gauge', - }, - 'Jitter': {'name': 'asynchronous_metrics.Jitter', 'type': 'gauge'}, - 'LoadAverage1': {'name': 'asynchronous_metrics.LoadAverage1', 'type': 'gauge'}, - 'LoadAverage15': {'name': 'asynchronous_metrics.LoadAverage15', 'type': 'gauge'}, - 'LoadAverage5': {'name': 'asynchronous_metrics.LoadAverage5', 'type': 'gauge'}, - 'MMapCacheCells': {'name': 'asynchronous_metrics.MMapCacheCells', 'type': 'gauge'}, - 'MarkCacheBytes': {'name': 'asynchronous_metrics.MarkCacheBytes', 'type': 'gauge'}, - 'MarkCacheFiles': {'name': 'asynchronous_metrics.MarkCacheFiles', 'type': 'gauge'}, - 'MaxPartCountForPartition': {'name': 'asynchronous_metrics.MaxPartCountForPartition', 'type': 'gauge'}, - 'MemoryCode': {'name': 'asynchronous_metrics.MemoryCode', 'type': 'gauge'}, - 'MemoryDataAndStack': {'name': 'asynchronous_metrics.MemoryDataAndStack', 'type': 'gauge'}, - 'MemoryResident': {'name': 'asynchronous_metrics.MemoryResident', 'type': 'gauge'}, - 'MemoryResidentMax': {'name': 'asynchronous_metrics.MemoryResidentMax', 'type': 'gauge'}, - 'MemoryShared': {'name': 'asynchronous_metrics.MemoryShared', 'type': 'gauge'}, - 'MemoryVirtual': {'name': 'asynchronous_metrics.MemoryVirtual', 'type': 'gauge'}, - 'NetworkTCPReceiveQueue': {'name': 'asynchronous_metrics.NetworkTCPReceiveQueue', 'type': 'gauge'}, - 'NetworkTCPSocketRemoteAddresses': { - 'name': 'asynchronous_metrics.NetworkTCPSocketRemoteAddresses', - 'type': 'gauge', - }, - 'NetworkTCPSockets': {'name': 'asynchronous_metrics.NetworkTCPSockets', 'type': 'gauge'}, - 'NetworkTCPTransmitQueue': {'name': 'asynchronous_metrics.NetworkTCPTransmitQueue', 'type': 'gauge'}, - 'NetworkTCPUnrecoveredRetransmits': { - 'name': 'asynchronous_metrics.NetworkTCPUnrecoveredRetransmits', - 'type': 'gauge', - }, - 'NumberOfDatabases': {'name': 'asynchronous_metrics.NumberOfDatabases', 'type': 'gauge'}, - 'NumberOfDetachedByUserParts': { - 'name': 'asynchronous_metrics.NumberOfDetachedByUserParts', - 'type': 'gauge', - }, - 'NumberOfDetachedParts': {'name': 'asynchronous_metrics.NumberOfDetachedParts', 'type': 'gauge'}, - 'NumberOfPendingMutations': {'name': 'asynchronous_metrics.NumberOfPendingMutations', 'type': 'gauge'}, - 'NumberOfPendingMutationsOverExecutionTime': { - 'name': 'asynchronous_metrics.NumberOfPendingMutationsOverExecutionTime', - 'type': 'gauge', - }, - 'NumberOfStuckMutations': {'name': 'asynchronous_metrics.NumberOfStuckMutations', 'type': 'gauge'}, - 'NumberOfTables': {'name': 'asynchronous_metrics.NumberOfTables', 'type': 'gauge'}, - 'NumberOfTablesSystem': {'name': 'asynchronous_metrics.NumberOfTablesSystem', 'type': 'gauge'}, - 'OSCPUOverload': {'name': 'asynchronous_metrics.OSCPUOverload', 'type': 'gauge'}, - 'OSContextSwitches': {'name': 'asynchronous_metrics.OSContextSwitches', 'type': 'gauge'}, - 'OSGuestNiceTimeNormalized': { - 'name': 'asynchronous_metrics.OSGuestNiceTimeNormalized', - 'type': 'gauge', - }, - 'OSGuestTimeNormalized': {'name': 'asynchronous_metrics.OSGuestTimeNormalized', 'type': 'gauge'}, - 'OSIOWaitTimeNormalized': {'name': 'asynchronous_metrics.OSIOWaitTimeNormalized', 'type': 'gauge'}, - 'OSIdleTimeNormalized': {'name': 'asynchronous_metrics.OSIdleTimeNormalized', 'type': 'gauge'}, - 'OSInterrupts': {'name': 'asynchronous_metrics.OSInterrupts', 'type': 'gauge'}, - 'OSIrqTimeNormalized': {'name': 'asynchronous_metrics.OSIrqTimeNormalized', 'type': 'gauge'}, - 'OSMemoryAvailable': {'name': 'asynchronous_metrics.OSMemoryAvailable', 'type': 'gauge'}, - 'OSMemoryBuffers': {'name': 'asynchronous_metrics.OSMemoryBuffers', 'type': 'gauge'}, - 'OSMemoryCached': {'name': 'asynchronous_metrics.OSMemoryCached', 'type': 'gauge'}, - 'OSMemoryFreePlusCached': {'name': 'asynchronous_metrics.OSMemoryFreePlusCached', 'type': 'gauge'}, - 'OSMemoryFreeWithoutCached': { - 'name': 'asynchronous_metrics.OSMemoryFreeWithoutCached', - 'type': 'gauge', - }, - 'OSMemorySwapCached': {'name': 'asynchronous_metrics.OSMemorySwapCached', 'type': 'gauge'}, - 'OSMemoryTotal': {'name': 'asynchronous_metrics.OSMemoryTotal', 'type': 'gauge'}, - 'OSNiceTimeNormalized': {'name': 'asynchronous_metrics.OSNiceTimeNormalized', 'type': 'gauge'}, - 'OSOpenFiles': {'name': 'asynchronous_metrics.OSOpenFiles', 'type': 'gauge'}, - 'OSProcessesBlocked': {'name': 'asynchronous_metrics.OSProcessesBlocked', 'type': 'gauge'}, - 'OSProcessesCreated': {'name': 'asynchronous_metrics.OSProcessesCreated', 'type': 'gauge'}, - 'OSProcessesRunning': {'name': 'asynchronous_metrics.OSProcessesRunning', 'type': 'gauge'}, - 'OSSoftIrqTimeNormalized': {'name': 'asynchronous_metrics.OSSoftIrqTimeNormalized', 'type': 'gauge'}, - 'OSStealTimeNormalized': {'name': 'asynchronous_metrics.OSStealTimeNormalized', 'type': 'gauge'}, - 'OSSystemTimeNormalized': {'name': 'asynchronous_metrics.OSSystemTimeNormalized', 'type': 'gauge'}, - 'OSThreadsRunnable': {'name': 'asynchronous_metrics.OSThreadsRunnable', 'type': 'gauge'}, - 'OSThreadsTotal': {'name': 'asynchronous_metrics.OSThreadsTotal', 'type': 'gauge'}, - 'OSUptime': {'name': 'asynchronous_metrics.OSUptime', 'type': 'gauge'}, - 'OSUserTimeNormalized': {'name': 'asynchronous_metrics.OSUserTimeNormalized', 'type': 'gauge'}, - 'PageCacheBytes': {'name': 'asynchronous_metrics.PageCacheBytes', 'type': 'gauge'}, - 'PageCacheCells': {'name': 'asynchronous_metrics.PageCacheCells', 'type': 'gauge'}, - 'PageCacheMaxBytes': {'name': 'asynchronous_metrics.PageCacheMaxBytes', 'type': 'gauge'}, - 'PageCachePinnedBytes': {'name': 'asynchronous_metrics.PageCachePinnedBytes', 'type': 'gauge'}, - 'PrimaryIndexCacheBytes': {'name': 'asynchronous_metrics.PrimaryIndexCacheBytes', 'type': 'gauge'}, - 'PrimaryIndexCacheFiles': {'name': 'asynchronous_metrics.PrimaryIndexCacheFiles', 'type': 'gauge'}, - 'QueryCacheBytes': {'name': 'asynchronous_metrics.QueryCacheBytes', 'type': 'gauge'}, - 'QueryCacheEntries': {'name': 'asynchronous_metrics.QueryCacheEntries', 'type': 'gauge'}, - 'ReplicasMaxAbsoluteDelay': {'name': 'asynchronous_metrics.ReplicasMaxAbsoluteDelay', 'type': 'gauge'}, - 'ReplicasMaxInsertsInQueue': { - 'name': 'asynchronous_metrics.ReplicasMaxInsertsInQueue', - 'type': 'gauge', - }, - 'ReplicasMaxMergesInQueue': {'name': 'asynchronous_metrics.ReplicasMaxMergesInQueue', 'type': 'gauge'}, - 'ReplicasMaxQueueSize': {'name': 'asynchronous_metrics.ReplicasMaxQueueSize', 'type': 'gauge'}, - 'ReplicasMaxRelativeDelay': {'name': 'asynchronous_metrics.ReplicasMaxRelativeDelay', 'type': 'gauge'}, - 'ReplicasSumInsertsInQueue': { - 'name': 'asynchronous_metrics.ReplicasSumInsertsInQueue', - 'type': 'gauge', - }, - 'ReplicasSumMergesInQueue': {'name': 'asynchronous_metrics.ReplicasSumMergesInQueue', 'type': 'gauge'}, - 'ReplicasSumQueueSize': {'name': 'asynchronous_metrics.ReplicasSumQueueSize', 'type': 'gauge'}, - 'TotalBytesOfMergeTreeTables': { - 'name': 'asynchronous_metrics.TotalBytesOfMergeTreeTables', - 'type': 'gauge', - }, - 'TotalBytesOfMergeTreeTablesSystem': { - 'name': 'asynchronous_metrics.TotalBytesOfMergeTreeTablesSystem', - 'type': 'gauge', - }, - 'TotalIndexGranularityBytesInMemory': { - 'name': 'asynchronous_metrics.TotalIndexGranularityBytesInMemory', - 'type': 'gauge', - }, - 'TotalIndexGranularityBytesInMemoryAllocated': { - 'name': 'asynchronous_metrics.TotalIndexGranularityBytesInMemoryAllocated', - 'type': 'gauge', - }, - 'TotalPartsOfMergeTreeTables': { - 'name': 'asynchronous_metrics.TotalPartsOfMergeTreeTables', - 'type': 'gauge', - }, - 'TotalPartsOfMergeTreeTablesSystem': { - 'name': 'asynchronous_metrics.TotalPartsOfMergeTreeTablesSystem', - 'type': 'gauge', - }, - 'TotalPrimaryKeyBytesInMemory': { - 'name': 'asynchronous_metrics.TotalPrimaryKeyBytesInMemory', - 'type': 'gauge', - }, - 'TotalPrimaryKeyBytesInMemoryAllocated': { - 'name': 'asynchronous_metrics.TotalPrimaryKeyBytesInMemoryAllocated', - 'type': 'gauge', - }, - 'TotalRowsOfMergeTreeTables': { - 'name': 'asynchronous_metrics.TotalRowsOfMergeTreeTables', - 'type': 'gauge', - }, - 'TotalRowsOfMergeTreeTablesSystem': { - 'name': 'asynchronous_metrics.TotalRowsOfMergeTreeTablesSystem', - 'type': 'gauge', - }, - 'TrackedMemory': {'name': 'asynchronous_metrics.TrackedMemory', 'type': 'gauge'}, - 'UncompressedCacheBytes': {'name': 'asynchronous_metrics.UncompressedCacheBytes', 'type': 'gauge'}, - 'UncompressedCacheCells': {'name': 'asynchronous_metrics.UncompressedCacheCells', 'type': 'gauge'}, - 'UnreclaimableRSS': {'name': 'asynchronous_metrics.UnreclaimableRSS', 'type': 'gauge'}, - 'Uptime': {'name': 'asynchronous_metrics.Uptime', 'type': 'gauge'}, - 'VMMaxMapCount': {'name': 'asynchronous_metrics.VMMaxMapCount', 'type': 'gauge'}, - 'VMNumMaps': {'name': 'asynchronous_metrics.VMNumMaps', 'type': 'gauge'}, - 'jemalloc.epoch': {'name': 'asynchronous_metrics.jemalloc.epoch', 'type': 'gauge'}, - }, - }, - ], -} diff --git a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_errors.py b/clickhouse/datadog_checks/clickhouse/advanced_queries/system_errors.py deleted file mode 100644 index 685e5b6ffbe8a..0000000000000 --- a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_errors.py +++ /dev/null @@ -1,15 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# https://clickhouse.com/docs/operations/system-tables/errors -SystemErrors = { - 'name': 'system.errors', - 'query': 'SELECT value, name, code, remote FROM system.errors WHERE value > 0', - 'columns': [ - {'name': 'errors.raised', 'type': 'monotonic_count'}, - {'name': 'error_name', 'type': 'tag'}, - {'name': 'error_code', 'type': 'tag'}, - {'name': 'remote', 'type': 'tag', 'boolean': True}, - ], -} diff --git a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_events.py b/clickhouse/datadog_checks/clickhouse/advanced_queries/system_events.py deleted file mode 100644 index 3a8ef7ef2662c..0000000000000 --- a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_events.py +++ /dev/null @@ -1,3073 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# This file is autogenerated. -# To change this file you should edit scripts/templates/system_events.tpl and then run the following command: -# hatch run metrics:generate - -# https://clickhouse.com/docs/operations/system-tables/events -SystemEvents = { - 'name': 'system_events', - 'query': 'SELECT value, event FROM system.events', - 'columns': [ - {'name': 'metric_value', 'type': 'source'}, - { - 'name': 'metric_name', - 'type': 'match', - 'source': 'metric_value', - 'items': { - 'AIORead': {'name': 'events.AIORead', 'type': 'monotonic_gauge'}, - 'AIOReadBytes': {'name': 'events.AIOReadBytes', 'type': 'monotonic_gauge'}, - 'AIOWrite': {'name': 'events.AIOWrite', 'type': 'monotonic_gauge'}, - 'AIOWriteBytes': {'name': 'events.AIOWriteBytes', 'type': 'monotonic_gauge'}, - 'AddressesDiscovered': {'name': 'events.AddressesDiscovered', 'type': 'monotonic_gauge'}, - 'AddressesExpired': {'name': 'events.AddressesExpired', 'type': 'monotonic_gauge'}, - 'AddressesMarkedAsFailed': {'name': 'events.AddressesMarkedAsFailed', 'type': 'monotonic_gauge'}, - 'AggregatingSortedMilliseconds': { - 'name': 'events.AggregatingSortedMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'AggregationHashTablesInitializedAsTwoLevel': { - 'name': 'events.AggregationHashTablesInitializedAsTwoLevel', - 'type': 'monotonic_gauge', - }, - 'AggregationOptimizedEqualRangesOfKeys': { - 'name': 'events.AggregationOptimizedEqualRangesOfKeys', - 'type': 'monotonic_gauge', - }, - 'AggregationPreallocatedElementsInHashTables': { - 'name': 'events.AggregationPreallocatedElementsInHashTables', - 'type': 'monotonic_gauge', - }, - 'AnalyzePatchRangesMicroseconds': { - 'name': 'events.AnalyzePatchRangesMicroseconds', - 'type': 'monotonic_gauge', - }, - 'ApplyPatchesMicroseconds': {'name': 'events.ApplyPatchesMicroseconds', 'type': 'monotonic_gauge'}, - 'ArenaAllocBytes': {'name': 'events.ArenaAllocBytes', 'type': 'monotonic_gauge'}, - 'ArenaAllocChunks': {'name': 'events.ArenaAllocChunks', 'type': 'monotonic_gauge'}, - 'AsyncInsertBytes': {'name': 'events.AsyncInsertBytes', 'type': 'monotonic_gauge'}, - 'AsyncInsertCacheHits': {'name': 'events.AsyncInsertCacheHits', 'type': 'monotonic_gauge'}, - 'AsyncInsertQuery': {'name': 'events.AsyncInsertQuery', 'type': 'monotonic_gauge'}, - 'AsyncInsertRows': {'name': 'events.AsyncInsertRows', 'type': 'monotonic_gauge'}, - 'AsyncLoaderWaitMicroseconds': { - 'name': 'events.AsyncLoaderWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'AsyncLoggingConsoleDroppedMessages': { - 'name': 'events.AsyncLoggingConsoleDroppedMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingConsoleTotalMessages': { - 'name': 'events.AsyncLoggingConsoleTotalMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingErrorFileLogDroppedMessages': { - 'name': 'events.AsyncLoggingErrorFileLogDroppedMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingErrorFileLogTotalMessages': { - 'name': 'events.AsyncLoggingErrorFileLogTotalMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingFileLogDroppedMessages': { - 'name': 'events.AsyncLoggingFileLogDroppedMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingFileLogTotalMessages': { - 'name': 'events.AsyncLoggingFileLogTotalMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingSyslogDroppedMessages': { - 'name': 'events.AsyncLoggingSyslogDroppedMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingSyslogTotalMessages': { - 'name': 'events.AsyncLoggingSyslogTotalMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingTextLogDroppedMessages': { - 'name': 'events.AsyncLoggingTextLogDroppedMessages', - 'type': 'monotonic_gauge', - }, - 'AsyncLoggingTextLogTotalMessages': { - 'name': 'events.AsyncLoggingTextLogTotalMessages', - 'type': 'monotonic_gauge', - }, - 'AsynchronousReadWaitMicroseconds': { - 'name': 'events.AsynchronousReadWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'AsynchronousReaderIgnoredBytes': { - 'name': 'events.AsynchronousReaderIgnoredBytes', - 'type': 'monotonic_gauge', - }, - 'AsynchronousRemoteReadWaitMicroseconds': { - 'name': 'events.AsynchronousRemoteReadWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'AzureCommitBlockList': {'name': 'events.AzureCommitBlockList', 'type': 'monotonic_gauge'}, - 'AzureCopyObject': {'name': 'events.AzureCopyObject', 'type': 'monotonic_gauge'}, - 'AzureCreateContainer': {'name': 'events.AzureCreateContainer', 'type': 'monotonic_gauge'}, - 'AzureDeleteObjects': {'name': 'events.AzureDeleteObjects', 'type': 'monotonic_gauge'}, - 'AzureGetObject': {'name': 'events.AzureGetObject', 'type': 'monotonic_gauge'}, - 'AzureGetProperties': {'name': 'events.AzureGetProperties', 'type': 'monotonic_gauge'}, - 'AzureGetRequestThrottlerCount': { - 'name': 'events.AzureGetRequestThrottlerCount', - 'type': 'monotonic_gauge', - }, - 'AzureGetRequestThrottlerSleepMicroseconds': { - 'name': 'events.AzureGetRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'AzureListObjects': {'name': 'events.AzureListObjects', 'type': 'monotonic_gauge'}, - 'AzurePutRequestThrottlerCount': { - 'name': 'events.AzurePutRequestThrottlerCount', - 'type': 'monotonic_gauge', - }, - 'AzurePutRequestThrottlerSleepMicroseconds': { - 'name': 'events.AzurePutRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'AzureReadMicroseconds': { - 'name': 'events.AzureReadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'AzureReadRequestsCount': {'name': 'events.AzureReadRequestsCount', 'type': 'monotonic_gauge'}, - 'AzureReadRequestsErrors': {'name': 'events.AzureReadRequestsErrors', 'type': 'monotonic_gauge'}, - 'AzureReadRequestsRedirects': {'name': 'events.AzureReadRequestsRedirects', 'type': 'monotonic_gauge'}, - 'AzureReadRequestsThrottling': { - 'name': 'events.AzureReadRequestsThrottling', - 'type': 'monotonic_gauge', - }, - 'AzureStageBlock': {'name': 'events.AzureStageBlock', 'type': 'monotonic_gauge'}, - 'AzureUpload': {'name': 'events.AzureUpload', 'type': 'monotonic_gauge'}, - 'AzureWriteMicroseconds': { - 'name': 'events.AzureWriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'AzureWriteRequestsCount': {'name': 'events.AzureWriteRequestsCount', 'type': 'monotonic_gauge'}, - 'AzureWriteRequestsErrors': {'name': 'events.AzureWriteRequestsErrors', 'type': 'monotonic_gauge'}, - 'AzureWriteRequestsRedirects': { - 'name': 'events.AzureWriteRequestsRedirects', - 'type': 'monotonic_gauge', - }, - 'AzureWriteRequestsThrottling': { - 'name': 'events.AzureWriteRequestsThrottling', - 'type': 'monotonic_gauge', - }, - 'BackgroundLoadingMarksTasks': { - 'name': 'events.BackgroundLoadingMarksTasks', - 'type': 'monotonic_gauge', - }, - 'BackupEntriesCollectorForTablesDataMicroseconds': { - 'name': 'events.BackupEntriesCollectorForTablesDataMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'BackupEntriesCollectorMicroseconds': { - 'name': 'events.BackupEntriesCollectorMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'BackupEntriesCollectorRunPostTasksMicroseconds': { - 'name': 'events.BackupEntriesCollectorRunPostTasksMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'BackupLockFileReads': {'name': 'events.BackupLockFileReads', 'type': 'monotonic_gauge'}, - 'BackupPreparingFileInfosMicroseconds': { - 'name': 'events.BackupPreparingFileInfosMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'BackupReadLocalBytesToCalculateChecksums': { - 'name': 'events.BackupReadLocalBytesToCalculateChecksums', - 'type': 'monotonic_gauge', - }, - 'BackupReadLocalFilesToCalculateChecksums': { - 'name': 'events.BackupReadLocalFilesToCalculateChecksums', - 'type': 'monotonic_gauge', - }, - 'BackupReadMetadataMicroseconds': { - 'name': 'events.BackupReadMetadataMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'BackupReadRemoteBytesToCalculateChecksums': { - 'name': 'events.BackupReadRemoteBytesToCalculateChecksums', - 'type': 'monotonic_gauge', - }, - 'BackupReadRemoteFilesToCalculateChecksums': { - 'name': 'events.BackupReadRemoteFilesToCalculateChecksums', - 'type': 'monotonic_gauge', - }, - 'BackupThrottlerBytes': {'name': 'events.BackupThrottlerBytes', 'type': 'monotonic_gauge'}, - 'BackupThrottlerSleepMicroseconds': { - 'name': 'events.BackupThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'BackupWriteMetadataMicroseconds': { - 'name': 'events.BackupWriteMetadataMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'BackupsOpenedForRead': {'name': 'events.BackupsOpenedForRead', 'type': 'monotonic_gauge'}, - 'BackupsOpenedForUnlock': {'name': 'events.BackupsOpenedForUnlock', 'type': 'monotonic_gauge'}, - 'BackupsOpenedForWrite': {'name': 'events.BackupsOpenedForWrite', 'type': 'monotonic_gauge'}, - 'BuildPatchesJoinMicroseconds': { - 'name': 'events.BuildPatchesJoinMicroseconds', - 'type': 'monotonic_gauge', - }, - 'BuildPatchesMergeMicroseconds': { - 'name': 'events.BuildPatchesMergeMicroseconds', - 'type': 'monotonic_gauge', - }, - 'CacheWarmerBytesDownloaded': {'name': 'events.CacheWarmerBytesDownloaded', 'type': 'monotonic_gauge'}, - 'CacheWarmerDataPartsDownloaded': { - 'name': 'events.CacheWarmerDataPartsDownloaded', - 'type': 'monotonic_gauge', - }, - 'CachedReadBufferCacheWriteBytes': { - 'name': 'events.CachedReadBufferCacheWriteBytes', - 'type': 'monotonic_gauge', - }, - 'CachedReadBufferCacheWriteMicroseconds': { - 'name': 'events.CachedReadBufferCacheWriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CachedReadBufferCreateBufferMicroseconds': { - 'name': 'events.CachedReadBufferCreateBufferMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CachedReadBufferPredownloadedBytes': { - 'name': 'events.CachedReadBufferPredownloadedBytes', - 'type': 'monotonic_gauge', - }, - 'CachedReadBufferReadFromCacheBytes': { - 'name': 'events.CachedReadBufferReadFromCacheBytes', - 'type': 'monotonic_gauge', - }, - 'CachedReadBufferReadFromCacheHits': { - 'name': 'events.CachedReadBufferReadFromCacheHits', - 'type': 'monotonic_gauge', - }, - 'CachedReadBufferReadFromCacheMicroseconds': { - 'name': 'events.CachedReadBufferReadFromCacheMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CachedReadBufferReadFromCacheMisses': { - 'name': 'events.CachedReadBufferReadFromCacheMisses', - 'type': 'monotonic_gauge', - }, - 'CachedReadBufferReadFromSourceBytes': { - 'name': 'events.CachedReadBufferReadFromSourceBytes', - 'type': 'monotonic_gauge', - }, - 'CachedReadBufferReadFromSourceMicroseconds': { - 'name': 'events.CachedReadBufferReadFromSourceMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CachedReadBufferWaitReadBufferMicroseconds': { - 'name': 'events.CachedReadBufferWaitReadBufferMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CachedWriteBufferCacheWriteBytes': { - 'name': 'events.CachedWriteBufferCacheWriteBytes', - 'type': 'monotonic_gauge', - }, - 'CachedWriteBufferCacheWriteMicroseconds': { - 'name': 'events.CachedWriteBufferCacheWriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CannotRemoveEphemeralNode': {'name': 'events.CannotRemoveEphemeralNode', 'type': 'monotonic_gauge'}, - 'CannotWriteToWriteBufferDiscard': { - 'name': 'events.CannotWriteToWriteBufferDiscard', - 'type': 'monotonic_gauge', - }, - 'CoalescingSortedMilliseconds': { - 'name': 'events.CoalescingSortedMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'CollapsingSortedMilliseconds': { - 'name': 'events.CollapsingSortedMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'CommonBackgroundExecutorTaskCancelMicroseconds': { - 'name': 'events.CommonBackgroundExecutorTaskCancelMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CommonBackgroundExecutorTaskExecuteStepMicroseconds': { - 'name': 'events.CommonBackgroundExecutorTaskExecuteStepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CommonBackgroundExecutorTaskResetMicroseconds': { - 'name': 'events.CommonBackgroundExecutorTaskResetMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CommonBackgroundExecutorWaitMicroseconds': { - 'name': 'events.CommonBackgroundExecutorWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CompileExpressionsBytes': {'name': 'events.CompileExpressionsBytes', 'type': 'monotonic_gauge'}, - 'CompileExpressionsMicroseconds': { - 'name': 'events.CompileExpressionsMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CompileFunction': {'name': 'events.CompileFunction', 'type': 'monotonic_gauge'}, - 'CompiledFunctionExecute': {'name': 'events.CompiledFunctionExecute', 'type': 'monotonic_gauge'}, - 'CompressedReadBufferBlocks': {'name': 'events.CompressedReadBufferBlocks', 'type': 'monotonic_gauge'}, - 'CompressedReadBufferBytes': {'name': 'events.CompressedReadBufferBytes', 'type': 'monotonic_gauge'}, - 'CompressedReadBufferChecksumDoesntMatch': { - 'name': 'events.CompressedReadBufferChecksumDoesntMatch', - 'type': 'monotonic_gauge', - }, - 'CompressedReadBufferChecksumDoesntMatchMicroseconds': { - 'name': 'events.CompressedReadBufferChecksumDoesntMatchMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CompressedReadBufferChecksumDoesntMatchSingleBitMismatch': { - 'name': 'events.CompressedReadBufferChecksumDoesntMatchSingleBitMismatch', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlDownscales': { - 'name': 'events.ConcurrencyControlDownscales', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlPreemptedMicroseconds': { - 'name': 'events.ConcurrencyControlPreemptedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ConcurrencyControlPreemptions': { - 'name': 'events.ConcurrencyControlPreemptions', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlQueriesDelayed': { - 'name': 'events.ConcurrencyControlQueriesDelayed', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlSlotsAcquired': { - 'name': 'events.ConcurrencyControlSlotsAcquired', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlSlotsAcquiredNonCompeting': { - 'name': 'events.ConcurrencyControlSlotsAcquiredNonCompeting', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlSlotsDelayed': { - 'name': 'events.ConcurrencyControlSlotsDelayed', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlSlotsGranted': { - 'name': 'events.ConcurrencyControlSlotsGranted', - 'type': 'monotonic_gauge', - }, - 'ConcurrencyControlUpscales': {'name': 'events.ConcurrencyControlUpscales', 'type': 'monotonic_gauge'}, - 'ConcurrencyControlWaitMicroseconds': { - 'name': 'events.ConcurrencyControlWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ConcurrentQuerySlotsAcquired': { - 'name': 'events.ConcurrentQuerySlotsAcquired', - 'type': 'monotonic_gauge', - }, - 'ConcurrentQueryWaitMicroseconds': { - 'name': 'events.ConcurrentQueryWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ConnectionPoolIsFullMicroseconds': { - 'name': 'events.ConnectionPoolIsFullMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ContextLock': {'name': 'events.ContextLock', 'type': 'monotonic_gauge'}, - 'ContextLockWaitMicroseconds': { - 'name': 'events.ContextLockWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeAssignmentRequest': { - 'name': 'events.CoordinatedMergesMergeAssignmentRequest', - 'type': 'monotonic_gauge', - }, - 'CoordinatedMergesMergeAssignmentRequestMicroseconds': { - 'name': 'events.CoordinatedMergesMergeAssignmentRequestMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeAssignmentResponse': { - 'name': 'events.CoordinatedMergesMergeAssignmentResponse', - 'type': 'monotonic_gauge', - }, - 'CoordinatedMergesMergeAssignmentResponseMicroseconds': { - 'name': 'events.CoordinatedMergesMergeAssignmentResponseMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeCoordinatorFetchMetadataMicroseconds': { - 'name': 'events.CoordinatedMergesMergeCoordinatorFetchMetadataMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeCoordinatorFilterMicroseconds': { - 'name': 'events.CoordinatedMergesMergeCoordinatorFilterMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeCoordinatorLockStateExclusivelyCount': { - 'name': 'events.CoordinatedMergesMergeCoordinatorLockStateExclusivelyCount', - 'type': 'monotonic_gauge', - }, - 'CoordinatedMergesMergeCoordinatorLockStateExclusivelyMicroseconds': { - 'name': 'events.CoordinatedMergesMergeCoordinatorLockStateExclusivelyMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeCoordinatorLockStateForShareCount': { - 'name': 'events.CoordinatedMergesMergeCoordinatorLockStateForShareCount', - 'type': 'monotonic_gauge', - }, - 'CoordinatedMergesMergeCoordinatorLockStateForShareMicroseconds': { - 'name': 'events.CoordinatedMergesMergeCoordinatorLockStateForShareMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeCoordinatorSelectMergesMicroseconds': { - 'name': 'events.CoordinatedMergesMergeCoordinatorSelectMergesMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeCoordinatorUpdateCount': { - 'name': 'events.CoordinatedMergesMergeCoordinatorUpdateCount', - 'type': 'monotonic_gauge', - }, - 'CoordinatedMergesMergeCoordinatorUpdateMicroseconds': { - 'name': 'events.CoordinatedMergesMergeCoordinatorUpdateMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CoordinatedMergesMergeWorkerUpdateCount': { - 'name': 'events.CoordinatedMergesMergeWorkerUpdateCount', - 'type': 'monotonic_gauge', - }, - 'CoordinatedMergesMergeWorkerUpdateMicroseconds': { - 'name': 'events.CoordinatedMergesMergeWorkerUpdateMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'CreatedLogEntryForMerge': {'name': 'events.CreatedLogEntryForMerge', 'type': 'monotonic_gauge'}, - 'CreatedLogEntryForMutation': {'name': 'events.CreatedLogEntryForMutation', 'type': 'monotonic_gauge'}, - 'CreatedReadBufferDirectIO': {'name': 'events.CreatedReadBufferDirectIO', 'type': 'monotonic_gauge'}, - 'CreatedReadBufferDirectIOFailed': { - 'name': 'events.CreatedReadBufferDirectIOFailed', - 'type': 'monotonic_gauge', - }, - 'CreatedReadBufferMMap': {'name': 'events.CreatedReadBufferMMap', 'type': 'monotonic_gauge'}, - 'CreatedReadBufferMMapFailed': { - 'name': 'events.CreatedReadBufferMMapFailed', - 'type': 'monotonic_gauge', - }, - 'CreatedReadBufferOrdinary': {'name': 'events.CreatedReadBufferOrdinary', 'type': 'monotonic_gauge'}, - 'DNSError': {'name': 'events.DNSError', 'type': 'monotonic_gauge'}, - 'DataAfterMutationDiffersFromReplica': { - 'name': 'events.DataAfterMutationDiffersFromReplica', - 'type': 'monotonic_gauge', - }, - 'DefaultImplementationForNullsRows': { - 'name': 'events.DefaultImplementationForNullsRows', - 'type': 'monotonic_gauge', - }, - 'DefaultImplementationForNullsRowsWithNulls': { - 'name': 'events.DefaultImplementationForNullsRowsWithNulls', - 'type': 'monotonic_gauge', - }, - 'DelayedInserts': {'name': 'events.DelayedInserts', 'type': 'monotonic_gauge'}, - 'DelayedInsertsMilliseconds': { - 'name': 'events.DelayedInsertsMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'DelayedMutations': {'name': 'events.DelayedMutations', 'type': 'monotonic_gauge'}, - 'DelayedMutationsMilliseconds': { - 'name': 'events.DelayedMutationsMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'DeltaLakePartitionPrunedFiles': { - 'name': 'events.DeltaLakePartitionPrunedFiles', - 'type': 'monotonic_gauge', - }, - 'DictCacheKeysExpired': {'name': 'events.DictCacheKeysExpired', 'type': 'monotonic_gauge'}, - 'DictCacheKeysHit': {'name': 'events.DictCacheKeysHit', 'type': 'monotonic_gauge'}, - 'DictCacheKeysNotFound': {'name': 'events.DictCacheKeysNotFound', 'type': 'monotonic_gauge'}, - 'DictCacheKeysRequested': {'name': 'events.DictCacheKeysRequested', 'type': 'monotonic_gauge'}, - 'DictCacheKeysRequestedFound': { - 'name': 'events.DictCacheKeysRequestedFound', - 'type': 'monotonic_gauge', - }, - 'DictCacheKeysRequestedMiss': {'name': 'events.DictCacheKeysRequestedMiss', 'type': 'monotonic_gauge'}, - 'DictCacheLockReadNs': { - 'name': 'events.DictCacheLockReadNs', - 'type': 'temporal_percent', - 'scale': 'nanosecond', - }, - 'DictCacheLockWriteNs': { - 'name': 'events.DictCacheLockWriteNs', - 'type': 'temporal_percent', - 'scale': 'nanosecond', - }, - 'DictCacheRequestTimeNs': { - 'name': 'events.DictCacheRequestTimeNs', - 'type': 'temporal_percent', - 'scale': 'nanosecond', - }, - 'DictCacheRequests': {'name': 'events.DictCacheRequests', 'type': 'monotonic_gauge'}, - 'DirectorySync': {'name': 'events.DirectorySync', 'type': 'monotonic_gauge'}, - 'DirectorySyncElapsedMicroseconds': { - 'name': 'events.DirectorySyncElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskAzureCommitBlockList': {'name': 'events.DiskAzureCommitBlockList', 'type': 'monotonic_gauge'}, - 'DiskAzureCopyObject': {'name': 'events.DiskAzureCopyObject', 'type': 'monotonic_gauge'}, - 'DiskAzureCreateContainer': {'name': 'events.DiskAzureCreateContainer', 'type': 'monotonic_gauge'}, - 'DiskAzureDeleteObjects': {'name': 'events.DiskAzureDeleteObjects', 'type': 'monotonic_gauge'}, - 'DiskAzureGetObject': {'name': 'events.DiskAzureGetObject', 'type': 'monotonic_gauge'}, - 'DiskAzureGetProperties': {'name': 'events.DiskAzureGetProperties', 'type': 'monotonic_gauge'}, - 'DiskAzureGetRequestThrottlerCount': { - 'name': 'events.DiskAzureGetRequestThrottlerCount', - 'type': 'monotonic_gauge', - }, - 'DiskAzureGetRequestThrottlerSleepMicroseconds': { - 'name': 'events.DiskAzureGetRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskAzureListObjects': {'name': 'events.DiskAzureListObjects', 'type': 'monotonic_gauge'}, - 'DiskAzurePutRequestThrottlerCount': { - 'name': 'events.DiskAzurePutRequestThrottlerCount', - 'type': 'monotonic_gauge', - }, - 'DiskAzurePutRequestThrottlerSleepMicroseconds': { - 'name': 'events.DiskAzurePutRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskAzureReadMicroseconds': { - 'name': 'events.DiskAzureReadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskAzureReadRequestsCount': {'name': 'events.DiskAzureReadRequestsCount', 'type': 'monotonic_gauge'}, - 'DiskAzureReadRequestsErrors': { - 'name': 'events.DiskAzureReadRequestsErrors', - 'type': 'monotonic_gauge', - }, - 'DiskAzureReadRequestsRedirects': { - 'name': 'events.DiskAzureReadRequestsRedirects', - 'type': 'monotonic_gauge', - }, - 'DiskAzureReadRequestsThrottling': { - 'name': 'events.DiskAzureReadRequestsThrottling', - 'type': 'monotonic_gauge', - }, - 'DiskAzureStageBlock': {'name': 'events.DiskAzureStageBlock', 'type': 'monotonic_gauge'}, - 'DiskAzureUpload': {'name': 'events.DiskAzureUpload', 'type': 'monotonic_gauge'}, - 'DiskAzureWriteMicroseconds': { - 'name': 'events.DiskAzureWriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskAzureWriteRequestsCount': { - 'name': 'events.DiskAzureWriteRequestsCount', - 'type': 'monotonic_gauge', - }, - 'DiskAzureWriteRequestsErrors': { - 'name': 'events.DiskAzureWriteRequestsErrors', - 'type': 'monotonic_gauge', - }, - 'DiskAzureWriteRequestsRedirects': { - 'name': 'events.DiskAzureWriteRequestsRedirects', - 'type': 'monotonic_gauge', - }, - 'DiskAzureWriteRequestsThrottling': { - 'name': 'events.DiskAzureWriteRequestsThrottling', - 'type': 'monotonic_gauge', - }, - 'DiskConnectionsCreated': {'name': 'events.DiskConnectionsCreated', 'type': 'monotonic_gauge'}, - 'DiskConnectionsElapsedMicroseconds': { - 'name': 'events.DiskConnectionsElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskConnectionsErrors': {'name': 'events.DiskConnectionsErrors', 'type': 'monotonic_gauge'}, - 'DiskConnectionsExpired': {'name': 'events.DiskConnectionsExpired', 'type': 'monotonic_gauge'}, - 'DiskConnectionsPreserved': {'name': 'events.DiskConnectionsPreserved', 'type': 'monotonic_gauge'}, - 'DiskConnectionsReset': {'name': 'events.DiskConnectionsReset', 'type': 'monotonic_gauge'}, - 'DiskConnectionsReused': {'name': 'events.DiskConnectionsReused', 'type': 'monotonic_gauge'}, - 'DiskPlainRewritableAzureDirectoryCreated': { - 'name': 'events.DiskPlainRewritableAzureDirectoryCreated', - 'type': 'monotonic_gauge', - }, - 'DiskPlainRewritableAzureDirectoryRemoved': { - 'name': 'events.DiskPlainRewritableAzureDirectoryRemoved', - 'type': 'monotonic_gauge', - }, - 'DiskPlainRewritableLegacyLayoutDiskCount': { - 'name': 'events.DiskPlainRewritableLegacyLayoutDiskCount', - 'type': 'monotonic_gauge', - }, - 'DiskPlainRewritableLocalDirectoryCreated': { - 'name': 'events.DiskPlainRewritableLocalDirectoryCreated', - 'type': 'monotonic_gauge', - }, - 'DiskPlainRewritableLocalDirectoryRemoved': { - 'name': 'events.DiskPlainRewritableLocalDirectoryRemoved', - 'type': 'monotonic_gauge', - }, - 'DiskPlainRewritableS3DirectoryCreated': { - 'name': 'events.DiskPlainRewritableS3DirectoryCreated', - 'type': 'monotonic_gauge', - }, - 'DiskPlainRewritableS3DirectoryRemoved': { - 'name': 'events.DiskPlainRewritableS3DirectoryRemoved', - 'type': 'monotonic_gauge', - }, - 'DiskReadElapsedMicroseconds': { - 'name': 'events.DiskReadElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskS3AbortMultipartUpload': {'name': 'events.DiskS3AbortMultipartUpload', 'type': 'monotonic_gauge'}, - 'DiskS3CompleteMultipartUpload': { - 'name': 'events.DiskS3CompleteMultipartUpload', - 'type': 'monotonic_gauge', - }, - 'DiskS3CopyObject': {'name': 'events.DiskS3CopyObject', 'type': 'monotonic_gauge'}, - 'DiskS3CreateMultipartUpload': { - 'name': 'events.DiskS3CreateMultipartUpload', - 'type': 'monotonic_gauge', - }, - 'DiskS3DeleteObjects': {'name': 'events.DiskS3DeleteObjects', 'type': 'monotonic_gauge'}, - 'DiskS3GetObject': {'name': 'events.DiskS3GetObject', 'type': 'monotonic_gauge'}, - 'DiskS3GetObjectAttributes': {'name': 'events.DiskS3GetObjectAttributes', 'type': 'monotonic_gauge'}, - 'DiskS3GetRequestThrottlerCount': { - 'name': 'events.DiskS3GetRequestThrottlerCount', - 'type': 'monotonic_gauge', - }, - 'DiskS3GetRequestThrottlerSleepMicroseconds': { - 'name': 'events.DiskS3GetRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskS3HeadObject': {'name': 'events.DiskS3HeadObject', 'type': 'monotonic_gauge'}, - 'DiskS3ListObjects': {'name': 'events.DiskS3ListObjects', 'type': 'monotonic_gauge'}, - 'DiskS3PutObject': {'name': 'events.DiskS3PutObject', 'type': 'monotonic_gauge'}, - 'DiskS3PutRequestThrottlerCount': { - 'name': 'events.DiskS3PutRequestThrottlerCount', - 'type': 'monotonic_gauge', - }, - 'DiskS3PutRequestThrottlerSleepMicroseconds': { - 'name': 'events.DiskS3PutRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskS3ReadMicroseconds': { - 'name': 'events.DiskS3ReadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskS3ReadRequestAttempts': {'name': 'events.DiskS3ReadRequestAttempts', 'type': 'monotonic_gauge'}, - 'DiskS3ReadRequestRetryableErrors': { - 'name': 'events.DiskS3ReadRequestRetryableErrors', - 'type': 'monotonic_gauge', - }, - 'DiskS3ReadRequestsCount': {'name': 'events.DiskS3ReadRequestsCount', 'type': 'monotonic_gauge'}, - 'DiskS3ReadRequestsErrors': {'name': 'events.DiskS3ReadRequestsErrors', 'type': 'monotonic_gauge'}, - 'DiskS3ReadRequestsRedirects': { - 'name': 'events.DiskS3ReadRequestsRedirects', - 'type': 'monotonic_gauge', - }, - 'DiskS3ReadRequestsThrottling': { - 'name': 'events.DiskS3ReadRequestsThrottling', - 'type': 'monotonic_gauge', - }, - 'DiskS3UploadPart': {'name': 'events.DiskS3UploadPart', 'type': 'monotonic_gauge'}, - 'DiskS3UploadPartCopy': {'name': 'events.DiskS3UploadPartCopy', 'type': 'monotonic_gauge'}, - 'DiskS3WriteMicroseconds': { - 'name': 'events.DiskS3WriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DiskS3WriteRequestAttempts': {'name': 'events.DiskS3WriteRequestAttempts', 'type': 'monotonic_gauge'}, - 'DiskS3WriteRequestRetryableErrors': { - 'name': 'events.DiskS3WriteRequestRetryableErrors', - 'type': 'monotonic_gauge', - }, - 'DiskS3WriteRequestsCount': {'name': 'events.DiskS3WriteRequestsCount', 'type': 'monotonic_gauge'}, - 'DiskS3WriteRequestsErrors': {'name': 'events.DiskS3WriteRequestsErrors', 'type': 'monotonic_gauge'}, - 'DiskS3WriteRequestsRedirects': { - 'name': 'events.DiskS3WriteRequestsRedirects', - 'type': 'monotonic_gauge', - }, - 'DiskS3WriteRequestsThrottling': { - 'name': 'events.DiskS3WriteRequestsThrottling', - 'type': 'monotonic_gauge', - }, - 'DiskWriteElapsedMicroseconds': { - 'name': 'events.DiskWriteElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheConnectAttempts': {'name': 'events.DistrCacheConnectAttempts', 'type': 'monotonic_gauge'}, - 'DistrCacheConnectMicroseconds': { - 'name': 'events.DistrCacheConnectMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheDataPacketsBytes': {'name': 'events.DistrCacheDataPacketsBytes', 'type': 'monotonic_gauge'}, - 'DistrCacheFallbackReadMicroseconds': { - 'name': 'events.DistrCacheFallbackReadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheGetClient': {'name': 'events.DistrCacheGetClient', 'type': 'gauge'}, - 'DistrCacheGetClientMicroseconds': { - 'name': 'events.DistrCacheGetClientMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheGetResponseMicroseconds': { - 'name': 'events.DistrCacheGetResponseMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheHashRingRebuilds': {'name': 'events.DistrCacheHashRingRebuilds', 'type': 'monotonic_gauge'}, - 'DistrCacheHoldConnections': {'name': 'events.DistrCacheHoldConnections', 'type': 'gauge'}, - 'DistrCacheIgnoredBytesWhileWaitingProfileEvents': { - 'name': 'events.DistrCacheIgnoredBytesWhileWaitingProfileEvents', - 'type': 'monotonic_gauge', - }, - 'DistrCacheLockRegistryMicroseconds': { - 'name': 'events.DistrCacheLockRegistryMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheMakeRequestErrors': { - 'name': 'events.DistrCacheMakeRequestErrors', - 'type': 'monotonic_gauge', - }, - 'DistrCacheNextImplMicroseconds': { - 'name': 'events.DistrCacheNextImplMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheOpenedConnections': { - 'name': 'events.DistrCacheOpenedConnections', - 'type': 'monotonic_gauge', - }, - 'DistrCacheOpenedConnectionsBypassingPool': { - 'name': 'events.DistrCacheOpenedConnectionsBypassingPool', - 'type': 'monotonic_gauge', - }, - 'DistrCachePackets': {'name': 'events.DistrCachePackets', 'type': 'monotonic_gauge'}, - 'DistrCachePacketsBytes': {'name': 'events.DistrCachePacketsBytes', 'type': 'monotonic_gauge'}, - 'DistrCachePrecomputeRangesMicroseconds': { - 'name': 'events.DistrCachePrecomputeRangesMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheRangeChange': {'name': 'events.DistrCacheRangeChange', 'type': 'monotonic_gauge'}, - 'DistrCacheRangeResetBackward': { - 'name': 'events.DistrCacheRangeResetBackward', - 'type': 'monotonic_gauge', - }, - 'DistrCacheRangeResetForward': { - 'name': 'events.DistrCacheRangeResetForward', - 'type': 'monotonic_gauge', - }, - 'DistrCacheReadBytesFromCache': { - 'name': 'events.DistrCacheReadBytesFromCache', - 'type': 'monotonic_gauge', - }, - 'DistrCacheReadBytesFromFallbackBuffer': { - 'name': 'events.DistrCacheReadBytesFromFallbackBuffer', - 'type': 'monotonic_gauge', - }, - 'DistrCacheReadErrors': {'name': 'events.DistrCacheReadErrors', 'type': 'monotonic_gauge'}, - 'DistrCacheReadMicroseconds': { - 'name': 'events.DistrCacheReadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheReceiveResponseErrors': { - 'name': 'events.DistrCacheReceiveResponseErrors', - 'type': 'monotonic_gauge', - }, - 'DistrCacheReconnectsAfterTimeout': { - 'name': 'events.DistrCacheReconnectsAfterTimeout', - 'type': 'monotonic_gauge', - }, - 'DistrCacheRegistryUpdateMicroseconds': { - 'name': 'events.DistrCacheRegistryUpdateMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheRegistryUpdates': {'name': 'events.DistrCacheRegistryUpdates', 'type': 'monotonic_gauge'}, - 'DistrCacheReusedConnections': { - 'name': 'events.DistrCacheReusedConnections', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerAckRequestPackets': { - 'name': 'events.DistrCacheServerAckRequestPackets', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerCachedReadBufferCacheHits': { - 'name': 'events.DistrCacheServerCachedReadBufferCacheHits', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerCachedReadBufferCacheMisses': { - 'name': 'events.DistrCacheServerCachedReadBufferCacheMisses', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerContinueRequestPackets': { - 'name': 'events.DistrCacheServerContinueRequestPackets', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerCredentialsRefresh': { - 'name': 'events.DistrCacheServerCredentialsRefresh', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerEndRequestPackets': { - 'name': 'events.DistrCacheServerEndRequestPackets', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerNewS3CachedClients': { - 'name': 'events.DistrCacheServerNewS3CachedClients', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerProcessRequestMicroseconds': { - 'name': 'events.DistrCacheServerProcessRequestMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheServerReceivedCredentialsRefreshPackets': { - 'name': 'events.DistrCacheServerReceivedCredentialsRefreshPackets', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerReusedS3CachedClients': { - 'name': 'events.DistrCacheServerReusedS3CachedClients', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerStartRequestPackets': { - 'name': 'events.DistrCacheServerStartRequestPackets', - 'type': 'monotonic_gauge', - }, - 'DistrCacheServerSwitches': {'name': 'events.DistrCacheServerSwitches', 'type': 'monotonic_gauge'}, - 'DistrCacheServerUpdates': {'name': 'events.DistrCacheServerUpdates', 'type': 'monotonic_gauge'}, - 'DistrCacheStartRangeMicroseconds': { - 'name': 'events.DistrCacheStartRangeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'DistrCacheUnusedDataPacketsBytes': { - 'name': 'events.DistrCacheUnusedDataPacketsBytes', - 'type': 'monotonic_gauge', - }, - 'DistrCacheUnusedPackets': {'name': 'events.DistrCacheUnusedPackets', 'type': 'monotonic_gauge'}, - 'DistrCacheUnusedPacketsBufferAllocations': { - 'name': 'events.DistrCacheUnusedPacketsBufferAllocations', - 'type': 'monotonic_gauge', - }, - 'DistrCacheUnusedPacketsBytes': { - 'name': 'events.DistrCacheUnusedPacketsBytes', - 'type': 'monotonic_gauge', - }, - 'DistributedAsyncInsertionFailures': { - 'name': 'events.DistributedAsyncInsertionFailures', - 'type': 'monotonic_gauge', - }, - 'DistributedConnectionFailAtAll': { - 'name': 'events.DistributedConnectionFailAtAll', - 'type': 'monotonic_gauge', - }, - 'DistributedConnectionFailTry': { - 'name': 'events.DistributedConnectionFailTry', - 'type': 'monotonic_gauge', - }, - 'DistributedConnectionMissingTable': { - 'name': 'events.DistributedConnectionMissingTable', - 'type': 'monotonic_gauge', - }, - 'DistributedConnectionReconnectCount': { - 'name': 'events.DistributedConnectionReconnectCount', - 'type': 'monotonic_gauge', - }, - 'DistributedConnectionSkipReadOnlyReplica': { - 'name': 'events.DistributedConnectionSkipReadOnlyReplica', - 'type': 'monotonic_gauge', - }, - 'DistributedConnectionStaleReplica': { - 'name': 'events.DistributedConnectionStaleReplica', - 'type': 'monotonic_gauge', - }, - 'DistributedConnectionTries': {'name': 'events.DistributedConnectionTries', 'type': 'monotonic_gauge'}, - 'DistributedConnectionUsable': { - 'name': 'events.DistributedConnectionUsable', - 'type': 'monotonic_gauge', - }, - 'DistributedDelayedInserts': {'name': 'events.DistributedDelayedInserts', 'type': 'monotonic_gauge'}, - 'DistributedDelayedInsertsMilliseconds': { - 'name': 'events.DistributedDelayedInsertsMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'DistributedRejectedInserts': {'name': 'events.DistributedRejectedInserts', 'type': 'monotonic_gauge'}, - 'DistributedSyncInsertionTimeoutExceeded': { - 'name': 'events.DistributedSyncInsertionTimeoutExceeded', - 'type': 'monotonic_gauge', - }, - 'DuplicatedInsertedBlocks': {'name': 'events.DuplicatedInsertedBlocks', 'type': 'monotonic_gauge'}, - 'EngineFileLikeReadFiles': {'name': 'events.EngineFileLikeReadFiles', 'type': 'monotonic_gauge'}, - 'ExecuteShellCommand': {'name': 'events.ExecuteShellCommand', 'type': 'monotonic_gauge'}, - 'ExternalAggregationCompressedBytes': { - 'name': 'events.ExternalAggregationCompressedBytes', - 'type': 'monotonic_gauge', - }, - 'ExternalAggregationMerge': {'name': 'events.ExternalAggregationMerge', 'type': 'monotonic_gauge'}, - 'ExternalAggregationUncompressedBytes': { - 'name': 'events.ExternalAggregationUncompressedBytes', - 'type': 'monotonic_gauge', - }, - 'ExternalAggregationWritePart': { - 'name': 'events.ExternalAggregationWritePart', - 'type': 'monotonic_gauge', - }, - 'ExternalDataSourceLocalCacheReadBytes': { - 'name': 'events.ExternalDataSourceLocalCacheReadBytes', - 'type': 'monotonic_gauge', - }, - 'ExternalJoinCompressedBytes': { - 'name': 'events.ExternalJoinCompressedBytes', - 'type': 'monotonic_gauge', - }, - 'ExternalJoinMerge': {'name': 'events.ExternalJoinMerge', 'type': 'monotonic_gauge'}, - 'ExternalJoinUncompressedBytes': { - 'name': 'events.ExternalJoinUncompressedBytes', - 'type': 'monotonic_gauge', - }, - 'ExternalJoinWritePart': {'name': 'events.ExternalJoinWritePart', 'type': 'monotonic_gauge'}, - 'ExternalProcessingCompressedBytesTotal': { - 'name': 'events.ExternalProcessingCompressedBytesTotal', - 'type': 'monotonic_gauge', - }, - 'ExternalProcessingFilesTotal': { - 'name': 'events.ExternalProcessingFilesTotal', - 'type': 'monotonic_gauge', - }, - 'ExternalProcessingUncompressedBytesTotal': { - 'name': 'events.ExternalProcessingUncompressedBytesTotal', - 'type': 'monotonic_gauge', - }, - 'ExternalSortCompressedBytes': { - 'name': 'events.ExternalSortCompressedBytes', - 'type': 'monotonic_gauge', - }, - 'ExternalSortMerge': {'name': 'events.ExternalSortMerge', 'type': 'monotonic_gauge'}, - 'ExternalSortUncompressedBytes': { - 'name': 'events.ExternalSortUncompressedBytes', - 'type': 'monotonic_gauge', - }, - 'ExternalSortWritePart': {'name': 'events.ExternalSortWritePart', 'type': 'monotonic_gauge'}, - 'FailedAsyncInsertQuery': {'name': 'events.FailedAsyncInsertQuery', 'type': 'monotonic_gauge'}, - 'FailedInsertQuery': {'name': 'events.FailedInsertQuery', 'type': 'monotonic_gauge'}, - 'FailedQuery': {'name': 'events.FailedQuery', 'type': 'monotonic_gauge'}, - 'FailedSelectQuery': {'name': 'events.FailedSelectQuery', 'type': 'monotonic_gauge'}, - 'FetchBackgroundExecutorTaskCancelMicroseconds': { - 'name': 'events.FetchBackgroundExecutorTaskCancelMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FetchBackgroundExecutorTaskExecuteStepMicroseconds': { - 'name': 'events.FetchBackgroundExecutorTaskExecuteStepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FetchBackgroundExecutorTaskResetMicroseconds': { - 'name': 'events.FetchBackgroundExecutorTaskResetMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FetchBackgroundExecutorWaitMicroseconds': { - 'name': 'events.FetchBackgroundExecutorWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileOpen': {'name': 'events.FileOpen', 'type': 'monotonic_gauge'}, - 'FileSegmentCacheWriteMicroseconds': { - 'name': 'events.FileSegmentCacheWriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentCompleteMicroseconds': { - 'name': 'events.FileSegmentCompleteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentFailToIncreasePriority': { - 'name': 'events.FileSegmentFailToIncreasePriority', - 'type': 'monotonic_gauge', - }, - 'FileSegmentHolderCompleteMicroseconds': { - 'name': 'events.FileSegmentHolderCompleteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentLockMicroseconds': { - 'name': 'events.FileSegmentLockMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentPredownloadMicroseconds': { - 'name': 'events.FileSegmentPredownloadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentReadMicroseconds': { - 'name': 'events.FileSegmentReadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentRemoveMicroseconds': { - 'name': 'events.FileSegmentRemoveMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentUseMicroseconds': { - 'name': 'events.FileSegmentUseMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentUsedBytes': {'name': 'events.FileSegmentUsedBytes', 'type': 'monotonic_gauge'}, - 'FileSegmentWaitMicroseconds': { - 'name': 'events.FileSegmentWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentWaitReadBufferMicroseconds': { - 'name': 'events.FileSegmentWaitReadBufferMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSegmentWriteMicroseconds': { - 'name': 'events.FileSegmentWriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FileSync': {'name': 'events.FileSync', 'type': 'monotonic_gauge'}, - 'FileSyncElapsedMicroseconds': { - 'name': 'events.FileSyncElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheBackgroundDownloadQueuePush': { - 'name': 'events.FilesystemCacheBackgroundDownloadQueuePush', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheBackgroundEvictedBytes': { - 'name': 'events.FilesystemCacheBackgroundEvictedBytes', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheBackgroundEvictedFileSegments': { - 'name': 'events.FilesystemCacheBackgroundEvictedFileSegments', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheCreatedKeyDirectories': { - 'name': 'events.FilesystemCacheCreatedKeyDirectories', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheEvictMicroseconds': { - 'name': 'events.FilesystemCacheEvictMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheEvictedBytes': { - 'name': 'events.FilesystemCacheEvictedBytes', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheEvictedFileSegments': { - 'name': 'events.FilesystemCacheEvictedFileSegments', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheEvictedFileSegmentsDuringPriorityIncrease': { - 'name': 'events.FilesystemCacheEvictedFileSegmentsDuringPriorityIncrease', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheEvictionReusedIterator': { - 'name': 'events.FilesystemCacheEvictionReusedIterator', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheEvictionSkippedEvictingFileSegments': { - 'name': 'events.FilesystemCacheEvictionSkippedEvictingFileSegments', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheEvictionSkippedFileSegments': { - 'name': 'events.FilesystemCacheEvictionSkippedFileSegments', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheEvictionTries': { - 'name': 'events.FilesystemCacheEvictionTries', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheFailToReserveSpaceBecauseOfCacheResize': { - 'name': 'events.FilesystemCacheFailToReserveSpaceBecauseOfCacheResize', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheFailToReserveSpaceBecauseOfLockContention': { - 'name': 'events.FilesystemCacheFailToReserveSpaceBecauseOfLockContention', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheFailedEvictionCandidates': { - 'name': 'events.FilesystemCacheFailedEvictionCandidates', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheFreeSpaceKeepingThreadRun': { - 'name': 'events.FilesystemCacheFreeSpaceKeepingThreadRun', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheFreeSpaceKeepingThreadWorkMilliseconds': { - 'name': 'events.FilesystemCacheFreeSpaceKeepingThreadWorkMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'FilesystemCacheGetMicroseconds': { - 'name': 'events.FilesystemCacheGetMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheGetOrSetMicroseconds': { - 'name': 'events.FilesystemCacheGetOrSetMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheHoldFileSegments': { - 'name': 'events.FilesystemCacheHoldFileSegments', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheLoadMetadataMicroseconds': { - 'name': 'events.FilesystemCacheLoadMetadataMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheLockCacheMicroseconds': { - 'name': 'events.FilesystemCacheLockCacheMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheLockKeyMicroseconds': { - 'name': 'events.FilesystemCacheLockKeyMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheLockMetadataMicroseconds': { - 'name': 'events.FilesystemCacheLockMetadataMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheReserveAttempts': { - 'name': 'events.FilesystemCacheReserveAttempts', - 'type': 'monotonic_gauge', - }, - 'FilesystemCacheReserveMicroseconds': { - 'name': 'events.FilesystemCacheReserveMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilesystemCacheUnusedHoldFileSegments': { - 'name': 'events.FilesystemCacheUnusedHoldFileSegments', - 'type': 'monotonic_gauge', - }, - 'FilterTransformPassedBytes': {'name': 'events.FilterTransformPassedBytes', 'type': 'monotonic_gauge'}, - 'FilterTransformPassedRows': {'name': 'events.FilterTransformPassedRows', 'type': 'monotonic_gauge'}, - 'FilteringMarksWithPrimaryKeyMicroseconds': { - 'name': 'events.FilteringMarksWithPrimaryKeyMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FilteringMarksWithSecondaryKeysMicroseconds': { - 'name': 'events.FilteringMarksWithSecondaryKeysMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'FunctionExecute': {'name': 'events.FunctionExecute', 'type': 'monotonic_gauge'}, - 'GWPAsanAllocateFailed': {'name': 'events.GWPAsanAllocateFailed', 'type': 'monotonic_gauge'}, - 'GWPAsanAllocateSuccess': {'name': 'events.GWPAsanAllocateSuccess', 'type': 'monotonic_gauge'}, - 'GWPAsanFree': {'name': 'events.GWPAsanFree', 'type': 'monotonic_gauge'}, - 'GatheredColumns': {'name': 'events.GatheredColumns', 'type': 'monotonic_gauge'}, - 'GatheringColumnMilliseconds': { - 'name': 'events.GatheringColumnMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'GlobalThreadPoolExpansions': {'name': 'events.GlobalThreadPoolExpansions', 'type': 'monotonic_gauge'}, - 'GlobalThreadPoolJobWaitTimeMicroseconds': { - 'name': 'events.GlobalThreadPoolJobWaitTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'GlobalThreadPoolJobs': {'name': 'events.GlobalThreadPoolJobs', 'type': 'monotonic_gauge'}, - 'GlobalThreadPoolLockWaitMicroseconds': { - 'name': 'events.GlobalThreadPoolLockWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'GlobalThreadPoolShrinks': {'name': 'events.GlobalThreadPoolShrinks', 'type': 'monotonic_gauge'}, - 'GlobalThreadPoolThreadCreationMicroseconds': { - 'name': 'events.GlobalThreadPoolThreadCreationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'HTTPConnectionsCreated': {'name': 'events.HTTPConnectionsCreated', 'type': 'monotonic_gauge'}, - 'HTTPConnectionsElapsedMicroseconds': { - 'name': 'events.HTTPConnectionsElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'HTTPConnectionsErrors': {'name': 'events.HTTPConnectionsErrors', 'type': 'monotonic_gauge'}, - 'HTTPConnectionsExpired': {'name': 'events.HTTPConnectionsExpired', 'type': 'monotonic_gauge'}, - 'HTTPConnectionsPreserved': {'name': 'events.HTTPConnectionsPreserved', 'type': 'monotonic_gauge'}, - 'HTTPConnectionsReset': {'name': 'events.HTTPConnectionsReset', 'type': 'monotonic_gauge'}, - 'HTTPConnectionsReused': {'name': 'events.HTTPConnectionsReused', 'type': 'monotonic_gauge'}, - 'HTTPServerConnectionsClosed': { - 'name': 'events.HTTPServerConnectionsClosed', - 'type': 'monotonic_gauge', - }, - 'HTTPServerConnectionsCreated': { - 'name': 'events.HTTPServerConnectionsCreated', - 'type': 'monotonic_gauge', - }, - 'HTTPServerConnectionsExpired': { - 'name': 'events.HTTPServerConnectionsExpired', - 'type': 'monotonic_gauge', - }, - 'HTTPServerConnectionsPreserved': { - 'name': 'events.HTTPServerConnectionsPreserved', - 'type': 'monotonic_gauge', - }, - 'HTTPServerConnectionsReset': {'name': 'events.HTTPServerConnectionsReset', 'type': 'monotonic_gauge'}, - 'HTTPServerConnectionsReused': { - 'name': 'events.HTTPServerConnectionsReused', - 'type': 'monotonic_gauge', - }, - 'HardPageFaults': {'name': 'events.HardPageFaults', 'type': 'monotonic_gauge'}, - 'HashJoinPreallocatedElementsInHashTables': { - 'name': 'events.HashJoinPreallocatedElementsInHashTables', - 'type': 'monotonic_gauge', - }, - 'HedgedRequestsChangeReplica': { - 'name': 'events.HedgedRequestsChangeReplica', - 'type': 'monotonic_gauge', - }, - 'IOBufferAllocBytes': {'name': 'events.IOBufferAllocBytes', 'type': 'monotonic_gauge'}, - 'IOBufferAllocs': {'name': 'events.IOBufferAllocs', 'type': 'monotonic_gauge'}, - 'IOUringCQEsCompleted': {'name': 'events.IOUringCQEsCompleted', 'type': 'monotonic_gauge'}, - 'IOUringCQEsFailed': {'name': 'events.IOUringCQEsFailed', 'type': 'monotonic_gauge'}, - 'IOUringSQEsResubmitsAsync': {'name': 'events.IOUringSQEsResubmitsAsync', 'type': 'monotonic_gauge'}, - 'IOUringSQEsResubmitsSync': {'name': 'events.IOUringSQEsResubmitsSync', 'type': 'monotonic_gauge'}, - 'IOUringSQEsSubmitted': {'name': 'events.IOUringSQEsSubmitted', 'type': 'monotonic_gauge'}, - 'IcebergIteratorInitializationMicroseconds': { - 'name': 'events.IcebergIteratorInitializationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'IcebergMetadataFilesCacheHits': { - 'name': 'events.IcebergMetadataFilesCacheHits', - 'type': 'monotonic_gauge', - }, - 'IcebergMetadataFilesCacheMisses': { - 'name': 'events.IcebergMetadataFilesCacheMisses', - 'type': 'monotonic_gauge', - }, - 'IcebergMetadataFilesCacheWeightLost': { - 'name': 'events.IcebergMetadataFilesCacheWeightLost', - 'type': 'monotonic_gauge', - }, - 'IcebergMetadataReadWaitTimeMicroseconds': { - 'name': 'events.IcebergMetadataReadWaitTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'IcebergMetadataReturnedObjectInfos': { - 'name': 'events.IcebergMetadataReturnedObjectInfos', - 'type': 'monotonic_gauge', - }, - 'IcebergMetadataUpdateMicroseconds': { - 'name': 'events.IcebergMetadataUpdateMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'IcebergMinMaxIndexPrunedFiles': { - 'name': 'events.IcebergMinMaxIndexPrunedFiles', - 'type': 'monotonic_gauge', - }, - 'IcebergPartitionPrunedFiles': { - 'name': 'events.IcebergPartitionPrunedFiles', - 'type': 'monotonic_gauge', - }, - 'IcebergPartitionPrunnedFiles': { - 'name': 'events.IcebergPartitionPrunnedFiles', - 'type': 'monotonic_gauge', - }, - 'IcebergTrivialCountOptimizationApplied': { - 'name': 'events.IcebergTrivialCountOptimizationApplied', - 'type': 'monotonic_gauge', - }, - 'IcebergVersionHintUsed': {'name': 'events.IcebergVersionHintUsed', 'type': 'monotonic_gauge'}, - 'IgnoredColdParts': {'name': 'events.IgnoredColdParts', 'type': 'monotonic_gauge'}, - 'IndexBinarySearchAlgorithm': {'name': 'events.IndexBinarySearchAlgorithm', 'type': 'monotonic_gauge'}, - 'IndexGenericExclusionSearchAlgorithm': { - 'name': 'events.IndexGenericExclusionSearchAlgorithm', - 'type': 'monotonic_gauge', - }, - 'InitialQuery': {'name': 'events.InitialQuery', 'type': 'monotonic_gauge'}, - 'InsertQueriesWithSubqueries': { - 'name': 'events.InsertQueriesWithSubqueries', - 'type': 'monotonic_gauge', - }, - 'InsertQuery': {'name': 'events.InsertQuery', 'type': 'monotonic_gauge'}, - 'InsertQueryTimeMicroseconds': { - 'name': 'events.InsertQueryTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'InsertedBytes': {'name': 'events.InsertedBytes', 'type': 'monotonic_gauge'}, - 'InsertedCompactParts': {'name': 'events.InsertedCompactParts', 'type': 'monotonic_gauge'}, - 'InsertedRows': {'name': 'events.InsertedRows', 'type': 'monotonic_gauge'}, - 'InsertedWideParts': {'name': 'events.InsertedWideParts', 'type': 'monotonic_gauge'}, - 'InterfaceHTTPReceiveBytes': {'name': 'events.InterfaceHTTPReceiveBytes', 'type': 'monotonic_gauge'}, - 'InterfaceHTTPSendBytes': {'name': 'events.InterfaceHTTPSendBytes', 'type': 'monotonic_gauge'}, - 'InterfaceInterserverReceiveBytes': { - 'name': 'events.InterfaceInterserverReceiveBytes', - 'type': 'monotonic_gauge', - }, - 'InterfaceInterserverSendBytes': { - 'name': 'events.InterfaceInterserverSendBytes', - 'type': 'monotonic_gauge', - }, - 'InterfaceMySQLReceiveBytes': {'name': 'events.InterfaceMySQLReceiveBytes', 'type': 'monotonic_gauge'}, - 'InterfaceMySQLSendBytes': {'name': 'events.InterfaceMySQLSendBytes', 'type': 'monotonic_gauge'}, - 'InterfaceNativeReceiveBytes': { - 'name': 'events.InterfaceNativeReceiveBytes', - 'type': 'monotonic_gauge', - }, - 'InterfaceNativeSendBytes': {'name': 'events.InterfaceNativeSendBytes', 'type': 'monotonic_gauge'}, - 'InterfacePostgreSQLReceiveBytes': { - 'name': 'events.InterfacePostgreSQLReceiveBytes', - 'type': 'monotonic_gauge', - }, - 'InterfacePostgreSQLSendBytes': { - 'name': 'events.InterfacePostgreSQLSendBytes', - 'type': 'monotonic_gauge', - }, - 'InterfacePrometheusReceiveBytes': { - 'name': 'events.InterfacePrometheusReceiveBytes', - 'type': 'monotonic_gauge', - }, - 'InterfacePrometheusSendBytes': { - 'name': 'events.InterfacePrometheusSendBytes', - 'type': 'monotonic_gauge', - }, - 'JoinBuildTableRowCount': {'name': 'events.JoinBuildTableRowCount', 'type': 'monotonic_gauge'}, - 'JoinProbeTableRowCount': {'name': 'events.JoinProbeTableRowCount', 'type': 'monotonic_gauge'}, - 'JoinResultRowCount': {'name': 'events.JoinResultRowCount', 'type': 'monotonic_gauge'}, - 'KafkaBackgroundReads': {'name': 'events.KafkaBackgroundReads', 'type': 'monotonic_gauge'}, - 'KafkaCommitFailures': {'name': 'events.KafkaCommitFailures', 'type': 'monotonic_gauge'}, - 'KafkaCommits': {'name': 'events.KafkaCommits', 'type': 'monotonic_gauge'}, - 'KafkaConsumerErrors': {'name': 'events.KafkaConsumerErrors', 'type': 'monotonic_gauge'}, - 'KafkaDirectReads': {'name': 'events.KafkaDirectReads', 'type': 'monotonic_gauge'}, - 'KafkaMessagesFailed': {'name': 'events.KafkaMessagesFailed', 'type': 'monotonic_gauge'}, - 'KafkaMessagesPolled': {'name': 'events.KafkaMessagesPolled', 'type': 'monotonic_gauge'}, - 'KafkaMessagesProduced': {'name': 'events.KafkaMessagesProduced', 'type': 'monotonic_gauge'}, - 'KafkaMessagesRead': {'name': 'events.KafkaMessagesRead', 'type': 'monotonic_gauge'}, - 'KafkaProducerErrors': {'name': 'events.KafkaProducerErrors', 'type': 'monotonic_gauge'}, - 'KafkaProducerFlushes': {'name': 'events.KafkaProducerFlushes', 'type': 'monotonic_gauge'}, - 'KafkaRebalanceAssignments': {'name': 'events.KafkaRebalanceAssignments', 'type': 'monotonic_gauge'}, - 'KafkaRebalanceErrors': {'name': 'events.KafkaRebalanceErrors', 'type': 'monotonic_gauge'}, - 'KafkaRebalanceRevocations': {'name': 'events.KafkaRebalanceRevocations', 'type': 'monotonic_gauge'}, - 'KafkaRowsRead': {'name': 'events.KafkaRowsRead', 'type': 'monotonic_gauge'}, - 'KafkaRowsRejected': {'name': 'events.KafkaRowsRejected', 'type': 'monotonic_gauge'}, - 'KafkaRowsWritten': {'name': 'events.KafkaRowsWritten', 'type': 'monotonic_gauge'}, - 'KafkaWrites': {'name': 'events.KafkaWrites', 'type': 'monotonic_gauge'}, - 'KeeperBatchMaxCount': {'name': 'events.KeeperBatchMaxCount', 'type': 'monotonic_gauge'}, - 'KeeperBatchMaxTotalSize': {'name': 'events.KeeperBatchMaxTotalSize', 'type': 'monotonic_gauge'}, - 'KeeperCheckRequest': {'name': 'events.KeeperCheckRequest', 'type': 'monotonic_gauge'}, - 'KeeperCommitWaitElapsedMicroseconds': { - 'name': 'events.KeeperCommitWaitElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'KeeperCommits': {'name': 'events.KeeperCommits', 'type': 'monotonic_gauge'}, - 'KeeperCommitsFailed': {'name': 'events.KeeperCommitsFailed', 'type': 'monotonic_gauge'}, - 'KeeperCreateRequest': {'name': 'events.KeeperCreateRequest', 'type': 'monotonic_gauge'}, - 'KeeperExistsRequest': {'name': 'events.KeeperExistsRequest', 'type': 'monotonic_gauge'}, - 'KeeperGetRequest': {'name': 'events.KeeperGetRequest', 'type': 'monotonic_gauge'}, - 'KeeperLatency': {'name': 'events.KeeperLatency', 'type': 'temporal_percent', 'scale': 'millisecond'}, - 'KeeperListRequest': {'name': 'events.KeeperListRequest', 'type': 'monotonic_gauge'}, - 'KeeperLogsEntryReadFromCommitCache': { - 'name': 'events.KeeperLogsEntryReadFromCommitCache', - 'type': 'monotonic_gauge', - }, - 'KeeperLogsEntryReadFromFile': { - 'name': 'events.KeeperLogsEntryReadFromFile', - 'type': 'monotonic_gauge', - }, - 'KeeperLogsEntryReadFromLatestCache': { - 'name': 'events.KeeperLogsEntryReadFromLatestCache', - 'type': 'monotonic_gauge', - }, - 'KeeperLogsPrefetchedEntries': { - 'name': 'events.KeeperLogsPrefetchedEntries', - 'type': 'monotonic_gauge', - }, - 'KeeperMultiReadRequest': {'name': 'events.KeeperMultiReadRequest', 'type': 'monotonic_gauge'}, - 'KeeperMultiRequest': {'name': 'events.KeeperMultiRequest', 'type': 'monotonic_gauge'}, - 'KeeperPacketsReceived': {'name': 'events.KeeperPacketsReceived', 'type': 'monotonic_gauge'}, - 'KeeperPacketsSent': {'name': 'events.KeeperPacketsSent', 'type': 'monotonic_gauge'}, - 'KeeperPreprocessElapsedMicroseconds': { - 'name': 'events.KeeperPreprocessElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'KeeperProcessElapsedMicroseconds': { - 'name': 'events.KeeperProcessElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'KeeperReadSnapshot': {'name': 'events.KeeperReadSnapshot', 'type': 'monotonic_gauge'}, - 'KeeperReconfigRequest': {'name': 'events.KeeperReconfigRequest', 'type': 'monotonic_gauge'}, - 'KeeperRemoveRequest': {'name': 'events.KeeperRemoveRequest', 'type': 'monotonic_gauge'}, - 'KeeperRequestRejectedDueToSoftMemoryLimitCount': { - 'name': 'events.KeeperRequestRejectedDueToSoftMemoryLimitCount', - 'type': 'monotonic_gauge', - }, - 'KeeperRequestTotal': {'name': 'events.KeeperRequestTotal', 'type': 'monotonic_gauge'}, - 'KeeperSaveSnapshot': {'name': 'events.KeeperSaveSnapshot', 'type': 'monotonic_gauge'}, - 'KeeperSetRequest': {'name': 'events.KeeperSetRequest', 'type': 'monotonic_gauge'}, - 'KeeperSnapshotApplys': {'name': 'events.KeeperSnapshotApplys', 'type': 'monotonic_gauge'}, - 'KeeperSnapshotApplysFailed': {'name': 'events.KeeperSnapshotApplysFailed', 'type': 'monotonic_gauge'}, - 'KeeperSnapshotCreations': {'name': 'events.KeeperSnapshotCreations', 'type': 'monotonic_gauge'}, - 'KeeperSnapshotCreationsFailed': { - 'name': 'events.KeeperSnapshotCreationsFailed', - 'type': 'monotonic_gauge', - }, - 'KeeperStorageLockWaitMicroseconds': { - 'name': 'events.KeeperStorageLockWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'KeeperTotalElapsedMicroseconds': { - 'name': 'events.KeeperTotalElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LoadedDataParts': {'name': 'events.LoadedDataParts', 'type': 'monotonic_gauge'}, - 'LoadedDataPartsMicroseconds': { - 'name': 'events.LoadedDataPartsMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LoadedMarksCount': {'name': 'events.LoadedMarksCount', 'type': 'monotonic_gauge'}, - 'LoadedMarksFiles': {'name': 'events.LoadedMarksFiles', 'type': 'monotonic_gauge'}, - 'LoadedMarksMemoryBytes': {'name': 'events.LoadedMarksMemoryBytes', 'type': 'monotonic_gauge'}, - 'LoadedPrimaryIndexBytes': {'name': 'events.LoadedPrimaryIndexBytes', 'type': 'monotonic_gauge'}, - 'LoadedPrimaryIndexFiles': {'name': 'events.LoadedPrimaryIndexFiles', 'type': 'monotonic_gauge'}, - 'LoadedPrimaryIndexRows': {'name': 'events.LoadedPrimaryIndexRows', 'type': 'monotonic_gauge'}, - 'LoadingMarksTasksCanceled': {'name': 'events.LoadingMarksTasksCanceled', 'type': 'monotonic_gauge'}, - 'LocalReadThrottlerBytes': {'name': 'events.LocalReadThrottlerBytes', 'type': 'monotonic_gauge'}, - 'LocalReadThrottlerSleepMicroseconds': { - 'name': 'events.LocalReadThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LocalThreadPoolBusyMicroseconds': { - 'name': 'events.LocalThreadPoolBusyMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LocalThreadPoolExpansions': {'name': 'events.LocalThreadPoolExpansions', 'type': 'monotonic_gauge'}, - 'LocalThreadPoolJobWaitTimeMicroseconds': { - 'name': 'events.LocalThreadPoolJobWaitTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LocalThreadPoolJobs': { - 'name': 'events.LocalThreadPoolJobs', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LocalThreadPoolLockWaitMicroseconds': { - 'name': 'events.LocalThreadPoolLockWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LocalThreadPoolShrinks': {'name': 'events.LocalThreadPoolShrinks', 'type': 'monotonic_gauge'}, - 'LocalThreadPoolThreadCreationMicroseconds': { - 'name': 'events.LocalThreadPoolThreadCreationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LocalWriteThrottlerBytes': {'name': 'events.LocalWriteThrottlerBytes', 'type': 'monotonic_gauge'}, - 'LocalWriteThrottlerSleepMicroseconds': { - 'name': 'events.LocalWriteThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'LogDebug': {'name': 'events.LogDebug', 'type': 'monotonic_gauge'}, - 'LogError': {'name': 'events.LogError', 'type': 'monotonic_gauge'}, - 'LogFatal': {'name': 'events.LogFatal', 'type': 'monotonic_gauge'}, - 'LogInfo': {'name': 'events.LogInfo', 'type': 'monotonic_gauge'}, - 'LogTest': {'name': 'events.LogTest', 'type': 'monotonic_gauge'}, - 'LogTrace': {'name': 'events.LogTrace', 'type': 'monotonic_gauge'}, - 'LogWarning': {'name': 'events.LogWarning', 'type': 'monotonic_gauge'}, - 'LoggerElapsedNanoseconds': { - 'name': 'events.LoggerElapsedNanoseconds', - 'type': 'temporal_percent', - 'scale': 'nanosecond', - }, - 'MMappedFileCacheHits': {'name': 'events.MMappedFileCacheHits', 'type': 'monotonic_gauge'}, - 'MMappedFileCacheMisses': {'name': 'events.MMappedFileCacheMisses', 'type': 'monotonic_gauge'}, - 'MainConfigLoads': {'name': 'events.MainConfigLoads', 'type': 'monotonic_gauge'}, - 'MarkCacheEvictedBytes': {'name': 'events.MarkCacheEvictedBytes', 'type': 'monotonic_gauge'}, - 'MarkCacheEvictedFiles': {'name': 'events.MarkCacheEvictedFiles', 'type': 'monotonic_gauge'}, - 'MarkCacheEvictedMarks': {'name': 'events.MarkCacheEvictedMarks', 'type': 'monotonic_gauge'}, - 'MarkCacheHits': {'name': 'events.MarkCacheHits', 'type': 'monotonic_gauge'}, - 'MarkCacheMisses': {'name': 'events.MarkCacheMisses', 'type': 'monotonic_gauge'}, - 'MemoryAllocatorPurge': {'name': 'events.MemoryAllocatorPurge', 'type': 'monotonic_gauge'}, - 'MemoryAllocatorPurgeTimeMicroseconds': { - 'name': 'events.MemoryAllocatorPurgeTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MemoryOvercommitWaitTimeMicroseconds': { - 'name': 'events.MemoryOvercommitWaitTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MemoryWorkerRun': {'name': 'events.MemoryWorkerRun', 'type': 'monotonic_gauge'}, - 'MemoryWorkerRunElapsedMicroseconds': { - 'name': 'events.MemoryWorkerRunElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'Merge': {'name': 'events.Merge', 'type': 'monotonic_gauge'}, - 'MergeExecuteMilliseconds': { - 'name': 'events.MergeExecuteMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeHorizontalStageExecuteMilliseconds': { - 'name': 'events.MergeHorizontalStageExecuteMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeHorizontalStageTotalMilliseconds': { - 'name': 'events.MergeHorizontalStageTotalMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeMutateBackgroundExecutorTaskCancelMicroseconds': { - 'name': 'events.MergeMutateBackgroundExecutorTaskCancelMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeMutateBackgroundExecutorTaskExecuteStepMicroseconds': { - 'name': 'events.MergeMutateBackgroundExecutorTaskExecuteStepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeMutateBackgroundExecutorTaskResetMicroseconds': { - 'name': 'events.MergeMutateBackgroundExecutorTaskResetMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeMutateBackgroundExecutorWaitMicroseconds': { - 'name': 'events.MergeMutateBackgroundExecutorWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergePrewarmStageExecuteMilliseconds': { - 'name': 'events.MergePrewarmStageExecuteMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergePrewarmStageTotalMilliseconds': { - 'name': 'events.MergePrewarmStageTotalMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeProjectionStageExecuteMilliseconds': { - 'name': 'events.MergeProjectionStageExecuteMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeProjectionStageTotalMilliseconds': { - 'name': 'events.MergeProjectionStageTotalMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeSourceParts': {'name': 'events.MergeSourceParts', 'type': 'monotonic_gauge'}, - 'MergeTotalMilliseconds': { - 'name': 'events.MergeTotalMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeTreeAllRangesAnnouncementsSent': { - 'name': 'events.MergeTreeAllRangesAnnouncementsSent', - 'type': 'monotonic_gauge', - }, - 'MergeTreeAllRangesAnnouncementsSentElapsedMicroseconds': { - 'name': 'events.MergeTreeAllRangesAnnouncementsSentElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataProjectionWriterBlocks': { - 'name': 'events.MergeTreeDataProjectionWriterBlocks', - 'type': 'monotonic_gauge', - }, - 'MergeTreeDataProjectionWriterBlocksAlreadySorted': { - 'name': 'events.MergeTreeDataProjectionWriterBlocksAlreadySorted', - 'type': 'monotonic_gauge', - }, - 'MergeTreeDataProjectionWriterCompressedBytes': { - 'name': 'events.MergeTreeDataProjectionWriterCompressedBytes', - 'type': 'monotonic_gauge', - }, - 'MergeTreeDataProjectionWriterMergingBlocksMicroseconds': { - 'name': 'events.MergeTreeDataProjectionWriterMergingBlocksMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataProjectionWriterRows': { - 'name': 'events.MergeTreeDataProjectionWriterRows', - 'type': 'monotonic_gauge', - }, - 'MergeTreeDataProjectionWriterSortingBlocksMicroseconds': { - 'name': 'events.MergeTreeDataProjectionWriterSortingBlocksMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataProjectionWriterUncompressedBytes': { - 'name': 'events.MergeTreeDataProjectionWriterUncompressedBytes', - 'type': 'monotonic_gauge', - }, - 'MergeTreeDataWriterBlocks': {'name': 'events.MergeTreeDataWriterBlocks', 'type': 'monotonic_gauge'}, - 'MergeTreeDataWriterBlocksAlreadySorted': { - 'name': 'events.MergeTreeDataWriterBlocksAlreadySorted', - 'type': 'monotonic_gauge', - }, - 'MergeTreeDataWriterCompressedBytes': { - 'name': 'events.MergeTreeDataWriterCompressedBytes', - 'type': 'monotonic_gauge', - }, - 'MergeTreeDataWriterMergingBlocksMicroseconds': { - 'name': 'events.MergeTreeDataWriterMergingBlocksMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataWriterProjectionsCalculationMicroseconds': { - 'name': 'events.MergeTreeDataWriterProjectionsCalculationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataWriterRows': {'name': 'events.MergeTreeDataWriterRows', 'type': 'monotonic_gauge'}, - 'MergeTreeDataWriterSkipIndicesCalculationMicroseconds': { - 'name': 'events.MergeTreeDataWriterSkipIndicesCalculationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataWriterSortingBlocksMicroseconds': { - 'name': 'events.MergeTreeDataWriterSortingBlocksMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataWriterStatisticsCalculationMicroseconds': { - 'name': 'events.MergeTreeDataWriterStatisticsCalculationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeDataWriterUncompressedBytes': { - 'name': 'events.MergeTreeDataWriterUncompressedBytes', - 'type': 'monotonic_gauge', - }, - 'MergeTreePrefetchedReadPoolInit': { - 'name': 'events.MergeTreePrefetchedReadPoolInit', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeTreeReadTaskRequestsReceived': { - 'name': 'events.MergeTreeReadTaskRequestsReceived', - 'type': 'monotonic_gauge', - }, - 'MergeTreeReadTaskRequestsSent': { - 'name': 'events.MergeTreeReadTaskRequestsSent', - 'type': 'monotonic_gauge', - }, - 'MergeTreeReadTaskRequestsSentElapsedMicroseconds': { - 'name': 'events.MergeTreeReadTaskRequestsSentElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergeVerticalStageExecuteMilliseconds': { - 'name': 'events.MergeVerticalStageExecuteMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergeVerticalStageTotalMilliseconds': { - 'name': 'events.MergeVerticalStageTotalMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MergedColumns': {'name': 'events.MergedColumns', 'type': 'monotonic_gauge'}, - 'MergedIntoCompactParts': {'name': 'events.MergedIntoCompactParts', 'type': 'monotonic_gauge'}, - 'MergedIntoWideParts': {'name': 'events.MergedIntoWideParts', 'type': 'monotonic_gauge'}, - 'MergedRows': {'name': 'events.MergedRows', 'type': 'monotonic_gauge'}, - 'MergedUncompressedBytes': {'name': 'events.MergedUncompressedBytes', 'type': 'monotonic_gauge'}, - 'MergerMutatorPartsInRangesForMergeCount': { - 'name': 'events.MergerMutatorPartsInRangesForMergeCount', - 'type': 'monotonic_gauge', - }, - 'MergerMutatorPrepareRangesForMergeElapsedMicroseconds': { - 'name': 'events.MergerMutatorPrepareRangesForMergeElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergerMutatorRangesForMergeCount': { - 'name': 'events.MergerMutatorRangesForMergeCount', - 'type': 'monotonic_gauge', - }, - 'MergerMutatorSelectPartsForMergeElapsedMicroseconds': { - 'name': 'events.MergerMutatorSelectPartsForMergeElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergerMutatorSelectRangePartsCount': { - 'name': 'events.MergerMutatorSelectRangePartsCount', - 'type': 'monotonic_gauge', - }, - 'MergerMutatorsGetPartsForMergeElapsedMicroseconds': { - 'name': 'events.MergerMutatorsGetPartsForMergeElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergesThrottlerBytes': {'name': 'events.MergesThrottlerBytes', 'type': 'monotonic_gauge'}, - 'MergesThrottlerSleepMicroseconds': { - 'name': 'events.MergesThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MergingSortedMilliseconds': { - 'name': 'events.MergingSortedMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MetadataFromKeeperBackgroundCleanupErrors': { - 'name': 'events.MetadataFromKeeperBackgroundCleanupErrors', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperBackgroundCleanupObjects': { - 'name': 'events.MetadataFromKeeperBackgroundCleanupObjects', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperBackgroundCleanupTransactions': { - 'name': 'events.MetadataFromKeeperBackgroundCleanupTransactions', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperCacheHit': {'name': 'events.MetadataFromKeeperCacheHit', 'type': 'monotonic_gauge'}, - 'MetadataFromKeeperCacheMiss': { - 'name': 'events.MetadataFromKeeperCacheMiss', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperCacheUpdateMicroseconds': { - 'name': 'events.MetadataFromKeeperCacheUpdateMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MetadataFromKeeperCleanupTransactionCommit': { - 'name': 'events.MetadataFromKeeperCleanupTransactionCommit', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperCleanupTransactionCommitRetry': { - 'name': 'events.MetadataFromKeeperCleanupTransactionCommitRetry', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperIndividualOperations': { - 'name': 'events.MetadataFromKeeperIndividualOperations', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperIndividualOperationsMicroseconds': { - 'name': 'events.MetadataFromKeeperIndividualOperationsMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MetadataFromKeeperOperations': { - 'name': 'events.MetadataFromKeeperOperations', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperReconnects': { - 'name': 'events.MetadataFromKeeperReconnects', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperTransactionCommit': { - 'name': 'events.MetadataFromKeeperTransactionCommit', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperTransactionCommitRetry': { - 'name': 'events.MetadataFromKeeperTransactionCommitRetry', - 'type': 'monotonic_gauge', - }, - 'MetadataFromKeeperUpdateCacheOneLevel': { - 'name': 'events.MetadataFromKeeperUpdateCacheOneLevel', - 'type': 'monotonic_gauge', - }, - 'MoveBackgroundExecutorTaskCancelMicroseconds': { - 'name': 'events.MoveBackgroundExecutorTaskCancelMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MoveBackgroundExecutorTaskExecuteStepMicroseconds': { - 'name': 'events.MoveBackgroundExecutorTaskExecuteStepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MoveBackgroundExecutorTaskResetMicroseconds': { - 'name': 'events.MoveBackgroundExecutorTaskResetMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MoveBackgroundExecutorWaitMicroseconds': { - 'name': 'events.MoveBackgroundExecutorWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MutateTaskProjectionsCalculationMicroseconds': { - 'name': 'events.MutateTaskProjectionsCalculationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'MutatedRows': {'name': 'events.MutatedRows', 'type': 'monotonic_gauge'}, - 'MutatedUncompressedBytes': {'name': 'events.MutatedUncompressedBytes', 'type': 'monotonic_gauge'}, - 'MutationAffectedRowsUpperBound': { - 'name': 'events.MutationAffectedRowsUpperBound', - 'type': 'monotonic_gauge', - }, - 'MutationAllPartColumns': {'name': 'events.MutationAllPartColumns', 'type': 'monotonic_gauge'}, - 'MutationCreatedEmptyParts': {'name': 'events.MutationCreatedEmptyParts', 'type': 'monotonic_gauge'}, - 'MutationExecuteMilliseconds': { - 'name': 'events.MutationExecuteMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MutationSomePartColumns': {'name': 'events.MutationSomePartColumns', 'type': 'monotonic_gauge'}, - 'MutationTotalMilliseconds': { - 'name': 'events.MutationTotalMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'MutationTotalParts': {'name': 'events.MutationTotalParts', 'type': 'monotonic_gauge'}, - 'MutationUntouchedParts': {'name': 'events.MutationUntouchedParts', 'type': 'monotonic_gauge'}, - 'MutationsAppliedOnFlyInAllParts': {'name': 'events.MutationsAppliedOnFlyInAllParts', 'type': 'gauge'}, - 'MutationsAppliedOnFlyInAllReadTasks': { - 'name': 'events.MutationsAppliedOnFlyInAllReadTasks', - 'type': 'monotonic_gauge', - }, - 'MutationsThrottlerBytes': {'name': 'events.MutationsThrottlerBytes', 'type': 'monotonic_gauge'}, - 'MutationsThrottlerSleepMicroseconds': { - 'name': 'events.MutationsThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'NetworkReceiveBytes': {'name': 'events.NetworkReceiveBytes', 'type': 'monotonic_gauge'}, - 'NetworkReceiveElapsedMicroseconds': { - 'name': 'events.NetworkReceiveElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'NetworkSendBytes': {'name': 'events.NetworkSendBytes', 'type': 'monotonic_gauge'}, - 'NetworkSendElapsedMicroseconds': { - 'name': 'events.NetworkSendElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'NotCreatedLogEntryForMerge': {'name': 'events.NotCreatedLogEntryForMerge', 'type': 'monotonic_gauge'}, - 'NotCreatedLogEntryForMutation': { - 'name': 'events.NotCreatedLogEntryForMutation', - 'type': 'monotonic_gauge', - }, - 'OSCPUVirtualTimeMicroseconds': { - 'name': 'events.OSCPUVirtualTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'OSCPUWaitMicroseconds': { - 'name': 'events.OSCPUWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'OSIOWaitMicroseconds': { - 'name': 'events.OSIOWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'OSReadBytes': {'name': 'events.OSReadBytes', 'type': 'monotonic_gauge'}, - 'OSReadChars': {'name': 'events.OSReadChars', 'type': 'monotonic_gauge'}, - 'OSWriteBytes': {'name': 'events.OSWriteBytes', 'type': 'monotonic_gauge'}, - 'OSWriteChars': {'name': 'events.OSWriteChars', 'type': 'monotonic_gauge'}, - 'ObjectStorageQueueCancelledFiles': { - 'name': 'events.ObjectStorageQueueCancelledFiles', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueCleanupMaxSetSizeOrTTLMicroseconds': { - 'name': 'events.ObjectStorageQueueCleanupMaxSetSizeOrTTLMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ObjectStorageQueueCommitRequests': { - 'name': 'events.ObjectStorageQueueCommitRequests', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueExceptionsDuringInsert': { - 'name': 'events.ObjectStorageQueueExceptionsDuringInsert', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueExceptionsDuringRead': { - 'name': 'events.ObjectStorageQueueExceptionsDuringRead', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueFailedFiles': { - 'name': 'events.ObjectStorageQueueFailedFiles', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueFailedToBatchSetProcessing': { - 'name': 'events.ObjectStorageQueueFailedToBatchSetProcessing', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueFilteredFiles': { - 'name': 'events.ObjectStorageQueueFilteredFiles', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueInsertIterations': { - 'name': 'events.ObjectStorageQueueInsertIterations', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueListedFiles': { - 'name': 'events.ObjectStorageQueueListedFiles', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueLockLocalFileStatusesMicroseconds': { - 'name': 'events.ObjectStorageQueueLockLocalFileStatusesMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ObjectStorageQueueProcessedFiles': { - 'name': 'events.ObjectStorageQueueProcessedFiles', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueProcessedRows': { - 'name': 'events.ObjectStorageQueueProcessedRows', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueuePullMicroseconds': { - 'name': 'events.ObjectStorageQueuePullMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ObjectStorageQueueReadBytes': { - 'name': 'events.ObjectStorageQueueReadBytes', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueReadFiles': { - 'name': 'events.ObjectStorageQueueReadFiles', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueReadRows': {'name': 'events.ObjectStorageQueueReadRows', 'type': 'monotonic_gauge'}, - 'ObjectStorageQueueRemovedObjects': { - 'name': 'events.ObjectStorageQueueRemovedObjects', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueSuccessfulCommits': { - 'name': 'events.ObjectStorageQueueSuccessfulCommits', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueTrySetProcessingFailed': { - 'name': 'events.ObjectStorageQueueTrySetProcessingFailed', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueTrySetProcessingRequests': { - 'name': 'events.ObjectStorageQueueTrySetProcessingRequests', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueTrySetProcessingSucceeded': { - 'name': 'events.ObjectStorageQueueTrySetProcessingSucceeded', - 'type': 'monotonic_gauge', - }, - 'ObjectStorageQueueUnsuccessfulCommits': { - 'name': 'events.ObjectStorageQueueUnsuccessfulCommits', - 'type': 'monotonic_gauge', - }, - 'ObsoleteReplicatedParts': {'name': 'events.ObsoleteReplicatedParts', 'type': 'monotonic_gauge'}, - 'OpenedFileCacheHits': {'name': 'events.OpenedFileCacheHits', 'type': 'monotonic_gauge'}, - 'OpenedFileCacheMicroseconds': { - 'name': 'events.OpenedFileCacheMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'OpenedFileCacheMisses': {'name': 'events.OpenedFileCacheMisses', 'type': 'monotonic_gauge'}, - 'OtherQueryTimeMicroseconds': { - 'name': 'events.OtherQueryTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'OverflowAny': {'name': 'events.OverflowAny', 'type': 'monotonic_gauge'}, - 'OverflowBreak': {'name': 'events.OverflowBreak', 'type': 'monotonic_gauge'}, - 'OverflowThrow': {'name': 'events.OverflowThrow', 'type': 'monotonic_gauge'}, - 'PageCacheBytesUnpinnedRoundedToHugePages': { - 'name': 'events.PageCacheBytesUnpinnedRoundedToHugePages', - 'type': 'gauge', - }, - 'PageCacheBytesUnpinnedRoundedToPages': { - 'name': 'events.PageCacheBytesUnpinnedRoundedToPages', - 'type': 'gauge', - }, - 'PageCacheChunkDataHits': {'name': 'events.PageCacheChunkDataHits', 'type': 'gauge'}, - 'PageCacheChunkDataMisses': {'name': 'events.PageCacheChunkDataMisses', 'type': 'gauge'}, - 'PageCacheChunkDataPartialHits': {'name': 'events.PageCacheChunkDataPartialHits', 'type': 'gauge'}, - 'PageCacheChunkMisses': {'name': 'events.PageCacheChunkMisses', 'type': 'gauge'}, - 'PageCacheChunkShared': {'name': 'events.PageCacheChunkShared', 'type': 'gauge'}, - 'PageCacheHits': {'name': 'events.PageCacheHits', 'type': 'monotonic_gauge'}, - 'PageCacheMisses': {'name': 'events.PageCacheMisses', 'type': 'monotonic_gauge'}, - 'PageCacheOvercommitResize': {'name': 'events.PageCacheOvercommitResize', 'type': 'monotonic_gauge'}, - 'PageCacheReadBytes': {'name': 'events.PageCacheReadBytes', 'type': 'monotonic_gauge'}, - 'PageCacheResized': {'name': 'events.PageCacheResized', 'type': 'monotonic_gauge'}, - 'PageCacheWeightLost': {'name': 'events.PageCacheWeightLost', 'type': 'monotonic_gauge'}, - 'ParallelReplicasAnnouncementMicroseconds': { - 'name': 'events.ParallelReplicasAnnouncementMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasAvailableCount': { - 'name': 'events.ParallelReplicasAvailableCount', - 'type': 'monotonic_gauge', - }, - 'ParallelReplicasCollectingOwnedSegmentsMicroseconds': { - 'name': 'events.ParallelReplicasCollectingOwnedSegmentsMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasDeniedRequests': { - 'name': 'events.ParallelReplicasDeniedRequests', - 'type': 'monotonic_gauge', - }, - 'ParallelReplicasHandleAnnouncementMicroseconds': { - 'name': 'events.ParallelReplicasHandleAnnouncementMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasHandleRequestMicroseconds': { - 'name': 'events.ParallelReplicasHandleRequestMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasNumRequests': { - 'name': 'events.ParallelReplicasNumRequests', - 'type': 'monotonic_gauge', - }, - 'ParallelReplicasProcessingPartsMicroseconds': { - 'name': 'events.ParallelReplicasProcessingPartsMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasQueryCount': {'name': 'events.ParallelReplicasQueryCount', 'type': 'monotonic_gauge'}, - 'ParallelReplicasReadAssignedForStealingMarks': { - 'name': 'events.ParallelReplicasReadAssignedForStealingMarks', - 'type': 'monotonic_gauge', - }, - 'ParallelReplicasReadAssignedMarks': { - 'name': 'events.ParallelReplicasReadAssignedMarks', - 'type': 'monotonic_gauge', - }, - 'ParallelReplicasReadMarks': {'name': 'events.ParallelReplicasReadMarks', 'type': 'monotonic_gauge'}, - 'ParallelReplicasReadRequestMicroseconds': { - 'name': 'events.ParallelReplicasReadRequestMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasReadUnassignedMarks': { - 'name': 'events.ParallelReplicasReadUnassignedMarks', - 'type': 'monotonic_gauge', - }, - 'ParallelReplicasStealingByHashMicroseconds': { - 'name': 'events.ParallelReplicasStealingByHashMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasStealingLeftoversMicroseconds': { - 'name': 'events.ParallelReplicasStealingLeftoversMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParallelReplicasUnavailableCount': { - 'name': 'events.ParallelReplicasUnavailableCount', - 'type': 'monotonic_gauge', - }, - 'ParallelReplicasUsedCount': {'name': 'events.ParallelReplicasUsedCount', 'type': 'monotonic_gauge'}, - 'ParquetDecodingTaskBatches': {'name': 'events.ParquetDecodingTaskBatches', 'type': 'monotonic_gauge'}, - 'ParquetDecodingTasks': {'name': 'events.ParquetDecodingTasks', 'type': 'monotonic_gauge'}, - 'ParquetFetchWaitTimeMicroseconds': { - 'name': 'events.ParquetFetchWaitTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ParquetPrunedRowGroups': {'name': 'events.ParquetPrunedRowGroups', 'type': 'monotonic_gauge'}, - 'ParquetReadRowGroups': {'name': 'events.ParquetReadRowGroups', 'type': 'monotonic_gauge'}, - 'PartsLockHoldMicroseconds': { - 'name': 'events.PartsLockHoldMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'PartsLockWaitMicroseconds': { - 'name': 'events.PartsLockWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'PartsWithAppliedMutationsOnFly': {'name': 'events.PartsWithAppliedMutationsOnFly', 'type': 'gauge'}, - 'PatchesAcquireLockMicroseconds': { - 'name': 'events.PatchesAcquireLockMicroseconds', - 'type': 'monotonic_gauge', - }, - 'PatchesAcquireLockTries': {'name': 'events.PatchesAcquireLockTries', 'type': 'monotonic_gauge'}, - 'PatchesAppliedInAllReadTasks': { - 'name': 'events.PatchesAppliedInAllReadTasks', - 'type': 'monotonic_gauge', - }, - 'PatchesJoinAppliedInAllReadTasks': { - 'name': 'events.PatchesJoinAppliedInAllReadTasks', - 'type': 'monotonic_gauge', - }, - 'PatchesMergeAppliedInAllReadTasks': { - 'name': 'events.PatchesMergeAppliedInAllReadTasks', - 'type': 'monotonic_gauge', - }, - 'PatchesReadUncompressedBytes': { - 'name': 'events.PatchesReadUncompressedBytes', - 'type': 'monotonic_gauge', - }, - 'PerfAlignmentFaults': {'name': 'events.PerfAlignmentFaults', 'type': 'monotonic_gauge'}, - 'PerfBranchInstructions': {'name': 'events.PerfBranchInstructions', 'type': 'monotonic_gauge'}, - 'PerfBranchMisses': {'name': 'events.PerfBranchMisses', 'type': 'monotonic_gauge'}, - 'PerfBusCycles': {'name': 'events.PerfBusCycles', 'type': 'monotonic_gauge'}, - 'PerfCPUClock': {'name': 'events.PerfCPUClock', 'type': 'monotonic_gauge'}, - 'PerfCPUCycles': {'name': 'events.PerfCPUCycles', 'type': 'monotonic_gauge'}, - 'PerfCPUMigrations': {'name': 'events.PerfCPUMigrations', 'type': 'monotonic_gauge'}, - 'PerfCacheMisses': {'name': 'events.PerfCacheMisses', 'type': 'monotonic_gauge'}, - 'PerfCacheReferences': {'name': 'events.PerfCacheReferences', 'type': 'monotonic_gauge'}, - 'PerfContextSwitches': {'name': 'events.PerfContextSwitches', 'type': 'monotonic_gauge'}, - 'PerfDataTLBMisses': {'name': 'events.PerfDataTLBMisses', 'type': 'monotonic_gauge'}, - 'PerfDataTLBReferences': {'name': 'events.PerfDataTLBReferences', 'type': 'monotonic_gauge'}, - 'PerfEmulationFaults': {'name': 'events.PerfEmulationFaults', 'type': 'monotonic_gauge'}, - 'PerfInstructionTLBMisses': {'name': 'events.PerfInstructionTLBMisses', 'type': 'monotonic_gauge'}, - 'PerfInstructionTLBReferences': { - 'name': 'events.PerfInstructionTLBReferences', - 'type': 'monotonic_gauge', - }, - 'PerfInstructions': {'name': 'events.PerfInstructions', 'type': 'monotonic_gauge'}, - 'PerfLocalMemoryMisses': {'name': 'events.PerfLocalMemoryMisses', 'type': 'monotonic_gauge'}, - 'PerfLocalMemoryReferences': {'name': 'events.PerfLocalMemoryReferences', 'type': 'monotonic_gauge'}, - 'PerfMinEnabledRunningTime': {'name': 'events.PerfMinEnabledRunningTime', 'type': 'monotonic_gauge'}, - 'PerfMinEnabledTime': {'name': 'events.PerfMinEnabledTime', 'type': 'monotonic_gauge'}, - 'PerfRefCPUCycles': {'name': 'events.PerfRefCPUCycles', 'type': 'monotonic_gauge'}, - 'PerfStalledCyclesBackend': {'name': 'events.PerfStalledCyclesBackend', 'type': 'monotonic_gauge'}, - 'PerfStalledCyclesFrontend': {'name': 'events.PerfStalledCyclesFrontend', 'type': 'monotonic_gauge'}, - 'PerfTaskClock': {'name': 'events.PerfTaskClock', 'type': 'monotonic_gauge'}, - 'PolygonsAddedToPool': {'name': 'events.PolygonsAddedToPool', 'type': 'monotonic_gauge'}, - 'PolygonsInPoolAllocatedBytes': { - 'name': 'events.PolygonsInPoolAllocatedBytes', - 'type': 'monotonic_gauge', - }, - 'PreferredWarmedUnmergedParts': { - 'name': 'events.PreferredWarmedUnmergedParts', - 'type': 'monotonic_gauge', - }, - 'PrimaryIndexCacheHits': {'name': 'events.PrimaryIndexCacheHits', 'type': 'monotonic_gauge'}, - 'PrimaryIndexCacheMisses': {'name': 'events.PrimaryIndexCacheMisses', 'type': 'monotonic_gauge'}, - 'QueriesWithSubqueries': {'name': 'events.QueriesWithSubqueries', 'type': 'monotonic_gauge'}, - 'Query': {'name': 'events.Query', 'type': 'monotonic_gauge'}, - 'QueryBackupThrottlerBytes': {'name': 'events.QueryBackupThrottlerBytes', 'type': 'monotonic_gauge'}, - 'QueryBackupThrottlerSleepMicroseconds': { - 'name': 'events.QueryBackupThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'QueryCacheHits': {'name': 'events.QueryCacheHits', 'type': 'monotonic_gauge'}, - 'QueryCacheMisses': {'name': 'events.QueryCacheMisses', 'type': 'monotonic_gauge'}, - 'QueryConditionCacheHits': {'name': 'events.QueryConditionCacheHits', 'type': 'monotonic_gauge'}, - 'QueryConditionCacheMisses': {'name': 'events.QueryConditionCacheMisses', 'type': 'monotonic_gauge'}, - 'QueryLocalReadThrottlerBytes': { - 'name': 'events.QueryLocalReadThrottlerBytes', - 'type': 'monotonic_gauge', - }, - 'QueryLocalReadThrottlerSleepMicroseconds': { - 'name': 'events.QueryLocalReadThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'QueryLocalWriteThrottlerBytes': { - 'name': 'events.QueryLocalWriteThrottlerBytes', - 'type': 'monotonic_gauge', - }, - 'QueryLocalWriteThrottlerSleepMicroseconds': { - 'name': 'events.QueryLocalWriteThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'QueryMaskingRulesMatch': {'name': 'events.QueryMaskingRulesMatch', 'type': 'monotonic_gauge'}, - 'QueryMemoryLimitExceeded': {'name': 'events.QueryMemoryLimitExceeded', 'type': 'monotonic_gauge'}, - 'QueryPreempted': {'name': 'events.QueryPreempted', 'type': 'monotonic_gauge'}, - 'QueryProfilerConcurrencyOverruns': { - 'name': 'events.QueryProfilerConcurrencyOverruns', - 'type': 'monotonic_gauge', - }, - 'QueryProfilerErrors': {'name': 'events.QueryProfilerErrors', 'type': 'monotonic_gauge'}, - 'QueryProfilerRuns': {'name': 'events.QueryProfilerRuns', 'type': 'monotonic_gauge'}, - 'QueryProfilerSignalOverruns': { - 'name': 'events.QueryProfilerSignalOverruns', - 'type': 'monotonic_gauge', - }, - 'QueryRemoteReadThrottlerBytes': { - 'name': 'events.QueryRemoteReadThrottlerBytes', - 'type': 'monotonic_gauge', - }, - 'QueryRemoteReadThrottlerSleepMicroseconds': { - 'name': 'events.QueryRemoteReadThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'QueryRemoteWriteThrottlerBytes': { - 'name': 'events.QueryRemoteWriteThrottlerBytes', - 'type': 'monotonic_gauge', - }, - 'QueryRemoteWriteThrottlerSleepMicroseconds': { - 'name': 'events.QueryRemoteWriteThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'QueryTimeMicroseconds': { - 'name': 'events.QueryTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'RWLockAcquiredReadLocks': {'name': 'events.RWLockAcquiredReadLocks', 'type': 'monotonic_gauge'}, - 'RWLockAcquiredWriteLocks': {'name': 'events.RWLockAcquiredWriteLocks', 'type': 'monotonic_gauge'}, - 'RWLockReadersWaitMilliseconds': { - 'name': 'events.RWLockReadersWaitMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'RWLockWritersWaitMilliseconds': { - 'name': 'events.RWLockWritersWaitMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'ReadBackoff': {'name': 'events.ReadBackoff', 'type': 'monotonic_gauge'}, - 'ReadBufferFromAzureBytes': {'name': 'events.ReadBufferFromAzureBytes', 'type': 'monotonic_gauge'}, - 'ReadBufferFromAzureInitMicroseconds': { - 'name': 'events.ReadBufferFromAzureInitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ReadBufferFromAzureMicroseconds': { - 'name': 'events.ReadBufferFromAzureMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ReadBufferFromAzureRequestsErrors': { - 'name': 'events.ReadBufferFromAzureRequestsErrors', - 'type': 'monotonic_gauge', - }, - 'ReadBufferFromFileDescriptorRead': { - 'name': 'events.ReadBufferFromFileDescriptorRead', - 'type': 'monotonic_gauge', - }, - 'ReadBufferFromFileDescriptorReadBytes': { - 'name': 'events.ReadBufferFromFileDescriptorReadBytes', - 'type': 'monotonic_gauge', - }, - 'ReadBufferFromFileDescriptorReadFailed': { - 'name': 'events.ReadBufferFromFileDescriptorReadFailed', - 'type': 'monotonic_gauge', - }, - 'ReadBufferFromS3Bytes': {'name': 'events.ReadBufferFromS3Bytes', 'type': 'monotonic_gauge'}, - 'ReadBufferFromS3InitMicroseconds': { - 'name': 'events.ReadBufferFromS3InitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ReadBufferFromS3Microseconds': { - 'name': 'events.ReadBufferFromS3Microseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ReadBufferFromS3RequestsErrors': { - 'name': 'events.ReadBufferFromS3RequestsErrors', - 'type': 'monotonic_gauge', - }, - 'ReadBufferSeekCancelConnection': { - 'name': 'events.ReadBufferSeekCancelConnection', - 'type': 'monotonic_gauge', - }, - 'ReadCompressedBytes': {'name': 'events.ReadCompressedBytes', 'type': 'monotonic_gauge'}, - 'ReadPatchesMicroseconds': {'name': 'events.ReadPatchesMicroseconds', 'type': 'monotonic_gauge'}, - 'ReadTaskRequestsReceived': {'name': 'events.ReadTaskRequestsReceived', 'type': 'monotonic_gauge'}, - 'ReadTaskRequestsSent': {'name': 'events.ReadTaskRequestsSent', 'type': 'monotonic_gauge'}, - 'ReadTaskRequestsSentElapsedMicroseconds': { - 'name': 'events.ReadTaskRequestsSentElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ReadTasksWithAppliedMutationsOnFly': { - 'name': 'events.ReadTasksWithAppliedMutationsOnFly', - 'type': 'monotonic_gauge', - }, - 'ReadTasksWithAppliedPatches': { - 'name': 'events.ReadTasksWithAppliedPatches', - 'type': 'monotonic_gauge', - }, - 'ReadWriteBufferFromHTTPBytes': { - 'name': 'events.ReadWriteBufferFromHTTPBytes', - 'type': 'monotonic_gauge', - }, - 'ReadWriteBufferFromHTTPRequestsSent': { - 'name': 'events.ReadWriteBufferFromHTTPRequestsSent', - 'type': 'monotonic_gauge', - }, - 'RealTimeMicroseconds': { - 'name': 'events.RealTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'RefreshableViewLockTableRetry': { - 'name': 'events.RefreshableViewLockTableRetry', - 'type': 'monotonic_gauge', - }, - 'RefreshableViewRefreshFailed': { - 'name': 'events.RefreshableViewRefreshFailed', - 'type': 'monotonic_gauge', - }, - 'RefreshableViewRefreshSuccess': { - 'name': 'events.RefreshableViewRefreshSuccess', - 'type': 'monotonic_gauge', - }, - 'RefreshableViewSyncReplicaRetry': { - 'name': 'events.RefreshableViewSyncReplicaRetry', - 'type': 'monotonic_gauge', - }, - 'RefreshableViewSyncReplicaSuccess': { - 'name': 'events.RefreshableViewSyncReplicaSuccess', - 'type': 'monotonic_gauge', - }, - 'RegexpLocalCacheHit': {'name': 'events.RegexpLocalCacheHit', 'type': 'monotonic_gauge'}, - 'RegexpLocalCacheMiss': {'name': 'events.RegexpLocalCacheMiss', 'type': 'monotonic_gauge'}, - 'RegexpWithMultipleNeedlesCreated': { - 'name': 'events.RegexpWithMultipleNeedlesCreated', - 'type': 'monotonic_gauge', - }, - 'RegexpWithMultipleNeedlesGlobalCacheHit': { - 'name': 'events.RegexpWithMultipleNeedlesGlobalCacheHit', - 'type': 'monotonic_gauge', - }, - 'RegexpWithMultipleNeedlesGlobalCacheMiss': { - 'name': 'events.RegexpWithMultipleNeedlesGlobalCacheMiss', - 'type': 'monotonic_gauge', - }, - 'RejectedInserts': {'name': 'events.RejectedInserts', 'type': 'monotonic_gauge'}, - 'RejectedLightweightUpdates': {'name': 'events.RejectedLightweightUpdates', 'type': 'monotonic_gauge'}, - 'RejectedMutations': {'name': 'events.RejectedMutations', 'type': 'monotonic_gauge'}, - 'RemoteFSBuffers': {'name': 'events.RemoteFSBuffers', 'type': 'monotonic_gauge'}, - 'RemoteFSCancelledPrefetches': { - 'name': 'events.RemoteFSCancelledPrefetches', - 'type': 'monotonic_gauge', - }, - 'RemoteFSLazySeeks': {'name': 'events.RemoteFSLazySeeks', 'type': 'monotonic_gauge'}, - 'RemoteFSPrefetchedBytes': {'name': 'events.RemoteFSPrefetchedBytes', 'type': 'monotonic_gauge'}, - 'RemoteFSPrefetchedReads': {'name': 'events.RemoteFSPrefetchedReads', 'type': 'monotonic_gauge'}, - 'RemoteFSPrefetches': {'name': 'events.RemoteFSPrefetches', 'type': 'monotonic_gauge'}, - 'RemoteFSSeeks': {'name': 'events.RemoteFSSeeks', 'type': 'monotonic_gauge'}, - 'RemoteFSSeeksWithReset': {'name': 'events.RemoteFSSeeksWithReset', 'type': 'monotonic_gauge'}, - 'RemoteFSUnprefetchedBytes': {'name': 'events.RemoteFSUnprefetchedBytes', 'type': 'monotonic_gauge'}, - 'RemoteFSUnprefetchedReads': {'name': 'events.RemoteFSUnprefetchedReads', 'type': 'monotonic_gauge'}, - 'RemoteFSUnusedPrefetches': {'name': 'events.RemoteFSUnusedPrefetches', 'type': 'monotonic_gauge'}, - 'RemoteReadThrottlerBytes': {'name': 'events.RemoteReadThrottlerBytes', 'type': 'monotonic_gauge'}, - 'RemoteReadThrottlerSleepMicroseconds': { - 'name': 'events.RemoteReadThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'RemoteWriteThrottlerBytes': {'name': 'events.RemoteWriteThrottlerBytes', 'type': 'monotonic_gauge'}, - 'RemoteWriteThrottlerSleepMicroseconds': { - 'name': 'events.RemoteWriteThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ReplacingSortedMilliseconds': { - 'name': 'events.ReplacingSortedMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'ReplicaPartialShutdown': {'name': 'events.ReplicaPartialShutdown', 'type': 'monotonic_gauge'}, - 'ReplicatedCoveredPartsInZooKeeperOnStart': { - 'name': 'events.ReplicatedCoveredPartsInZooKeeperOnStart', - 'type': 'monotonic_gauge', - }, - 'ReplicatedDataLoss': {'name': 'events.ReplicatedDataLoss', 'type': 'monotonic_gauge'}, - 'ReplicatedPartChecks': {'name': 'events.ReplicatedPartChecks', 'type': 'monotonic_gauge'}, - 'ReplicatedPartChecksFailed': {'name': 'events.ReplicatedPartChecksFailed', 'type': 'monotonic_gauge'}, - 'ReplicatedPartFailedFetches': { - 'name': 'events.ReplicatedPartFailedFetches', - 'type': 'monotonic_gauge', - }, - 'ReplicatedPartFetches': {'name': 'events.ReplicatedPartFetches', 'type': 'monotonic_gauge'}, - 'ReplicatedPartFetchesOfMerged': { - 'name': 'events.ReplicatedPartFetchesOfMerged', - 'type': 'monotonic_gauge', - }, - 'ReplicatedPartMerges': {'name': 'events.ReplicatedPartMerges', 'type': 'monotonic_gauge'}, - 'ReplicatedPartMutations': {'name': 'events.ReplicatedPartMutations', 'type': 'monotonic_gauge'}, - 'RestorePartsSkippedBytes': {'name': 'events.RestorePartsSkippedBytes', 'type': 'monotonic_gauge'}, - 'RestorePartsSkippedFiles': {'name': 'events.RestorePartsSkippedFiles', 'type': 'monotonic_gauge'}, - 'RowsReadByMainReader': {'name': 'events.RowsReadByMainReader', 'type': 'monotonic_gauge'}, - 'RowsReadByPrewhereReaders': {'name': 'events.RowsReadByPrewhereReaders', 'type': 'monotonic_gauge'}, - 'S3AbortMultipartUpload': {'name': 'events.S3AbortMultipartUpload', 'type': 'monotonic_gauge'}, - 'S3Clients': {'name': 'events.S3Clients', 'type': 'monotonic_gauge'}, - 'S3CompleteMultipartUpload': {'name': 'events.S3CompleteMultipartUpload', 'type': 'monotonic_gauge'}, - 'S3CopyObject': {'name': 'events.S3CopyObject', 'type': 'monotonic_gauge'}, - 'S3CreateMultipartUpload': {'name': 'events.S3CreateMultipartUpload', 'type': 'monotonic_gauge'}, - 'S3DeleteObjects': {'name': 'events.S3DeleteObjects', 'type': 'monotonic_gauge'}, - 'S3GetObject': {'name': 'events.S3GetObject', 'type': 'monotonic_gauge'}, - 'S3GetObjectAttributes': {'name': 'events.S3GetObjectAttributes', 'type': 'monotonic_gauge'}, - 'S3GetRequestThrottlerCount': {'name': 'events.S3GetRequestThrottlerCount', 'type': 'monotonic_gauge'}, - 'S3GetRequestThrottlerSleepMicroseconds': { - 'name': 'events.S3GetRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'S3HeadObject': {'name': 'events.S3HeadObject', 'type': 'monotonic_gauge'}, - 'S3ListObjects': {'name': 'events.S3ListObjects', 'type': 'monotonic_gauge'}, - 'S3PutObject': {'name': 'events.S3PutObject', 'type': 'monotonic_gauge'}, - 'S3PutRequestThrottlerCount': {'name': 'events.S3PutRequestThrottlerCount', 'type': 'monotonic_gauge'}, - 'S3PutRequestThrottlerSleepMicroseconds': { - 'name': 'events.S3PutRequestThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'S3QueueSetFileFailedMicroseconds': { - 'name': 'events.S3QueueSetFileFailedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'S3QueueSetFileProcessedMicroseconds': { - 'name': 'events.S3QueueSetFileProcessedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'S3QueueSetFileProcessingMicroseconds': { - 'name': 'events.S3QueueSetFileProcessingMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'S3ReadMicroseconds': { - 'name': 'events.S3ReadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'S3ReadRequestAttempts': {'name': 'events.S3ReadRequestAttempts', 'type': 'monotonic_gauge'}, - 'S3ReadRequestRetryableErrors': { - 'name': 'events.S3ReadRequestRetryableErrors', - 'type': 'monotonic_gauge', - }, - 'S3ReadRequestsCount': {'name': 'events.S3ReadRequestsCount', 'type': 'monotonic_gauge'}, - 'S3ReadRequestsErrors': {'name': 'events.S3ReadRequestsErrors', 'type': 'monotonic_gauge'}, - 'S3ReadRequestsRedirects': {'name': 'events.S3ReadRequestsRedirects', 'type': 'monotonic_gauge'}, - 'S3ReadRequestsThrottling': {'name': 'events.S3ReadRequestsThrottling', 'type': 'monotonic_gauge'}, - 'S3UploadPart': {'name': 'events.S3UploadPart', 'type': 'monotonic_gauge'}, - 'S3UploadPartCopy': {'name': 'events.S3UploadPartCopy', 'type': 'monotonic_gauge'}, - 'S3WriteMicroseconds': { - 'name': 'events.S3WriteMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'S3WriteRequestAttempts': {'name': 'events.S3WriteRequestAttempts', 'type': 'monotonic_gauge'}, - 'S3WriteRequestRetryableErrors': { - 'name': 'events.S3WriteRequestRetryableErrors', - 'type': 'monotonic_gauge', - }, - 'S3WriteRequestsCount': {'name': 'events.S3WriteRequestsCount', 'type': 'monotonic_gauge'}, - 'S3WriteRequestsErrors': {'name': 'events.S3WriteRequestsErrors', 'type': 'monotonic_gauge'}, - 'S3WriteRequestsRedirects': {'name': 'events.S3WriteRequestsRedirects', 'type': 'monotonic_gauge'}, - 'S3WriteRequestsThrottling': {'name': 'events.S3WriteRequestsThrottling', 'type': 'monotonic_gauge'}, - 'ScalarSubqueriesCacheMiss': {'name': 'events.ScalarSubqueriesCacheMiss', 'type': 'monotonic_gauge'}, - 'ScalarSubqueriesGlobalCacheHit': { - 'name': 'events.ScalarSubqueriesGlobalCacheHit', - 'type': 'monotonic_gauge', - }, - 'ScalarSubqueriesLocalCacheHit': { - 'name': 'events.ScalarSubqueriesLocalCacheHit', - 'type': 'monotonic_gauge', - }, - 'SchedulerIOReadBytes': {'name': 'events.SchedulerIOReadBytes', 'type': 'monotonic_gauge'}, - 'SchedulerIOReadRequests': {'name': 'events.SchedulerIOReadRequests', 'type': 'monotonic_gauge'}, - 'SchedulerIOReadWaitMicroseconds': { - 'name': 'events.SchedulerIOReadWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SchedulerIOWriteBytes': {'name': 'events.SchedulerIOWriteBytes', 'type': 'monotonic_gauge'}, - 'SchedulerIOWriteRequests': {'name': 'events.SchedulerIOWriteRequests', 'type': 'monotonic_gauge'}, - 'SchedulerIOWriteWaitMicroseconds': { - 'name': 'events.SchedulerIOWriteWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SchemaInferenceCacheEvictions': { - 'name': 'events.SchemaInferenceCacheEvictions', - 'type': 'monotonic_gauge', - }, - 'SchemaInferenceCacheHits': {'name': 'events.SchemaInferenceCacheHits', 'type': 'monotonic_gauge'}, - 'SchemaInferenceCacheInvalidations': { - 'name': 'events.SchemaInferenceCacheInvalidations', - 'type': 'monotonic_gauge', - }, - 'SchemaInferenceCacheMisses': {'name': 'events.SchemaInferenceCacheMisses', 'type': 'monotonic_gauge'}, - 'SchemaInferenceCacheNumRowsHits': { - 'name': 'events.SchemaInferenceCacheNumRowsHits', - 'type': 'monotonic_gauge', - }, - 'SchemaInferenceCacheNumRowsMisses': { - 'name': 'events.SchemaInferenceCacheNumRowsMisses', - 'type': 'monotonic_gauge', - }, - 'SchemaInferenceCacheSchemaHits': { - 'name': 'events.SchemaInferenceCacheSchemaHits', - 'type': 'monotonic_gauge', - }, - 'SchemaInferenceCacheSchemaMisses': { - 'name': 'events.SchemaInferenceCacheSchemaMisses', - 'type': 'monotonic_gauge', - }, - 'Seek': {'name': 'events.Seek', 'type': 'monotonic_gauge'}, - 'SelectQueriesWithPrimaryKeyUsage': { - 'name': 'events.SelectQueriesWithPrimaryKeyUsage', - 'type': 'monotonic_gauge', - }, - 'SelectQueriesWithSubqueries': { - 'name': 'events.SelectQueriesWithSubqueries', - 'type': 'monotonic_gauge', - }, - 'SelectQuery': {'name': 'events.SelectQuery', 'type': 'monotonic_gauge'}, - 'SelectQueryTimeMicroseconds': { - 'name': 'events.SelectQueryTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SelectedBytes': {'name': 'events.SelectedBytes', 'type': 'monotonic_gauge'}, - 'SelectedMarks': {'name': 'events.SelectedMarks', 'type': 'monotonic_gauge'}, - 'SelectedMarksTotal': {'name': 'events.SelectedMarksTotal', 'type': 'monotonic_gauge'}, - 'SelectedParts': {'name': 'events.SelectedParts', 'type': 'monotonic_gauge'}, - 'SelectedPartsTotal': {'name': 'events.SelectedPartsTotal', 'type': 'monotonic_gauge'}, - 'SelectedRanges': {'name': 'events.SelectedRanges', 'type': 'monotonic_gauge'}, - 'SelectedRows': {'name': 'events.SelectedRows', 'type': 'monotonic_gauge'}, - 'ServerStartupMilliseconds': { - 'name': 'events.ServerStartupMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'SharedDatabaseCatalogFailedToApplyState': { - 'name': 'events.SharedDatabaseCatalogFailedToApplyState', - 'type': 'monotonic_gauge', - }, - 'SharedDatabaseCatalogStateApplicationMicroseconds': { - 'name': 'events.SharedDatabaseCatalogStateApplicationMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SharedMergeTreeCondemnedPartsKillRequest': { - 'name': 'events.SharedMergeTreeCondemnedPartsKillRequest', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeCondemnedPartsLockConfict': { - 'name': 'events.SharedMergeTreeCondemnedPartsLockConfict', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeCondemnedPartsRemoved': { - 'name': 'events.SharedMergeTreeCondemnedPartsRemoved', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeDataPartsFetchAttempt': { - 'name': 'events.SharedMergeTreeDataPartsFetchAttempt', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeDataPartsFetchFromPeer': { - 'name': 'events.SharedMergeTreeDataPartsFetchFromPeer', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeDataPartsFetchFromPeerMicroseconds': { - 'name': 'events.SharedMergeTreeDataPartsFetchFromPeerMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeDataPartsFetchFromS3': { - 'name': 'events.SharedMergeTreeDataPartsFetchFromS3', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeGetPartsBatchToLoadMicroseconds': { - 'name': 'events.SharedMergeTreeGetPartsBatchToLoadMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeHandleBlockingParts': { - 'name': 'events.SharedMergeTreeHandleBlockingParts', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeHandleBlockingPartsMicroseconds': { - 'name': 'events.SharedMergeTreeHandleBlockingPartsMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeHandleFetchPartsMicroseconds': { - 'name': 'events.SharedMergeTreeHandleFetchPartsMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeHandleOutdatedParts': { - 'name': 'events.SharedMergeTreeHandleOutdatedParts', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeHandleOutdatedPartsMicroseconds': { - 'name': 'events.SharedMergeTreeHandleOutdatedPartsMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeLoadChecksumAndIndexesMicroseconds': { - 'name': 'events.SharedMergeTreeLoadChecksumAndIndexesMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMergeMutationAssignmentAttempt': { - 'name': 'events.SharedMergeTreeMergeMutationAssignmentAttempt', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMergeMutationAssignmentFailedWithConflict': { - 'name': 'events.SharedMergeTreeMergeMutationAssignmentFailedWithConflict', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMergeMutationAssignmentFailedWithNothingToDo': { - 'name': 'events.SharedMergeTreeMergeMutationAssignmentFailedWithNothingToDo', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMergeMutationAssignmentSuccessful': { - 'name': 'events.SharedMergeTreeMergeMutationAssignmentSuccessful', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMergePartsMovedToCondemned': { - 'name': 'events.SharedMergeTreeMergePartsMovedToCondemned', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMergePartsMovedToOudated': { - 'name': 'events.SharedMergeTreeMergePartsMovedToOudated', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMergeSelectingTaskMicroseconds': { - 'name': 'events.SharedMergeTreeMergeSelectingTaskMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeMetadataCacheHintLoadedFromCache': { - 'name': 'events.SharedMergeTreeMetadataCacheHintLoadedFromCache', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeOptimizeAsync': { - 'name': 'events.SharedMergeTreeOptimizeAsync', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeOptimizeSync': { - 'name': 'events.SharedMergeTreeOptimizeSync', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeOutdatedPartsConfirmationInvocations': { - 'name': 'events.SharedMergeTreeOutdatedPartsConfirmationInvocations', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeOutdatedPartsConfirmationRequest': { - 'name': 'events.SharedMergeTreeOutdatedPartsConfirmationRequest', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeOutdatedPartsHTTPRequest': { - 'name': 'events.SharedMergeTreeOutdatedPartsHTTPRequest', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeOutdatedPartsHTTPResponse': { - 'name': 'events.SharedMergeTreeOutdatedPartsHTTPResponse', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeScheduleDataProcessingJob': { - 'name': 'events.SharedMergeTreeScheduleDataProcessingJob', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeScheduleDataProcessingJobMicroseconds': { - 'name': 'events.SharedMergeTreeScheduleDataProcessingJobMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeScheduleDataProcessingJobNothingToScheduled': { - 'name': 'events.SharedMergeTreeScheduleDataProcessingJobNothingToScheduled', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeTryUpdateDiskMetadataCacheForPartMicroseconds': { - 'name': 'events.SharedMergeTreeTryUpdateDiskMetadataCacheForPartMicroseconds', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdateMicroseconds': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdateMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SharedMergeTreeVirtualPartsUpdates': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdates', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdatesByLeader': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesByLeader', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdatesForMergesOrStatus': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesForMergesOrStatus', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdatesFromPeer': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesFromPeer', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdatesFromPeerMicroseconds': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesFromPeerMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SharedMergeTreeVirtualPartsUpdatesFromZooKeeper': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesFromZooKeeper', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdatesFromZooKeeperMicroseconds': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesFromZooKeeperMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SharedMergeTreeVirtualPartsUpdatesLeaderFailedElection': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesLeaderFailedElection', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdatesLeaderSuccessfulElection': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesLeaderSuccessfulElection', - 'type': 'monotonic_gauge', - }, - 'SharedMergeTreeVirtualPartsUpdatesPeerNotFound': { - 'name': 'events.SharedMergeTreeVirtualPartsUpdatesPeerNotFound', - 'type': 'monotonic_gauge', - }, - 'SleepFunctionCalls': {'name': 'events.SleepFunctionCalls', 'type': 'monotonic_gauge'}, - 'SleepFunctionElapsedMicroseconds': { - 'name': 'events.SleepFunctionElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SleepFunctionMicroseconds': { - 'name': 'events.SleepFunctionMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SlowRead': {'name': 'events.SlowRead', 'type': 'monotonic_gauge'}, - 'SoftPageFaults': {'name': 'events.SoftPageFaults', 'type': 'monotonic_gauge'}, - 'StorageBufferErrorOnFlush': {'name': 'events.StorageBufferErrorOnFlush', 'type': 'monotonic_gauge'}, - 'StorageBufferFlush': {'name': 'events.StorageBufferFlush', 'type': 'monotonic_gauge'}, - 'StorageBufferLayerLockReadersWaitMilliseconds': { - 'name': 'events.StorageBufferLayerLockReadersWaitMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'StorageBufferLayerLockWritersWaitMilliseconds': { - 'name': 'events.StorageBufferLayerLockWritersWaitMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'StorageBufferPassedAllMinThresholds': { - 'name': 'events.StorageBufferPassedAllMinThresholds', - 'type': 'monotonic_gauge', - }, - 'StorageBufferPassedBytesFlushThreshold': { - 'name': 'events.StorageBufferPassedBytesFlushThreshold', - 'type': 'monotonic_gauge', - }, - 'StorageBufferPassedBytesMaxThreshold': { - 'name': 'events.StorageBufferPassedBytesMaxThreshold', - 'type': 'monotonic_gauge', - }, - 'StorageBufferPassedRowsFlushThreshold': { - 'name': 'events.StorageBufferPassedRowsFlushThreshold', - 'type': 'monotonic_gauge', - }, - 'StorageBufferPassedRowsMaxThreshold': { - 'name': 'events.StorageBufferPassedRowsMaxThreshold', - 'type': 'monotonic_gauge', - }, - 'StorageBufferPassedTimeFlushThreshold': { - 'name': 'events.StorageBufferPassedTimeFlushThreshold', - 'type': 'monotonic_gauge', - }, - 'StorageBufferPassedTimeMaxThreshold': { - 'name': 'events.StorageBufferPassedTimeMaxThreshold', - 'type': 'monotonic_gauge', - }, - 'StorageConnectionsCreated': {'name': 'events.StorageConnectionsCreated', 'type': 'monotonic_gauge'}, - 'StorageConnectionsElapsedMicroseconds': { - 'name': 'events.StorageConnectionsElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'StorageConnectionsErrors': {'name': 'events.StorageConnectionsErrors', 'type': 'monotonic_gauge'}, - 'StorageConnectionsExpired': {'name': 'events.StorageConnectionsExpired', 'type': 'monotonic_gauge'}, - 'StorageConnectionsPreserved': { - 'name': 'events.StorageConnectionsPreserved', - 'type': 'monotonic_gauge', - }, - 'StorageConnectionsReset': {'name': 'events.StorageConnectionsReset', 'type': 'monotonic_gauge'}, - 'StorageConnectionsReused': {'name': 'events.StorageConnectionsReused', 'type': 'monotonic_gauge'}, - 'SummingSortedMilliseconds': { - 'name': 'events.SummingSortedMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'SuspendSendingQueryToShard': {'name': 'events.SuspendSendingQueryToShard', 'type': 'monotonic_gauge'}, - 'SynchronousReadWaitMicroseconds': { - 'name': 'events.SynchronousReadWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SynchronousRemoteReadWaitMicroseconds': { - 'name': 'events.SynchronousRemoteReadWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'SystemLogErrorOnFlush': {'name': 'events.SystemLogErrorOnFlush', 'type': 'monotonic_gauge'}, - 'SystemTimeMicroseconds': { - 'name': 'events.SystemTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'TableFunctionExecute': {'name': 'events.TableFunctionExecute', 'type': 'monotonic_gauge'}, - 'ThreadPoolReaderPageCacheHit': { - 'name': 'events.ThreadPoolReaderPageCacheHit', - 'type': 'monotonic_gauge', - }, - 'ThreadPoolReaderPageCacheHitBytes': { - 'name': 'events.ThreadPoolReaderPageCacheHitBytes', - 'type': 'monotonic_gauge', - }, - 'ThreadPoolReaderPageCacheHitElapsedMicroseconds': { - 'name': 'events.ThreadPoolReaderPageCacheHitElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ThreadPoolReaderPageCacheMiss': { - 'name': 'events.ThreadPoolReaderPageCacheMiss', - 'type': 'monotonic_gauge', - }, - 'ThreadPoolReaderPageCacheMissBytes': { - 'name': 'events.ThreadPoolReaderPageCacheMissBytes', - 'type': 'monotonic_gauge', - }, - 'ThreadPoolReaderPageCacheMissElapsedMicroseconds': { - 'name': 'events.ThreadPoolReaderPageCacheMissElapsedMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ThreadpoolReaderPrepareMicroseconds': { - 'name': 'events.ThreadpoolReaderPrepareMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ThreadpoolReaderReadBytes': {'name': 'events.ThreadpoolReaderReadBytes', 'type': 'monotonic_gauge'}, - 'ThreadpoolReaderSubmit': {'name': 'events.ThreadpoolReaderSubmit', 'type': 'monotonic_gauge'}, - 'ThreadpoolReaderSubmitLookupInCacheMicroseconds': { - 'name': 'events.ThreadpoolReaderSubmitLookupInCacheMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ThreadpoolReaderSubmitReadSynchronously': { - 'name': 'events.ThreadpoolReaderSubmitReadSynchronously', - 'type': 'monotonic_gauge', - }, - 'ThreadpoolReaderSubmitReadSynchronouslyBytes': { - 'name': 'events.ThreadpoolReaderSubmitReadSynchronouslyBytes', - 'type': 'monotonic_gauge', - }, - 'ThreadpoolReaderSubmitReadSynchronouslyMicroseconds': { - 'name': 'events.ThreadpoolReaderSubmitReadSynchronouslyMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ThreadpoolReaderTaskMicroseconds': { - 'name': 'events.ThreadpoolReaderTaskMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ThrottlerSleepMicroseconds': { - 'name': 'events.ThrottlerSleepMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'TinyS3Clients': {'name': 'events.TinyS3Clients', 'type': 'monotonic_gauge'}, - 'USearchAddComputedDistances': { - 'name': 'events.USearchAddComputedDistances', - 'type': 'monotonic_gauge', - }, - 'USearchAddCount': {'name': 'events.USearchAddCount', 'type': 'monotonic_gauge'}, - 'USearchAddVisitedMembers': {'name': 'events.USearchAddVisitedMembers', 'type': 'monotonic_gauge'}, - 'USearchSearchComputedDistances': { - 'name': 'events.USearchSearchComputedDistances', - 'type': 'monotonic_gauge', - }, - 'USearchSearchCount': {'name': 'events.USearchSearchCount', 'type': 'monotonic_gauge'}, - 'USearchSearchVisitedMembers': { - 'name': 'events.USearchSearchVisitedMembers', - 'type': 'monotonic_gauge', - }, - 'UncompressedCacheHits': {'name': 'events.UncompressedCacheHits', 'type': 'monotonic_gauge'}, - 'UncompressedCacheMisses': {'name': 'events.UncompressedCacheMisses', 'type': 'monotonic_gauge'}, - 'UncompressedCacheWeightLost': { - 'name': 'events.UncompressedCacheWeightLost', - 'type': 'monotonic_gauge', - }, - 'UserTimeMicroseconds': { - 'name': 'events.UserTimeMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'VectorSimilarityIndexCacheHits': { - 'name': 'events.VectorSimilarityIndexCacheHits', - 'type': 'monotonic_gauge', - }, - 'VectorSimilarityIndexCacheMisses': { - 'name': 'events.VectorSimilarityIndexCacheMisses', - 'type': 'monotonic_gauge', - }, - 'VectorSimilarityIndexCacheWeightLost': { - 'name': 'events.VectorSimilarityIndexCacheWeightLost', - 'type': 'monotonic_gauge', - }, - 'VersionedCollapsingSortedMilliseconds': { - 'name': 'events.VersionedCollapsingSortedMilliseconds', - 'type': 'temporal_percent', - 'scale': 'millisecond', - }, - 'WaitMarksLoadMicroseconds': { - 'name': 'events.WaitMarksLoadMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'WaitPrefetchTaskMicroseconds': { - 'name': 'events.WaitPrefetchTaskMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'WriteBufferFromFileDescriptorWrite': { - 'name': 'events.WriteBufferFromFileDescriptorWrite', - 'type': 'monotonic_gauge', - }, - 'WriteBufferFromFileDescriptorWriteBytes': { - 'name': 'events.WriteBufferFromFileDescriptorWriteBytes', - 'type': 'monotonic_gauge', - }, - 'WriteBufferFromFileDescriptorWriteFailed': { - 'name': 'events.WriteBufferFromFileDescriptorWriteFailed', - 'type': 'monotonic_gauge', - }, - 'WriteBufferFromHTTPBytes': {'name': 'events.WriteBufferFromHTTPBytes', 'type': 'monotonic_gauge'}, - 'WriteBufferFromHTTPRequestsSent': { - 'name': 'events.WriteBufferFromHTTPRequestsSent', - 'type': 'monotonic_gauge', - }, - 'WriteBufferFromS3Bytes': {'name': 'events.WriteBufferFromS3Bytes', 'type': 'monotonic_gauge'}, - 'WriteBufferFromS3Microseconds': { - 'name': 'events.WriteBufferFromS3Microseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'WriteBufferFromS3RequestsErrors': { - 'name': 'events.WriteBufferFromS3RequestsErrors', - 'type': 'monotonic_gauge', - }, - 'WriteBufferFromS3WaitInflightLimitMicroseconds': { - 'name': 'events.WriteBufferFromS3WaitInflightLimitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ZooKeeperBytesReceived': {'name': 'events.ZooKeeperBytesReceived', 'type': 'monotonic_gauge'}, - 'ZooKeeperBytesSent': {'name': 'events.ZooKeeperBytesSent', 'type': 'monotonic_gauge'}, - 'ZooKeeperCheck': {'name': 'events.ZooKeeperCheck', 'type': 'monotonic_gauge'}, - 'ZooKeeperClose': {'name': 'events.ZooKeeperClose', 'type': 'monotonic_gauge'}, - 'ZooKeeperCreate': {'name': 'events.ZooKeeperCreate', 'type': 'monotonic_gauge'}, - 'ZooKeeperExists': {'name': 'events.ZooKeeperExists', 'type': 'monotonic_gauge'}, - 'ZooKeeperGet': {'name': 'events.ZooKeeperGet', 'type': 'monotonic_gauge'}, - 'ZooKeeperGetACL': {'name': 'events.ZooKeeperGetACL', 'type': 'monotonic_gauge'}, - 'ZooKeeperHardwareExceptions': { - 'name': 'events.ZooKeeperHardwareExceptions', - 'type': 'monotonic_gauge', - }, - 'ZooKeeperInit': {'name': 'events.ZooKeeperInit', 'type': 'monotonic_gauge'}, - 'ZooKeeperList': {'name': 'events.ZooKeeperList', 'type': 'monotonic_gauge'}, - 'ZooKeeperMulti': {'name': 'events.ZooKeeperMulti', 'type': 'monotonic_gauge'}, - 'ZooKeeperMultiRead': {'name': 'events.ZooKeeperMultiRead', 'type': 'monotonic_gauge'}, - 'ZooKeeperMultiWrite': {'name': 'events.ZooKeeperMultiWrite', 'type': 'monotonic_gauge'}, - 'ZooKeeperOtherExceptions': {'name': 'events.ZooKeeperOtherExceptions', 'type': 'monotonic_gauge'}, - 'ZooKeeperReconfig': {'name': 'events.ZooKeeperReconfig', 'type': 'monotonic_gauge'}, - 'ZooKeeperRemove': {'name': 'events.ZooKeeperRemove', 'type': 'monotonic_gauge'}, - 'ZooKeeperSet': {'name': 'events.ZooKeeperSet', 'type': 'monotonic_gauge'}, - 'ZooKeeperSync': {'name': 'events.ZooKeeperSync', 'type': 'monotonic_gauge'}, - 'ZooKeeperTransactions': {'name': 'events.ZooKeeperTransactions', 'type': 'monotonic_gauge'}, - 'ZooKeeperUserExceptions': {'name': 'events.ZooKeeperUserExceptions', 'type': 'monotonic_gauge'}, - 'ZooKeeperWaitMicroseconds': { - 'name': 'events.ZooKeeperWaitMicroseconds', - 'type': 'temporal_percent', - 'scale': 'microsecond', - }, - 'ZooKeeperWatchResponse': {'name': 'events.ZooKeeperWatchResponse', 'type': 'monotonic_gauge'}, - }, - }, - ], -} diff --git a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_metrics.py b/clickhouse/datadog_checks/clickhouse/advanced_queries/system_metrics.py deleted file mode 100644 index e729780bcd24e..0000000000000 --- a/clickhouse/datadog_checks/clickhouse/advanced_queries/system_metrics.py +++ /dev/null @@ -1,780 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# This file is autogenerated. -# To change this file you should edit scripts/templates/system_metrics.tpl and then run the following command: -# hatch run metrics:generate - -# https://clickhouse.com/docs/operations/system-tables/metrics -SystemMetrics = { - 'name': 'system_metrics', - 'query': 'SELECT value, metric FROM system.metrics', - 'columns': [ - {'name': 'metric_value', 'type': 'source'}, - { - 'name': 'metric_name', - 'type': 'match', - 'source': 'metric_value', - 'items': { - 'ActiveTimersInQueryProfiler': {'name': 'metrics.ActiveTimersInQueryProfiler', 'type': 'gauge'}, - 'AddressesActive': {'name': 'metrics.AddressesActive', 'type': 'gauge'}, - 'AddressesBanned': {'name': 'metrics.AddressesBanned', 'type': 'gauge'}, - 'AggregatorThreads': {'name': 'metrics.AggregatorThreads', 'type': 'gauge'}, - 'AggregatorThreadsActive': {'name': 'metrics.AggregatorThreadsActive', 'type': 'gauge'}, - 'AggregatorThreadsScheduled': {'name': 'metrics.AggregatorThreadsScheduled', 'type': 'gauge'}, - 'AsyncInsertCacheSize': {'name': 'metrics.AsyncInsertCacheSize', 'type': 'gauge'}, - 'AsynchronousInsertQueueBytes': {'name': 'metrics.AsynchronousInsertQueueBytes', 'type': 'gauge'}, - 'AsynchronousInsertQueueSize': {'name': 'metrics.AsynchronousInsertQueueSize', 'type': 'gauge'}, - 'AsynchronousInsertThreads': {'name': 'metrics.AsynchronousInsertThreads', 'type': 'gauge'}, - 'AsynchronousInsertThreadsActive': {'name': 'metrics.AsynchronousInsertThreadsActive', 'type': 'gauge'}, - 'AsynchronousInsertThreadsScheduled': { - 'name': 'metrics.AsynchronousInsertThreadsScheduled', - 'type': 'gauge', - }, - 'AsynchronousReadWait': {'name': 'metrics.AsynchronousReadWait', 'type': 'gauge'}, - 'AttachedDatabase': {'name': 'metrics.AttachedDatabase', 'type': 'gauge'}, - 'AttachedDictionary': {'name': 'metrics.AttachedDictionary', 'type': 'gauge'}, - 'AttachedReplicatedTable': {'name': 'metrics.AttachedReplicatedTable', 'type': 'gauge'}, - 'AttachedTable': {'name': 'metrics.AttachedTable', 'type': 'gauge'}, - 'AttachedView': {'name': 'metrics.AttachedView', 'type': 'gauge'}, - 'AvroSchemaCacheBytes': {'name': 'metrics.AvroSchemaCacheBytes', 'type': 'gauge'}, - 'AvroSchemaCacheCells': {'name': 'metrics.AvroSchemaCacheCells', 'type': 'gauge'}, - 'AvroSchemaRegistryCacheBytes': {'name': 'metrics.AvroSchemaRegistryCacheBytes', 'type': 'gauge'}, - 'AvroSchemaRegistryCacheCells': {'name': 'metrics.AvroSchemaRegistryCacheCells', 'type': 'gauge'}, - 'AzureRequests': {'name': 'metrics.AzureRequests', 'type': 'gauge'}, - 'BackgroundBufferFlushSchedulePoolSize': { - 'name': 'metrics.BackgroundBufferFlushSchedulePoolSize', - 'type': 'gauge', - }, - 'BackgroundBufferFlushSchedulePoolTask': { - 'name': 'metrics.BackgroundBufferFlushSchedulePoolTask', - 'type': 'gauge', - }, - 'BackgroundCommonPoolSize': {'name': 'metrics.BackgroundCommonPoolSize', 'type': 'gauge'}, - 'BackgroundCommonPoolTask': {'name': 'metrics.BackgroundCommonPoolTask', 'type': 'gauge'}, - 'BackgroundDistributedSchedulePoolSize': { - 'name': 'metrics.BackgroundDistributedSchedulePoolSize', - 'type': 'gauge', - }, - 'BackgroundDistributedSchedulePoolTask': { - 'name': 'metrics.BackgroundDistributedSchedulePoolTask', - 'type': 'gauge', - }, - 'BackgroundFetchesPoolSize': {'name': 'metrics.BackgroundFetchesPoolSize', 'type': 'gauge'}, - 'BackgroundFetchesPoolTask': {'name': 'metrics.BackgroundFetchesPoolTask', 'type': 'gauge'}, - 'BackgroundMergesAndMutationsPoolSize': { - 'name': 'metrics.BackgroundMergesAndMutationsPoolSize', - 'type': 'gauge', - }, - 'BackgroundMergesAndMutationsPoolTask': { - 'name': 'metrics.BackgroundMergesAndMutationsPoolTask', - 'type': 'gauge', - }, - 'BackgroundMessageBrokerSchedulePoolSize': { - 'name': 'metrics.BackgroundMessageBrokerSchedulePoolSize', - 'type': 'gauge', - }, - 'BackgroundMessageBrokerSchedulePoolTask': { - 'name': 'metrics.BackgroundMessageBrokerSchedulePoolTask', - 'type': 'gauge', - }, - 'BackgroundMovePoolSize': {'name': 'metrics.BackgroundMovePoolSize', 'type': 'gauge'}, - 'BackgroundMovePoolTask': {'name': 'metrics.BackgroundMovePoolTask', 'type': 'gauge'}, - 'BackgroundSchedulePoolSize': {'name': 'metrics.BackgroundSchedulePoolSize', 'type': 'gauge'}, - 'BackgroundSchedulePoolTask': {'name': 'metrics.BackgroundSchedulePoolTask', 'type': 'gauge'}, - 'BackupsIOThreads': {'name': 'metrics.BackupsIOThreads', 'type': 'gauge'}, - 'BackupsIOThreadsActive': {'name': 'metrics.BackupsIOThreadsActive', 'type': 'gauge'}, - 'BackupsIOThreadsScheduled': {'name': 'metrics.BackupsIOThreadsScheduled', 'type': 'gauge'}, - 'BackupsThreads': {'name': 'metrics.BackupsThreads', 'type': 'gauge'}, - 'BackupsThreadsActive': {'name': 'metrics.BackupsThreadsActive', 'type': 'gauge'}, - 'BackupsThreadsScheduled': {'name': 'metrics.BackupsThreadsScheduled', 'type': 'gauge'}, - 'BrokenDisks': {'name': 'metrics.BrokenDisks', 'type': 'gauge'}, - 'BrokenDistributedBytesToInsert': {'name': 'metrics.BrokenDistributedBytesToInsert', 'type': 'gauge'}, - 'BrokenDistributedFilesToInsert': {'name': 'metrics.BrokenDistributedFilesToInsert', 'type': 'gauge'}, - 'BuildVectorSimilarityIndexThreads': { - 'name': 'metrics.BuildVectorSimilarityIndexThreads', - 'type': 'gauge', - }, - 'BuildVectorSimilarityIndexThreadsActive': { - 'name': 'metrics.BuildVectorSimilarityIndexThreadsActive', - 'type': 'gauge', - }, - 'BuildVectorSimilarityIndexThreadsScheduled': { - 'name': 'metrics.BuildVectorSimilarityIndexThreadsScheduled', - 'type': 'gauge', - }, - 'CacheDetachedFileSegments': {'name': 'metrics.CacheDetachedFileSegments', 'type': 'gauge'}, - 'CacheDictionaryThreads': {'name': 'metrics.CacheDictionaryThreads', 'type': 'gauge'}, - 'CacheDictionaryThreadsActive': {'name': 'metrics.CacheDictionaryThreadsActive', 'type': 'gauge'}, - 'CacheDictionaryThreadsScheduled': {'name': 'metrics.CacheDictionaryThreadsScheduled', 'type': 'gauge'}, - 'CacheDictionaryUpdateQueueBatches': { - 'name': 'metrics.CacheDictionaryUpdateQueueBatches', - 'type': 'gauge', - }, - 'CacheDictionaryUpdateQueueKeys': {'name': 'metrics.CacheDictionaryUpdateQueueKeys', 'type': 'gauge'}, - 'CacheFileSegments': {'name': 'metrics.CacheFileSegments', 'type': 'gauge'}, - 'CacheWarmerBytesInProgress': {'name': 'metrics.CacheWarmerBytesInProgress', 'type': 'gauge'}, - 'CompiledExpressionCacheBytes': {'name': 'metrics.CompiledExpressionCacheBytes', 'type': 'gauge'}, - 'CompiledExpressionCacheCount': {'name': 'metrics.CompiledExpressionCacheCount', 'type': 'gauge'}, - 'Compressing': {'name': 'metrics.Compressing', 'type': 'gauge'}, - 'CompressionThread': {'name': 'metrics.CompressionThread', 'type': 'gauge'}, - 'CompressionThreadActive': {'name': 'metrics.CompressionThreadActive', 'type': 'gauge'}, - 'CompressionThreadScheduled': {'name': 'metrics.CompressionThreadScheduled', 'type': 'gauge'}, - 'ConcurrencyControlAcquired': {'name': 'metrics.ConcurrencyControlAcquired', 'type': 'gauge'}, - 'ConcurrencyControlAcquiredNonCompeting': { - 'name': 'metrics.ConcurrencyControlAcquiredNonCompeting', - 'type': 'gauge', - }, - 'ConcurrencyControlPreempted': {'name': 'metrics.ConcurrencyControlPreempted', 'type': 'gauge'}, - 'ConcurrencyControlScheduled': {'name': 'metrics.ConcurrencyControlScheduled', 'type': 'gauge'}, - 'ConcurrencyControlSoftLimit': {'name': 'metrics.ConcurrencyControlSoftLimit', 'type': 'gauge'}, - 'ConcurrentHashJoinPoolThreads': {'name': 'metrics.ConcurrentHashJoinPoolThreads', 'type': 'gauge'}, - 'ConcurrentHashJoinPoolThreadsActive': { - 'name': 'metrics.ConcurrentHashJoinPoolThreadsActive', - 'type': 'gauge', - }, - 'ConcurrentHashJoinPoolThreadsScheduled': { - 'name': 'metrics.ConcurrentHashJoinPoolThreadsScheduled', - 'type': 'gauge', - }, - 'ConcurrentQueryAcquired': {'name': 'metrics.ConcurrentQueryAcquired', 'type': 'gauge'}, - 'ConcurrentQueryScheduled': {'name': 'metrics.ConcurrentQueryScheduled', 'type': 'gauge'}, - 'ContextLockWait': {'name': 'metrics.ContextLockWait', 'type': 'gauge'}, - 'CoordinatedMergesCoordinatorAssignedMerges': { - 'name': 'metrics.CoordinatedMergesCoordinatorAssignedMerges', - 'type': 'gauge', - }, - 'CoordinatedMergesCoordinatorRunningMerges': { - 'name': 'metrics.CoordinatedMergesCoordinatorRunningMerges', - 'type': 'gauge', - }, - 'CoordinatedMergesWorkerAssignedMerges': { - 'name': 'metrics.CoordinatedMergesWorkerAssignedMerges', - 'type': 'gauge', - }, - 'CreatedTimersInQueryProfiler': {'name': 'metrics.CreatedTimersInQueryProfiler', 'type': 'gauge'}, - 'DDLWorkerThreads': {'name': 'metrics.DDLWorkerThreads', 'type': 'gauge'}, - 'DDLWorkerThreadsActive': {'name': 'metrics.DDLWorkerThreadsActive', 'type': 'gauge'}, - 'DDLWorkerThreadsScheduled': {'name': 'metrics.DDLWorkerThreadsScheduled', 'type': 'gauge'}, - 'DNSAddressesCacheBytes': {'name': 'metrics.DNSAddressesCacheBytes', 'type': 'gauge'}, - 'DNSAddressesCacheSize': {'name': 'metrics.DNSAddressesCacheSize', 'type': 'gauge'}, - 'DNSHostsCacheBytes': {'name': 'metrics.DNSHostsCacheBytes', 'type': 'gauge'}, - 'DNSHostsCacheSize': {'name': 'metrics.DNSHostsCacheSize', 'type': 'gauge'}, - 'DWARFReaderThreads': {'name': 'metrics.DWARFReaderThreads', 'type': 'gauge'}, - 'DWARFReaderThreadsActive': {'name': 'metrics.DWARFReaderThreadsActive', 'type': 'gauge'}, - 'DWARFReaderThreadsScheduled': {'name': 'metrics.DWARFReaderThreadsScheduled', 'type': 'gauge'}, - 'DatabaseBackupThreads': {'name': 'metrics.DatabaseBackupThreads', 'type': 'gauge'}, - 'DatabaseBackupThreadsActive': {'name': 'metrics.DatabaseBackupThreadsActive', 'type': 'gauge'}, - 'DatabaseBackupThreadsScheduled': {'name': 'metrics.DatabaseBackupThreadsScheduled', 'type': 'gauge'}, - 'DatabaseCatalogThreads': {'name': 'metrics.DatabaseCatalogThreads', 'type': 'gauge'}, - 'DatabaseCatalogThreadsActive': {'name': 'metrics.DatabaseCatalogThreadsActive', 'type': 'gauge'}, - 'DatabaseCatalogThreadsScheduled': {'name': 'metrics.DatabaseCatalogThreadsScheduled', 'type': 'gauge'}, - 'DatabaseOnDiskThreads': {'name': 'metrics.DatabaseOnDiskThreads', 'type': 'gauge'}, - 'DatabaseOnDiskThreadsActive': {'name': 'metrics.DatabaseOnDiskThreadsActive', 'type': 'gauge'}, - 'DatabaseOnDiskThreadsScheduled': {'name': 'metrics.DatabaseOnDiskThreadsScheduled', 'type': 'gauge'}, - 'DatabaseReplicatedCreateTablesThreads': { - 'name': 'metrics.DatabaseReplicatedCreateTablesThreads', - 'type': 'gauge', - }, - 'DatabaseReplicatedCreateTablesThreadsActive': { - 'name': 'metrics.DatabaseReplicatedCreateTablesThreadsActive', - 'type': 'gauge', - }, - 'DatabaseReplicatedCreateTablesThreadsScheduled': { - 'name': 'metrics.DatabaseReplicatedCreateTablesThreadsScheduled', - 'type': 'gauge', - }, - 'Decompressing': {'name': 'metrics.Decompressing', 'type': 'gauge'}, - 'DelayedInserts': {'name': 'metrics.DelayedInserts', 'type': 'gauge'}, - 'DestroyAggregatesThreads': {'name': 'metrics.DestroyAggregatesThreads', 'type': 'gauge'}, - 'DestroyAggregatesThreadsActive': {'name': 'metrics.DestroyAggregatesThreadsActive', 'type': 'gauge'}, - 'DestroyAggregatesThreadsScheduled': { - 'name': 'metrics.DestroyAggregatesThreadsScheduled', - 'type': 'gauge', - }, - 'DictCacheRequests': {'name': 'metrics.DictCacheRequests', 'type': 'gauge'}, - 'DiskConnectionsStored': {'name': 'metrics.DiskConnectionsStored', 'type': 'gauge'}, - 'DiskConnectionsTotal': {'name': 'metrics.DiskConnectionsTotal', 'type': 'gauge'}, - 'DiskObjectStorageAsyncThreads': {'name': 'metrics.DiskObjectStorageAsyncThreads', 'type': 'gauge'}, - 'DiskObjectStorageAsyncThreadsActive': { - 'name': 'metrics.DiskObjectStorageAsyncThreadsActive', - 'type': 'gauge', - }, - 'DiskPlainRewritableAzureDirectoryMapSize': { - 'name': 'metrics.DiskPlainRewritableAzureDirectoryMapSize', - 'type': 'gauge', - }, - 'DiskPlainRewritableAzureFileCount': { - 'name': 'metrics.DiskPlainRewritableAzureFileCount', - 'type': 'gauge', - }, - 'DiskPlainRewritableAzureUniqueFileNamesCount': { - 'name': 'metrics.DiskPlainRewritableAzureUniqueFileNamesCount', - 'type': 'gauge', - }, - 'DiskPlainRewritableLocalDirectoryMapSize': { - 'name': 'metrics.DiskPlainRewritableLocalDirectoryMapSize', - 'type': 'gauge', - }, - 'DiskPlainRewritableLocalFileCount': { - 'name': 'metrics.DiskPlainRewritableLocalFileCount', - 'type': 'gauge', - }, - 'DiskPlainRewritableLocalUniqueFileNamesCount': { - 'name': 'metrics.DiskPlainRewritableLocalUniqueFileNamesCount', - 'type': 'gauge', - }, - 'DiskPlainRewritableS3DirectoryMapSize': { - 'name': 'metrics.DiskPlainRewritableS3DirectoryMapSize', - 'type': 'gauge', - }, - 'DiskPlainRewritableS3FileCount': {'name': 'metrics.DiskPlainRewritableS3FileCount', 'type': 'gauge'}, - 'DiskPlainRewritableS3UniqueFileNamesCount': { - 'name': 'metrics.DiskPlainRewritableS3UniqueFileNamesCount', - 'type': 'gauge', - }, - 'DiskS3NoSuchKeyErrors': {'name': 'metrics.DiskS3NoSuchKeyErrors', 'type': 'gauge'}, - 'DiskSpaceReservedForMerge': {'name': 'metrics.DiskSpaceReservedForMerge', 'type': 'gauge'}, - 'DistrCacheAllocatedConnections': {'name': 'metrics.DistrCacheAllocatedConnections', 'type': 'gauge'}, - 'DistrCacheBorrowedConnections': {'name': 'metrics.DistrCacheBorrowedConnections', 'type': 'gauge'}, - 'DistrCacheOpenedConnections': {'name': 'metrics.DistrCacheOpenedConnections', 'type': 'gauge'}, - 'DistrCacheReadRequests': {'name': 'metrics.DistrCacheReadRequests', 'type': 'gauge'}, - 'DistrCacheRegisteredServers': {'name': 'metrics.DistrCacheRegisteredServers', 'type': 'gauge'}, - 'DistrCacheRegisteredServersCurrentAZ': { - 'name': 'metrics.DistrCacheRegisteredServersCurrentAZ', - 'type': 'gauge', - }, - 'DistrCacheServerConnections': {'name': 'metrics.DistrCacheServerConnections', 'type': 'gauge'}, - 'DistrCacheServerRegistryConnections': { - 'name': 'metrics.DistrCacheServerRegistryConnections', - 'type': 'gauge', - }, - 'DistrCacheServerS3CachedClients': {'name': 'metrics.DistrCacheServerS3CachedClients', 'type': 'gauge'}, - 'DistrCacheUsedConnections': {'name': 'metrics.DistrCacheUsedConnections', 'type': 'gauge'}, - 'DistrCacheWriteRequests': {'name': 'metrics.DistrCacheWriteRequests', 'type': 'gauge'}, - 'DistributedBytesToInsert': {'name': 'metrics.DistributedBytesToInsert', 'type': 'gauge'}, - 'DistributedFilesToInsert': {'name': 'metrics.DistributedFilesToInsert', 'type': 'gauge'}, - 'DistributedInsertThreads': {'name': 'metrics.DistributedInsertThreads', 'type': 'gauge'}, - 'DistributedInsertThreadsActive': {'name': 'metrics.DistributedInsertThreadsActive', 'type': 'gauge'}, - 'DistributedInsertThreadsScheduled': { - 'name': 'metrics.DistributedInsertThreadsScheduled', - 'type': 'gauge', - }, - 'DistributedSend': {'name': 'metrics.DistributedSend', 'type': 'gauge'}, - 'DropDistributedCacheThreads': {'name': 'metrics.DropDistributedCacheThreads', 'type': 'gauge'}, - 'DropDistributedCacheThreadsActive': { - 'name': 'metrics.DropDistributedCacheThreadsActive', - 'type': 'gauge', - }, - 'DropDistributedCacheThreadsScheduled': { - 'name': 'metrics.DropDistributedCacheThreadsScheduled', - 'type': 'gauge', - }, - 'EphemeralNode': {'name': 'metrics.EphemeralNode', 'type': 'gauge'}, - 'FilesystemCacheDelayedCleanupElements': { - 'name': 'metrics.FilesystemCacheDelayedCleanupElements', - 'type': 'gauge', - }, - 'FilesystemCacheDownloadQueueElements': { - 'name': 'metrics.FilesystemCacheDownloadQueueElements', - 'type': 'gauge', - }, - 'FilesystemCacheElements': {'name': 'metrics.FilesystemCacheElements', 'type': 'gauge'}, - 'FilesystemCacheHoldFileSegments': {'name': 'metrics.FilesystemCacheHoldFileSegments', 'type': 'gauge'}, - 'FilesystemCacheKeys': {'name': 'metrics.FilesystemCacheKeys', 'type': 'gauge'}, - 'FilesystemCacheReadBuffers': {'name': 'metrics.FilesystemCacheReadBuffers', 'type': 'gauge'}, - 'FilesystemCacheReserveThreads': {'name': 'metrics.FilesystemCacheReserveThreads', 'type': 'gauge'}, - 'FilesystemCacheSize': {'name': 'metrics.FilesystemCacheSize', 'type': 'gauge'}, - 'FilesystemCacheSizeLimit': {'name': 'metrics.FilesystemCacheSizeLimit', 'type': 'gauge'}, - 'FilteringMarksWithPrimaryKey': {'name': 'metrics.FilteringMarksWithPrimaryKey', 'type': 'gauge'}, - 'FilteringMarksWithSecondaryKeys': {'name': 'metrics.FilteringMarksWithSecondaryKeys', 'type': 'gauge'}, - 'FormatParsingThreads': {'name': 'metrics.FormatParsingThreads', 'type': 'gauge'}, - 'FormatParsingThreadsActive': {'name': 'metrics.FormatParsingThreadsActive', 'type': 'gauge'}, - 'FormatParsingThreadsScheduled': {'name': 'metrics.FormatParsingThreadsScheduled', 'type': 'gauge'}, - 'GlobalThread': {'name': 'metrics.GlobalThread', 'type': 'gauge'}, - 'GlobalThreadActive': {'name': 'metrics.GlobalThreadActive', 'type': 'gauge'}, - 'GlobalThreadScheduled': {'name': 'metrics.GlobalThreadScheduled', 'type': 'gauge'}, - 'HTTPConnection': {'name': 'metrics.HTTPConnection', 'type': 'gauge'}, - 'HTTPConnectionsStored': {'name': 'metrics.HTTPConnectionsStored', 'type': 'gauge'}, - 'HTTPConnectionsTotal': {'name': 'metrics.HTTPConnectionsTotal', 'type': 'gauge'}, - 'HashedDictionaryThreads': {'name': 'metrics.HashedDictionaryThreads', 'type': 'gauge'}, - 'HashedDictionaryThreadsActive': {'name': 'metrics.HashedDictionaryThreadsActive', 'type': 'gauge'}, - 'HashedDictionaryThreadsScheduled': { - 'name': 'metrics.HashedDictionaryThreadsScheduled', - 'type': 'gauge', - }, - 'HiveFilesCacheBytes': {'name': 'metrics.HiveFilesCacheBytes', 'type': 'gauge'}, - 'HiveFilesCacheFiles': {'name': 'metrics.HiveFilesCacheFiles', 'type': 'gauge'}, - 'HiveMetadataFilesCacheBytes': {'name': 'metrics.HiveMetadataFilesCacheBytes', 'type': 'gauge'}, - 'HiveMetadataFilesCacheFiles': {'name': 'metrics.HiveMetadataFilesCacheFiles', 'type': 'gauge'}, - 'IDiskCopierThreads': {'name': 'metrics.IDiskCopierThreads', 'type': 'gauge'}, - 'IDiskCopierThreadsActive': {'name': 'metrics.IDiskCopierThreadsActive', 'type': 'gauge'}, - 'IDiskCopierThreadsScheduled': {'name': 'metrics.IDiskCopierThreadsScheduled', 'type': 'gauge'}, - 'IOPrefetchThreads': {'name': 'metrics.IOPrefetchThreads', 'type': 'gauge'}, - 'IOPrefetchThreadsActive': {'name': 'metrics.IOPrefetchThreadsActive', 'type': 'gauge'}, - 'IOPrefetchThreadsScheduled': {'name': 'metrics.IOPrefetchThreadsScheduled', 'type': 'gauge'}, - 'IOThreads': {'name': 'metrics.IOThreads', 'type': 'gauge'}, - 'IOThreadsActive': {'name': 'metrics.IOThreadsActive', 'type': 'gauge'}, - 'IOThreadsScheduled': {'name': 'metrics.IOThreadsScheduled', 'type': 'gauge'}, - 'IOUringInFlightEvents': {'name': 'metrics.IOUringInFlightEvents', 'type': 'gauge'}, - 'IOUringPendingEvents': {'name': 'metrics.IOUringPendingEvents', 'type': 'gauge'}, - 'IOWriterThreads': {'name': 'metrics.IOWriterThreads', 'type': 'gauge'}, - 'IOWriterThreadsActive': {'name': 'metrics.IOWriterThreadsActive', 'type': 'gauge'}, - 'IOWriterThreadsScheduled': {'name': 'metrics.IOWriterThreadsScheduled', 'type': 'gauge'}, - 'IcebergCatalogThreads': {'name': 'metrics.IcebergCatalogThreads', 'type': 'gauge'}, - 'IcebergCatalogThreadsActive': {'name': 'metrics.IcebergCatalogThreadsActive', 'type': 'gauge'}, - 'IcebergCatalogThreadsScheduled': {'name': 'metrics.IcebergCatalogThreadsScheduled', 'type': 'gauge'}, - 'IcebergMetadataFilesCacheBytes': {'name': 'metrics.IcebergMetadataFilesCacheBytes', 'type': 'gauge'}, - 'IcebergMetadataFilesCacheFiles': {'name': 'metrics.IcebergMetadataFilesCacheFiles', 'type': 'gauge'}, - 'IndexMarkCacheBytes': {'name': 'metrics.IndexMarkCacheBytes', 'type': 'gauge'}, - 'IndexMarkCacheFiles': {'name': 'metrics.IndexMarkCacheFiles', 'type': 'gauge'}, - 'IndexUncompressedCacheBytes': {'name': 'metrics.IndexUncompressedCacheBytes', 'type': 'gauge'}, - 'IndexUncompressedCacheCells': {'name': 'metrics.IndexUncompressedCacheCells', 'type': 'gauge'}, - 'InterserverConnection': {'name': 'metrics.InterserverConnection', 'type': 'gauge'}, - 'IsServerShuttingDown': {'name': 'metrics.IsServerShuttingDown', 'type': 'gauge'}, - 'KafkaAssignedPartitions': {'name': 'metrics.KafkaAssignedPartitions', 'type': 'gauge'}, - 'KafkaBackgroundReads': {'name': 'metrics.KafkaBackgroundReads', 'type': 'gauge'}, - 'KafkaConsumers': {'name': 'metrics.KafkaConsumers', 'type': 'gauge'}, - 'KafkaConsumersInUse': {'name': 'metrics.KafkaConsumersInUse', 'type': 'gauge'}, - 'KafkaConsumersWithAssignment': {'name': 'metrics.KafkaConsumersWithAssignment', 'type': 'gauge'}, - 'KafkaLibrdkafkaThreads': {'name': 'metrics.KafkaLibrdkafkaThreads', 'type': 'gauge'}, - 'KafkaProducers': {'name': 'metrics.KafkaProducers', 'type': 'gauge'}, - 'KafkaWrites': {'name': 'metrics.KafkaWrites', 'type': 'gauge'}, - 'KeeperAliveConnections': {'name': 'metrics.KeeperAliveConnections', 'type': 'gauge'}, - 'KeeperOutstandingRequests': {'name': 'metrics.KeeperOutstandingRequests', 'type': 'gauge'}, - 'LicenseRemainingSeconds': {'name': 'metrics.LicenseRemainingSeconds', 'type': 'gauge'}, - 'LocalThread': {'name': 'metrics.LocalThread', 'type': 'gauge'}, - 'LocalThreadActive': {'name': 'metrics.LocalThreadActive', 'type': 'gauge'}, - 'LocalThreadScheduled': {'name': 'metrics.LocalThreadScheduled', 'type': 'gauge'}, - 'MMapCacheCells': {'name': 'metrics.MMapCacheCells', 'type': 'gauge'}, - 'MMappedFileBytes': {'name': 'metrics.MMappedFileBytes', 'type': 'gauge'}, - 'MMappedFiles': {'name': 'metrics.MMappedFiles', 'type': 'gauge'}, - 'MarkCacheBytes': {'name': 'metrics.MarkCacheBytes', 'type': 'gauge'}, - 'MarkCacheFiles': {'name': 'metrics.MarkCacheFiles', 'type': 'gauge'}, - 'MarksLoaderThreads': {'name': 'metrics.MarksLoaderThreads', 'type': 'gauge'}, - 'MarksLoaderThreadsActive': {'name': 'metrics.MarksLoaderThreadsActive', 'type': 'gauge'}, - 'MarksLoaderThreadsScheduled': {'name': 'metrics.MarksLoaderThreadsScheduled', 'type': 'gauge'}, - 'MaxDDLEntryID': {'name': 'metrics.MaxDDLEntryID', 'type': 'gauge'}, - 'MaxPushedDDLEntryID': {'name': 'metrics.MaxPushedDDLEntryID', 'type': 'gauge'}, - 'MemoryTracking': {'name': 'metrics.MemoryTracking', 'type': 'gauge'}, - 'MemoryTrackingUncorrected': {'name': 'metrics.MemoryTrackingUncorrected', 'type': 'gauge'}, - 'Merge': {'name': 'metrics.Merge', 'type': 'gauge'}, - 'MergeJoinBlocksCacheBytes': {'name': 'metrics.MergeJoinBlocksCacheBytes', 'type': 'gauge'}, - 'MergeJoinBlocksCacheCount': {'name': 'metrics.MergeJoinBlocksCacheCount', 'type': 'gauge'}, - 'MergeParts': {'name': 'metrics.MergeParts', 'type': 'gauge'}, - 'MergeTreeAllRangesAnnouncementsSent': { - 'name': 'metrics.MergeTreeAllRangesAnnouncementsSent', - 'type': 'gauge', - }, - 'MergeTreeBackgroundExecutorThreads': { - 'name': 'metrics.MergeTreeBackgroundExecutorThreads', - 'type': 'gauge', - }, - 'MergeTreeBackgroundExecutorThreadsActive': { - 'name': 'metrics.MergeTreeBackgroundExecutorThreadsActive', - 'type': 'gauge', - }, - 'MergeTreeBackgroundExecutorThreadsScheduled': { - 'name': 'metrics.MergeTreeBackgroundExecutorThreadsScheduled', - 'type': 'gauge', - }, - 'MergeTreeDataSelectExecutorThreads': { - 'name': 'metrics.MergeTreeDataSelectExecutorThreads', - 'type': 'gauge', - }, - 'MergeTreeDataSelectExecutorThreadsActive': { - 'name': 'metrics.MergeTreeDataSelectExecutorThreadsActive', - 'type': 'gauge', - }, - 'MergeTreeDataSelectExecutorThreadsScheduled': { - 'name': 'metrics.MergeTreeDataSelectExecutorThreadsScheduled', - 'type': 'gauge', - }, - 'MergeTreeFetchPartitionThreads': {'name': 'metrics.MergeTreeFetchPartitionThreads', 'type': 'gauge'}, - 'MergeTreeFetchPartitionThreadsActive': { - 'name': 'metrics.MergeTreeFetchPartitionThreadsActive', - 'type': 'gauge', - }, - 'MergeTreeFetchPartitionThreadsScheduled': { - 'name': 'metrics.MergeTreeFetchPartitionThreadsScheduled', - 'type': 'gauge', - }, - 'MergeTreeOutdatedPartsLoaderThreads': { - 'name': 'metrics.MergeTreeOutdatedPartsLoaderThreads', - 'type': 'gauge', - }, - 'MergeTreeOutdatedPartsLoaderThreadsActive': { - 'name': 'metrics.MergeTreeOutdatedPartsLoaderThreadsActive', - 'type': 'gauge', - }, - 'MergeTreeOutdatedPartsLoaderThreadsScheduled': { - 'name': 'metrics.MergeTreeOutdatedPartsLoaderThreadsScheduled', - 'type': 'gauge', - }, - 'MergeTreePartsCleanerThreads': {'name': 'metrics.MergeTreePartsCleanerThreads', 'type': 'gauge'}, - 'MergeTreePartsCleanerThreadsActive': { - 'name': 'metrics.MergeTreePartsCleanerThreadsActive', - 'type': 'gauge', - }, - 'MergeTreePartsCleanerThreadsScheduled': { - 'name': 'metrics.MergeTreePartsCleanerThreadsScheduled', - 'type': 'gauge', - }, - 'MergeTreePartsLoaderThreads': {'name': 'metrics.MergeTreePartsLoaderThreads', 'type': 'gauge'}, - 'MergeTreePartsLoaderThreadsActive': { - 'name': 'metrics.MergeTreePartsLoaderThreadsActive', - 'type': 'gauge', - }, - 'MergeTreePartsLoaderThreadsScheduled': { - 'name': 'metrics.MergeTreePartsLoaderThreadsScheduled', - 'type': 'gauge', - }, - 'MergeTreeReadTaskRequestsSent': {'name': 'metrics.MergeTreeReadTaskRequestsSent', 'type': 'gauge'}, - 'MergeTreeSubcolumnsReaderThreads': { - 'name': 'metrics.MergeTreeSubcolumnsReaderThreads', - 'type': 'gauge', - }, - 'MergeTreeSubcolumnsReaderThreadsActive': { - 'name': 'metrics.MergeTreeSubcolumnsReaderThreadsActive', - 'type': 'gauge', - }, - 'MergeTreeSubcolumnsReaderThreadsScheduled': { - 'name': 'metrics.MergeTreeSubcolumnsReaderThreadsScheduled', - 'type': 'gauge', - }, - 'MergeTreeUnexpectedPartsLoaderThreads': { - 'name': 'metrics.MergeTreeUnexpectedPartsLoaderThreads', - 'type': 'gauge', - }, - 'MergeTreeUnexpectedPartsLoaderThreadsActive': { - 'name': 'metrics.MergeTreeUnexpectedPartsLoaderThreadsActive', - 'type': 'gauge', - }, - 'MergeTreeUnexpectedPartsLoaderThreadsScheduled': { - 'name': 'metrics.MergeTreeUnexpectedPartsLoaderThreadsScheduled', - 'type': 'gauge', - }, - 'MergesMutationsMemoryTracking': {'name': 'metrics.MergesMutationsMemoryTracking', 'type': 'gauge'}, - 'MetadataFromKeeperCacheObjects': {'name': 'metrics.MetadataFromKeeperCacheObjects', 'type': 'gauge'}, - 'Move': {'name': 'metrics.Move', 'type': 'gauge'}, - 'MySQLConnection': {'name': 'metrics.MySQLConnection', 'type': 'gauge'}, - 'NetworkReceive': {'name': 'metrics.NetworkReceive', 'type': 'gauge'}, - 'NetworkSend': {'name': 'metrics.NetworkSend', 'type': 'gauge'}, - 'ObjectStorageAzureThreads': {'name': 'metrics.ObjectStorageAzureThreads', 'type': 'gauge'}, - 'ObjectStorageAzureThreadsActive': {'name': 'metrics.ObjectStorageAzureThreadsActive', 'type': 'gauge'}, - 'ObjectStorageAzureThreadsScheduled': { - 'name': 'metrics.ObjectStorageAzureThreadsScheduled', - 'type': 'gauge', - }, - 'ObjectStorageQueueRegisteredServers': { - 'name': 'metrics.ObjectStorageQueueRegisteredServers', - 'type': 'gauge', - }, - 'ObjectStorageQueueShutdownThreads': { - 'name': 'metrics.ObjectStorageQueueShutdownThreads', - 'type': 'gauge', - }, - 'ObjectStorageQueueShutdownThreadsActive': { - 'name': 'metrics.ObjectStorageQueueShutdownThreadsActive', - 'type': 'gauge', - }, - 'ObjectStorageQueueShutdownThreadsScheduled': { - 'name': 'metrics.ObjectStorageQueueShutdownThreadsScheduled', - 'type': 'gauge', - }, - 'ObjectStorageS3Threads': {'name': 'metrics.ObjectStorageS3Threads', 'type': 'gauge'}, - 'ObjectStorageS3ThreadsActive': {'name': 'metrics.ObjectStorageS3ThreadsActive', 'type': 'gauge'}, - 'ObjectStorageS3ThreadsScheduled': {'name': 'metrics.ObjectStorageS3ThreadsScheduled', 'type': 'gauge'}, - 'OpenFileForRead': {'name': 'metrics.OpenFileForRead', 'type': 'gauge'}, - 'OpenFileForWrite': {'name': 'metrics.OpenFileForWrite', 'type': 'gauge'}, - 'OutdatedPartsLoadingThreads': {'name': 'metrics.OutdatedPartsLoadingThreads', 'type': 'gauge'}, - 'OutdatedPartsLoadingThreadsActive': { - 'name': 'metrics.OutdatedPartsLoadingThreadsActive', - 'type': 'gauge', - }, - 'OutdatedPartsLoadingThreadsScheduled': { - 'name': 'metrics.OutdatedPartsLoadingThreadsScheduled', - 'type': 'gauge', - }, - 'PageCacheBytes': {'name': 'metrics.PageCacheBytes', 'type': 'gauge'}, - 'PageCacheCells': {'name': 'metrics.PageCacheCells', 'type': 'gauge'}, - 'ParallelCompressedWriteBufferThreads': { - 'name': 'metrics.ParallelCompressedWriteBufferThreads', - 'type': 'gauge', - }, - 'ParallelCompressedWriteBufferWait': { - 'name': 'metrics.ParallelCompressedWriteBufferWait', - 'type': 'gauge', - }, - 'ParallelFormattingOutputFormatThreads': { - 'name': 'metrics.ParallelFormattingOutputFormatThreads', - 'type': 'gauge', - }, - 'ParallelFormattingOutputFormatThreadsActive': { - 'name': 'metrics.ParallelFormattingOutputFormatThreadsActive', - 'type': 'gauge', - }, - 'ParallelFormattingOutputFormatThreadsScheduled': { - 'name': 'metrics.ParallelFormattingOutputFormatThreadsScheduled', - 'type': 'gauge', - }, - 'ParallelParsingInputFormatThreads': { - 'name': 'metrics.ParallelParsingInputFormatThreads', - 'type': 'gauge', - }, - 'ParallelParsingInputFormatThreadsActive': { - 'name': 'metrics.ParallelParsingInputFormatThreadsActive', - 'type': 'gauge', - }, - 'ParallelParsingInputFormatThreadsScheduled': { - 'name': 'metrics.ParallelParsingInputFormatThreadsScheduled', - 'type': 'gauge', - }, - 'ParallelWithQueryActiveThreads': {'name': 'metrics.ParallelWithQueryActiveThreads', 'type': 'gauge'}, - 'ParallelWithQueryScheduledThreads': { - 'name': 'metrics.ParallelWithQueryScheduledThreads', - 'type': 'gauge', - }, - 'ParallelWithQueryThreads': {'name': 'metrics.ParallelWithQueryThreads', 'type': 'gauge'}, - 'ParquetDecoderIOThreads': {'name': 'metrics.ParquetDecoderIOThreads', 'type': 'gauge'}, - 'ParquetDecoderIOThreadsActive': {'name': 'metrics.ParquetDecoderIOThreadsActive', 'type': 'gauge'}, - 'ParquetDecoderIOThreadsScheduled': { - 'name': 'metrics.ParquetDecoderIOThreadsScheduled', - 'type': 'gauge', - }, - 'ParquetDecoderThreads': {'name': 'metrics.ParquetDecoderThreads', 'type': 'gauge'}, - 'ParquetDecoderThreadsActive': {'name': 'metrics.ParquetDecoderThreadsActive', 'type': 'gauge'}, - 'ParquetDecoderThreadsScheduled': {'name': 'metrics.ParquetDecoderThreadsScheduled', 'type': 'gauge'}, - 'ParquetEncoderThreads': {'name': 'metrics.ParquetEncoderThreads', 'type': 'gauge'}, - 'ParquetEncoderThreadsActive': {'name': 'metrics.ParquetEncoderThreadsActive', 'type': 'gauge'}, - 'ParquetEncoderThreadsScheduled': {'name': 'metrics.ParquetEncoderThreadsScheduled', 'type': 'gauge'}, - 'PartMutation': {'name': 'metrics.PartMutation', 'type': 'gauge'}, - 'PartsActive': {'name': 'metrics.PartsActive', 'type': 'gauge'}, - 'PartsCommitted': {'name': 'metrics.PartsCommitted', 'type': 'gauge'}, - 'PartsCompact': {'name': 'metrics.PartsCompact', 'type': 'gauge'}, - 'PartsDeleteOnDestroy': {'name': 'metrics.PartsDeleteOnDestroy', 'type': 'gauge'}, - 'PartsDeleting': {'name': 'metrics.PartsDeleting', 'type': 'gauge'}, - 'PartsOutdated': {'name': 'metrics.PartsOutdated', 'type': 'gauge'}, - 'PartsPreActive': {'name': 'metrics.PartsPreActive', 'type': 'gauge'}, - 'PartsPreCommitted': {'name': 'metrics.PartsPreCommitted', 'type': 'gauge'}, - 'PartsTemporary': {'name': 'metrics.PartsTemporary', 'type': 'gauge'}, - 'PartsWide': {'name': 'metrics.PartsWide', 'type': 'gauge'}, - 'PendingAsyncInsert': {'name': 'metrics.PendingAsyncInsert', 'type': 'gauge'}, - 'PolygonDictionaryThreads': {'name': 'metrics.PolygonDictionaryThreads', 'type': 'gauge'}, - 'PolygonDictionaryThreadsActive': {'name': 'metrics.PolygonDictionaryThreadsActive', 'type': 'gauge'}, - 'PolygonDictionaryThreadsScheduled': { - 'name': 'metrics.PolygonDictionaryThreadsScheduled', - 'type': 'gauge', - }, - 'PostgreSQLConnection': {'name': 'metrics.PostgreSQLConnection', 'type': 'gauge'}, - 'PrimaryIndexCacheBytes': {'name': 'metrics.PrimaryIndexCacheBytes', 'type': 'gauge'}, - 'PrimaryIndexCacheFiles': {'name': 'metrics.PrimaryIndexCacheFiles', 'type': 'gauge'}, - 'Query': {'name': 'metrics.Query', 'type': 'gauge'}, - 'QueryCacheBytes': {'name': 'metrics.QueryCacheBytes', 'type': 'gauge'}, - 'QueryCacheEntries': {'name': 'metrics.QueryCacheEntries', 'type': 'gauge'}, - 'QueryConditionCacheBytes': {'name': 'metrics.QueryConditionCacheBytes', 'type': 'gauge'}, - 'QueryConditionCacheEntries': {'name': 'metrics.QueryConditionCacheEntries', 'type': 'gauge'}, - 'QueryPipelineExecutorThreads': {'name': 'metrics.QueryPipelineExecutorThreads', 'type': 'gauge'}, - 'QueryPipelineExecutorThreadsActive': { - 'name': 'metrics.QueryPipelineExecutorThreadsActive', - 'type': 'gauge', - }, - 'QueryPipelineExecutorThreadsScheduled': { - 'name': 'metrics.QueryPipelineExecutorThreadsScheduled', - 'type': 'gauge', - }, - 'QueryPreempted': {'name': 'metrics.QueryPreempted', 'type': 'gauge'}, - 'QueryThread': {'name': 'metrics.QueryThread', 'type': 'gauge'}, - 'RWLockActiveReaders': {'name': 'metrics.RWLockActiveReaders', 'type': 'gauge'}, - 'RWLockActiveWriters': {'name': 'metrics.RWLockActiveWriters', 'type': 'gauge'}, - 'RWLockWaitingReaders': {'name': 'metrics.RWLockWaitingReaders', 'type': 'gauge'}, - 'RWLockWaitingWriters': {'name': 'metrics.RWLockWaitingWriters', 'type': 'gauge'}, - 'Read': {'name': 'metrics.Read', 'type': 'gauge'}, - 'ReadTaskRequestsSent': {'name': 'metrics.ReadTaskRequestsSent', 'type': 'gauge'}, - 'ReadonlyDisks': {'name': 'metrics.ReadonlyDisks', 'type': 'gauge'}, - 'ReadonlyReplica': {'name': 'metrics.ReadonlyReplica', 'type': 'gauge'}, - 'RefreshableViews': {'name': 'metrics.RefreshableViews', 'type': 'gauge'}, - 'RefreshingViews': {'name': 'metrics.RefreshingViews', 'type': 'gauge'}, - 'RemoteRead': {'name': 'metrics.RemoteRead', 'type': 'gauge'}, - 'ReplicatedChecks': {'name': 'metrics.ReplicatedChecks', 'type': 'gauge'}, - 'ReplicatedFetch': {'name': 'metrics.ReplicatedFetch', 'type': 'gauge'}, - 'ReplicatedSend': {'name': 'metrics.ReplicatedSend', 'type': 'gauge'}, - 'RestartReplicaThreads': {'name': 'metrics.RestartReplicaThreads', 'type': 'gauge'}, - 'RestartReplicaThreadsActive': {'name': 'metrics.RestartReplicaThreadsActive', 'type': 'gauge'}, - 'RestartReplicaThreadsScheduled': {'name': 'metrics.RestartReplicaThreadsScheduled', 'type': 'gauge'}, - 'RestoreThreads': {'name': 'metrics.RestoreThreads', 'type': 'gauge'}, - 'RestoreThreadsActive': {'name': 'metrics.RestoreThreadsActive', 'type': 'gauge'}, - 'RestoreThreadsScheduled': {'name': 'metrics.RestoreThreadsScheduled', 'type': 'gauge'}, - 'Revision': {'name': 'metrics.Revision', 'type': 'gauge'}, - 'S3Requests': {'name': 'metrics.S3Requests', 'type': 'gauge'}, - 'SchedulerIOReadScheduled': {'name': 'metrics.SchedulerIOReadScheduled', 'type': 'gauge'}, - 'SchedulerIOWriteScheduled': {'name': 'metrics.SchedulerIOWriteScheduled', 'type': 'gauge'}, - 'SendExternalTables': {'name': 'metrics.SendExternalTables', 'type': 'gauge'}, - 'SendScalars': {'name': 'metrics.SendScalars', 'type': 'gauge'}, - 'SharedCatalogDropDetachLocalTablesErrors': { - 'name': 'metrics.SharedCatalogDropDetachLocalTablesErrors', - 'type': 'gauge', - }, - 'SharedCatalogDropLocalThreads': {'name': 'metrics.SharedCatalogDropLocalThreads', 'type': 'gauge'}, - 'SharedCatalogDropLocalThreadsActive': { - 'name': 'metrics.SharedCatalogDropLocalThreadsActive', - 'type': 'gauge', - }, - 'SharedCatalogDropLocalThreadsScheduled': { - 'name': 'metrics.SharedCatalogDropLocalThreadsScheduled', - 'type': 'gauge', - }, - 'SharedCatalogDropZooKeeperThreads': { - 'name': 'metrics.SharedCatalogDropZooKeeperThreads', - 'type': 'gauge', - }, - 'SharedCatalogDropZooKeeperThreadsActive': { - 'name': 'metrics.SharedCatalogDropZooKeeperThreadsActive', - 'type': 'gauge', - }, - 'SharedCatalogDropZooKeeperThreadsScheduled': { - 'name': 'metrics.SharedCatalogDropZooKeeperThreadsScheduled', - 'type': 'gauge', - }, - 'SharedCatalogNumberOfObjectsInState': { - 'name': 'metrics.SharedCatalogNumberOfObjectsInState', - 'type': 'gauge', - }, - 'SharedCatalogStateApplicationThreads': { - 'name': 'metrics.SharedCatalogStateApplicationThreads', - 'type': 'gauge', - }, - 'SharedCatalogStateApplicationThreadsActive': { - 'name': 'metrics.SharedCatalogStateApplicationThreadsActive', - 'type': 'gauge', - }, - 'SharedCatalogStateApplicationThreadsScheduled': { - 'name': 'metrics.SharedCatalogStateApplicationThreadsScheduled', - 'type': 'gauge', - }, - 'SharedDatabaseCatalogTablesInLocalDropDetachQueue': { - 'name': 'metrics.SharedDatabaseCatalogTablesInLocalDropDetachQueue', - 'type': 'gauge', - }, - 'SharedMergeTreeAssignedCurrentParts': { - 'name': 'metrics.SharedMergeTreeAssignedCurrentParts', - 'type': 'gauge', - }, - 'SharedMergeTreeCondemnedPartsInKeeper': { - 'name': 'metrics.SharedMergeTreeCondemnedPartsInKeeper', - 'type': 'gauge', - }, - 'SharedMergeTreeFetch': {'name': 'metrics.SharedMergeTreeFetch', 'type': 'gauge'}, - 'SharedMergeTreeOutdatedPartsInKeeper': { - 'name': 'metrics.SharedMergeTreeOutdatedPartsInKeeper', - 'type': 'gauge', - }, - 'SharedMergeTreeThreads': {'name': 'metrics.SharedMergeTreeThreads', 'type': 'gauge'}, - 'SharedMergeTreeThreadsActive': {'name': 'metrics.SharedMergeTreeThreadsActive', 'type': 'gauge'}, - 'SharedMergeTreeThreadsScheduled': {'name': 'metrics.SharedMergeTreeThreadsScheduled', 'type': 'gauge'}, - 'StartupScriptsExecutionState': {'name': 'metrics.StartupScriptsExecutionState', 'type': 'gauge'}, - 'StartupSystemTablesThreads': {'name': 'metrics.StartupSystemTablesThreads', 'type': 'gauge'}, - 'StartupSystemTablesThreadsActive': { - 'name': 'metrics.StartupSystemTablesThreadsActive', - 'type': 'gauge', - }, - 'StartupSystemTablesThreadsScheduled': { - 'name': 'metrics.StartupSystemTablesThreadsScheduled', - 'type': 'gauge', - }, - 'StatelessWorkerThreads': {'name': 'metrics.StatelessWorkerThreads', 'type': 'gauge'}, - 'StatelessWorkerThreadsActive': {'name': 'metrics.StatelessWorkerThreadsActive', 'type': 'gauge'}, - 'StatelessWorkerThreadsScheduled': {'name': 'metrics.StatelessWorkerThreadsScheduled', 'type': 'gauge'}, - 'StorageBufferBytes': {'name': 'metrics.StorageBufferBytes', 'type': 'gauge'}, - 'StorageBufferFlushThreads': {'name': 'metrics.StorageBufferFlushThreads', 'type': 'gauge'}, - 'StorageBufferFlushThreadsActive': {'name': 'metrics.StorageBufferFlushThreadsActive', 'type': 'gauge'}, - 'StorageBufferFlushThreadsScheduled': { - 'name': 'metrics.StorageBufferFlushThreadsScheduled', - 'type': 'gauge', - }, - 'StorageBufferRows': {'name': 'metrics.StorageBufferRows', 'type': 'gauge'}, - 'StorageConnectionsStored': {'name': 'metrics.StorageConnectionsStored', 'type': 'gauge'}, - 'StorageConnectionsTotal': {'name': 'metrics.StorageConnectionsTotal', 'type': 'gauge'}, - 'StorageDistributedThreads': {'name': 'metrics.StorageDistributedThreads', 'type': 'gauge'}, - 'StorageDistributedThreadsActive': {'name': 'metrics.StorageDistributedThreadsActive', 'type': 'gauge'}, - 'StorageDistributedThreadsScheduled': { - 'name': 'metrics.StorageDistributedThreadsScheduled', - 'type': 'gauge', - }, - 'StorageHiveThreads': {'name': 'metrics.StorageHiveThreads', 'type': 'gauge'}, - 'StorageHiveThreadsActive': {'name': 'metrics.StorageHiveThreadsActive', 'type': 'gauge'}, - 'StorageHiveThreadsScheduled': {'name': 'metrics.StorageHiveThreadsScheduled', 'type': 'gauge'}, - 'StorageObjectStorageThreads': {'name': 'metrics.StorageObjectStorageThreads', 'type': 'gauge'}, - 'StorageObjectStorageThreadsActive': { - 'name': 'metrics.StorageObjectStorageThreadsActive', - 'type': 'gauge', - }, - 'StorageObjectStorageThreadsScheduled': { - 'name': 'metrics.StorageObjectStorageThreadsScheduled', - 'type': 'gauge', - }, - 'StorageS3Threads': {'name': 'metrics.StorageS3Threads', 'type': 'gauge'}, - 'StorageS3ThreadsActive': {'name': 'metrics.StorageS3ThreadsActive', 'type': 'gauge'}, - 'StorageS3ThreadsScheduled': {'name': 'metrics.StorageS3ThreadsScheduled', 'type': 'gauge'}, - 'SystemReplicasThreads': {'name': 'metrics.SystemReplicasThreads', 'type': 'gauge'}, - 'SystemReplicasThreadsActive': {'name': 'metrics.SystemReplicasThreadsActive', 'type': 'gauge'}, - 'SystemReplicasThreadsScheduled': {'name': 'metrics.SystemReplicasThreadsScheduled', 'type': 'gauge'}, - 'TCPConnection': {'name': 'metrics.TCPConnection', 'type': 'gauge'}, - 'TablesLoaderBackgroundThreads': {'name': 'metrics.TablesLoaderBackgroundThreads', 'type': 'gauge'}, - 'TablesLoaderBackgroundThreadsActive': { - 'name': 'metrics.TablesLoaderBackgroundThreadsActive', - 'type': 'gauge', - }, - 'TablesLoaderBackgroundThreadsScheduled': { - 'name': 'metrics.TablesLoaderBackgroundThreadsScheduled', - 'type': 'gauge', - }, - 'TablesLoaderForegroundThreads': {'name': 'metrics.TablesLoaderForegroundThreads', 'type': 'gauge'}, - 'TablesLoaderForegroundThreadsActive': { - 'name': 'metrics.TablesLoaderForegroundThreadsActive', - 'type': 'gauge', - }, - 'TablesLoaderForegroundThreadsScheduled': { - 'name': 'metrics.TablesLoaderForegroundThreadsScheduled', - 'type': 'gauge', - }, - 'TablesToDropQueueSize': {'name': 'metrics.TablesToDropQueueSize', 'type': 'gauge'}, - 'TaskTrackerThreads': {'name': 'metrics.TaskTrackerThreads', 'type': 'gauge'}, - 'TaskTrackerThreadsActive': {'name': 'metrics.TaskTrackerThreadsActive', 'type': 'gauge'}, - 'TaskTrackerThreadsScheduled': {'name': 'metrics.TaskTrackerThreadsScheduled', 'type': 'gauge'}, - 'TemporaryFilesForAggregation': {'name': 'metrics.TemporaryFilesForAggregation', 'type': 'gauge'}, - 'TemporaryFilesForJoin': {'name': 'metrics.TemporaryFilesForJoin', 'type': 'gauge'}, - 'TemporaryFilesForMerge': {'name': 'metrics.TemporaryFilesForMerge', 'type': 'gauge'}, - 'TemporaryFilesForSort': {'name': 'metrics.TemporaryFilesForSort', 'type': 'gauge'}, - 'TemporaryFilesUnknown': {'name': 'metrics.TemporaryFilesUnknown', 'type': 'gauge'}, - 'ThreadPoolFSReaderThreads': {'name': 'metrics.ThreadPoolFSReaderThreads', 'type': 'gauge'}, - 'ThreadPoolFSReaderThreadsActive': {'name': 'metrics.ThreadPoolFSReaderThreadsActive', 'type': 'gauge'}, - 'ThreadPoolFSReaderThreadsScheduled': { - 'name': 'metrics.ThreadPoolFSReaderThreadsScheduled', - 'type': 'gauge', - }, - 'ThreadPoolRemoteFSReaderThreads': {'name': 'metrics.ThreadPoolRemoteFSReaderThreads', 'type': 'gauge'}, - 'ThreadPoolRemoteFSReaderThreadsActive': { - 'name': 'metrics.ThreadPoolRemoteFSReaderThreadsActive', - 'type': 'gauge', - }, - 'ThreadPoolRemoteFSReaderThreadsScheduled': { - 'name': 'metrics.ThreadPoolRemoteFSReaderThreadsScheduled', - 'type': 'gauge', - }, - 'ThreadsInOvercommitTracker': {'name': 'metrics.ThreadsInOvercommitTracker', 'type': 'gauge'}, - 'TotalTemporaryFiles': {'name': 'metrics.TotalTemporaryFiles', 'type': 'gauge'}, - 'UncompressedCacheBytes': {'name': 'metrics.UncompressedCacheBytes', 'type': 'gauge'}, - 'UncompressedCacheCells': {'name': 'metrics.UncompressedCacheCells', 'type': 'gauge'}, - 'VectorSimilarityIndexCacheBytes': {'name': 'metrics.VectorSimilarityIndexCacheBytes', 'type': 'gauge'}, - 'VectorSimilarityIndexCacheCells': {'name': 'metrics.VectorSimilarityIndexCacheCells', 'type': 'gauge'}, - 'VersionInteger': {'name': 'metrics.VersionInteger', 'type': 'gauge'}, - 'Write': {'name': 'metrics.Write', 'type': 'gauge'}, - 'ZooKeeperRequest': {'name': 'metrics.ZooKeeperRequest', 'type': 'gauge'}, - 'ZooKeeperSession': {'name': 'metrics.ZooKeeperSession', 'type': 'gauge'}, - 'ZooKeeperWatch': {'name': 'metrics.ZooKeeperWatch', 'type': 'gauge'}, - }, - }, - ], -} diff --git a/clickhouse/datadog_checks/clickhouse/clickhouse.py b/clickhouse/datadog_checks/clickhouse/clickhouse.py index e25a7992aed90..3f5e66416995d 100644 --- a/clickhouse/datadog_checks/clickhouse/clickhouse.py +++ b/clickhouse/datadog_checks/clickhouse/clickhouse.py @@ -69,6 +69,7 @@ def __init__(self, name, init_config, instances): self._error_sanitizer = ErrorSanitizer(self._config.password) self.check_initializations.append(self.validate_config) + self.check_initializations.append(advanced_queries.warm_cache) # Submit health event with config validation result # Tags are now available so health events will include them diff --git a/clickhouse/datadog_checks/clickhouse/data/system_async_metrics.json b/clickhouse/datadog_checks/clickhouse/data/system_async_metrics.json new file mode 100644 index 0000000000000..e23cf217e9c15 --- /dev/null +++ b/clickhouse/datadog_checks/clickhouse/data/system_async_metrics.json @@ -0,0 +1,135 @@ +{ + "name": "system_asynchronous_metrics", + "query": "SELECT value, metric FROM system.asynchronous_metrics", + "value_column": "metric_value", + "match_column": "metric_name", + "prefix": "asynchronous_metrics", + "items": { + "gauge": [ + "AsynchronousHeavyMetricsCalculationTimeSpent", + "AsynchronousHeavyMetricsUpdateInterval", + "AsynchronousMetricsCalculationTimeSpent", + "AsynchronousMetricsUpdateInterval", + "CGroupMaxCPU", + "CGroupMemoryTotal", + "CGroupMemoryUsed", + "CGroupSystemTime", + "CGroupSystemTimeNormalized", + "CGroupUserTime", + "CGroupUserTimeNormalized", + "CompiledExpressionCacheBytes", + "CompiledExpressionCacheCount", + "DictionaryTotalFailedUpdates", + "FilesystemCacheBytes", + "FilesystemCacheCapacity", + "FilesystemCacheFiles", + "FilesystemLogsPathAvailableBytes", + "FilesystemLogsPathAvailableINodes", + "FilesystemLogsPathTotalBytes", + "FilesystemLogsPathTotalINodes", + "FilesystemLogsPathUsedBytes", + "FilesystemLogsPathUsedINodes", + "FilesystemMainPathAvailableBytes", + "FilesystemMainPathAvailableINodes", + "FilesystemMainPathTotalBytes", + "FilesystemMainPathTotalINodes", + "FilesystemMainPathUsedBytes", + "FilesystemMainPathUsedINodes", + "HashTableStatsCacheEntries", + "HashTableStatsCacheHits", + "HashTableStatsCacheMisses", + "IndexMarkCacheBytes", + "IndexMarkCacheFiles", + "IndexUncompressedCacheBytes", + "IndexUncompressedCacheCells", + "Jitter", + "LoadAverage1", + "LoadAverage15", + "LoadAverage5", + "MMapCacheCells", + "MarkCacheBytes", + "MarkCacheFiles", + "MaxPartCountForPartition", + "MemoryCode", + "MemoryDataAndStack", + "MemoryResident", + "MemoryResidentMax", + "MemoryShared", + "MemoryVirtual", + "NetworkTCPReceiveQueue", + "NetworkTCPSocketRemoteAddresses", + "NetworkTCPSockets", + "NetworkTCPTransmitQueue", + "NetworkTCPUnrecoveredRetransmits", + "NumberOfDatabases", + "NumberOfDetachedByUserParts", + "NumberOfDetachedParts", + "NumberOfPendingMutations", + "NumberOfPendingMutationsOverExecutionTime", + "NumberOfStuckMutations", + "NumberOfTables", + "NumberOfTablesSystem", + "OSCPUOverload", + "OSContextSwitches", + "OSGuestNiceTimeNormalized", + "OSGuestTimeNormalized", + "OSIOWaitTimeNormalized", + "OSIdleTimeNormalized", + "OSInterrupts", + "OSIrqTimeNormalized", + "OSMemoryAvailable", + "OSMemoryBuffers", + "OSMemoryCached", + "OSMemoryFreePlusCached", + "OSMemoryFreeWithoutCached", + "OSMemorySwapCached", + "OSMemoryTotal", + "OSNiceTimeNormalized", + "OSOpenFiles", + "OSProcessesBlocked", + "OSProcessesCreated", + "OSProcessesRunning", + "OSSoftIrqTimeNormalized", + "OSStealTimeNormalized", + "OSSystemTimeNormalized", + "OSThreadsRunnable", + "OSThreadsTotal", + "OSUptime", + "OSUserTimeNormalized", + "PageCacheBytes", + "PageCacheCells", + "PageCacheMaxBytes", + "PageCachePinnedBytes", + "PrimaryIndexCacheBytes", + "PrimaryIndexCacheFiles", + "QueryCacheBytes", + "QueryCacheEntries", + "ReplicasMaxAbsoluteDelay", + "ReplicasMaxInsertsInQueue", + "ReplicasMaxMergesInQueue", + "ReplicasMaxQueueSize", + "ReplicasMaxRelativeDelay", + "ReplicasSumInsertsInQueue", + "ReplicasSumMergesInQueue", + "ReplicasSumQueueSize", + "TotalBytesOfMergeTreeTables", + "TotalBytesOfMergeTreeTablesSystem", + "TotalIndexGranularityBytesInMemory", + "TotalIndexGranularityBytesInMemoryAllocated", + "TotalPartsOfMergeTreeTables", + "TotalPartsOfMergeTreeTablesSystem", + "TotalPrimaryKeyBytesInMemory", + "TotalPrimaryKeyBytesInMemoryAllocated", + "TotalRowsOfMergeTreeTables", + "TotalRowsOfMergeTreeTablesSystem", + "TrackedMemory", + "UncompressedCacheBytes", + "UncompressedCacheCells", + "UnreclaimableRSS", + "Uptime", + "VMMaxMapCount", + "VMNumMaps", + "jemalloc.epoch" + ] + } +} diff --git a/clickhouse/datadog_checks/clickhouse/data/system_events.json b/clickhouse/datadog_checks/clickhouse/data/system_events.json new file mode 100644 index 0000000000000..ca6aa30ffbb35 --- /dev/null +++ b/clickhouse/datadog_checks/clickhouse/data/system_events.json @@ -0,0 +1,1052 @@ +{ + "name": "system_events", + "query": "SELECT value, event FROM system.events", + "value_column": "metric_value", + "match_column": "metric_name", + "prefix": "events", + "items": { + "gauge": [ + "DistrCacheGetClient", + "DistrCacheHoldConnections", + "MutationsAppliedOnFlyInAllParts", + "PageCacheBytesUnpinnedRoundedToHugePages", + "PageCacheBytesUnpinnedRoundedToPages", + "PageCacheChunkDataHits", + "PageCacheChunkDataMisses", + "PageCacheChunkDataPartialHits", + "PageCacheChunkMisses", + "PageCacheChunkShared", + "PartsWithAppliedMutationsOnFly" + ], + "monotonic_gauge": [ + "AIORead", + "AIOReadBytes", + "AIOWrite", + "AIOWriteBytes", + "AddressesDiscovered", + "AddressesExpired", + "AddressesMarkedAsFailed", + "AggregationHashTablesInitializedAsTwoLevel", + "AggregationOptimizedEqualRangesOfKeys", + "AggregationPreallocatedElementsInHashTables", + "AnalyzePatchRangesMicroseconds", + "ApplyPatchesMicroseconds", + "ArenaAllocBytes", + "ArenaAllocChunks", + "AsyncInsertBytes", + "AsyncInsertCacheHits", + "AsyncInsertQuery", + "AsyncInsertRows", + "AsyncLoggingConsoleDroppedMessages", + "AsyncLoggingConsoleTotalMessages", + "AsyncLoggingErrorFileLogDroppedMessages", + "AsyncLoggingErrorFileLogTotalMessages", + "AsyncLoggingFileLogDroppedMessages", + "AsyncLoggingFileLogTotalMessages", + "AsyncLoggingSyslogDroppedMessages", + "AsyncLoggingSyslogTotalMessages", + "AsyncLoggingTextLogDroppedMessages", + "AsyncLoggingTextLogTotalMessages", + "AsynchronousReaderIgnoredBytes", + "AzureCommitBlockList", + "AzureCopyObject", + "AzureCreateContainer", + "AzureDeleteObjects", + "AzureGetObject", + "AzureGetProperties", + "AzureGetRequestThrottlerCount", + "AzureListObjects", + "AzurePutRequestThrottlerCount", + "AzureReadRequestsCount", + "AzureReadRequestsErrors", + "AzureReadRequestsRedirects", + "AzureReadRequestsThrottling", + "AzureStageBlock", + "AzureUpload", + "AzureWriteRequestsCount", + "AzureWriteRequestsErrors", + "AzureWriteRequestsRedirects", + "AzureWriteRequestsThrottling", + "BackgroundLoadingMarksTasks", + "BackupLockFileReads", + "BackupReadLocalBytesToCalculateChecksums", + "BackupReadLocalFilesToCalculateChecksums", + "BackupReadRemoteBytesToCalculateChecksums", + "BackupReadRemoteFilesToCalculateChecksums", + "BackupThrottlerBytes", + "BackupsOpenedForRead", + "BackupsOpenedForUnlock", + "BackupsOpenedForWrite", + "BuildPatchesJoinMicroseconds", + "BuildPatchesMergeMicroseconds", + "CacheWarmerBytesDownloaded", + "CacheWarmerDataPartsDownloaded", + "CachedReadBufferCacheWriteBytes", + "CachedReadBufferPredownloadedBytes", + "CachedReadBufferReadFromCacheBytes", + "CachedReadBufferReadFromCacheHits", + "CachedReadBufferReadFromCacheMisses", + "CachedReadBufferReadFromSourceBytes", + "CachedWriteBufferCacheWriteBytes", + "CannotRemoveEphemeralNode", + "CannotWriteToWriteBufferDiscard", + "CompileExpressionsBytes", + "CompileFunction", + "CompiledFunctionExecute", + "CompressedReadBufferBlocks", + "CompressedReadBufferBytes", + "CompressedReadBufferChecksumDoesntMatch", + "CompressedReadBufferChecksumDoesntMatchSingleBitMismatch", + "ConcurrencyControlDownscales", + "ConcurrencyControlPreemptions", + "ConcurrencyControlQueriesDelayed", + "ConcurrencyControlSlotsAcquired", + "ConcurrencyControlSlotsAcquiredNonCompeting", + "ConcurrencyControlSlotsDelayed", + "ConcurrencyControlSlotsGranted", + "ConcurrencyControlUpscales", + "ConcurrentQuerySlotsAcquired", + "ContextLock", + "CoordinatedMergesMergeAssignmentRequest", + "CoordinatedMergesMergeAssignmentResponse", + "CoordinatedMergesMergeCoordinatorLockStateExclusivelyCount", + "CoordinatedMergesMergeCoordinatorLockStateForShareCount", + "CoordinatedMergesMergeCoordinatorUpdateCount", + "CoordinatedMergesMergeWorkerUpdateCount", + "CreatedLogEntryForMerge", + "CreatedLogEntryForMutation", + "CreatedReadBufferDirectIO", + "CreatedReadBufferDirectIOFailed", + "CreatedReadBufferMMap", + "CreatedReadBufferMMapFailed", + "CreatedReadBufferOrdinary", + "DNSError", + "DataAfterMutationDiffersFromReplica", + "DefaultImplementationForNullsRows", + "DefaultImplementationForNullsRowsWithNulls", + "DelayedInserts", + "DelayedMutations", + "DeltaLakePartitionPrunedFiles", + "DictCacheKeysExpired", + "DictCacheKeysHit", + "DictCacheKeysNotFound", + "DictCacheKeysRequested", + "DictCacheKeysRequestedFound", + "DictCacheKeysRequestedMiss", + "DictCacheRequests", + "DirectorySync", + "DiskAzureCommitBlockList", + "DiskAzureCopyObject", + "DiskAzureCreateContainer", + "DiskAzureDeleteObjects", + "DiskAzureGetObject", + "DiskAzureGetProperties", + "DiskAzureGetRequestThrottlerCount", + "DiskAzureListObjects", + "DiskAzurePutRequestThrottlerCount", + "DiskAzureReadRequestsCount", + "DiskAzureReadRequestsErrors", + "DiskAzureReadRequestsRedirects", + "DiskAzureReadRequestsThrottling", + "DiskAzureStageBlock", + "DiskAzureUpload", + "DiskAzureWriteRequestsCount", + "DiskAzureWriteRequestsErrors", + "DiskAzureWriteRequestsRedirects", + "DiskAzureWriteRequestsThrottling", + "DiskConnectionsCreated", + "DiskConnectionsErrors", + "DiskConnectionsExpired", + "DiskConnectionsPreserved", + "DiskConnectionsReset", + "DiskConnectionsReused", + "DiskPlainRewritableAzureDirectoryCreated", + "DiskPlainRewritableAzureDirectoryRemoved", + "DiskPlainRewritableLegacyLayoutDiskCount", + "DiskPlainRewritableLocalDirectoryCreated", + "DiskPlainRewritableLocalDirectoryRemoved", + "DiskPlainRewritableS3DirectoryCreated", + "DiskPlainRewritableS3DirectoryRemoved", + "DiskS3AbortMultipartUpload", + "DiskS3CompleteMultipartUpload", + "DiskS3CopyObject", + "DiskS3CreateMultipartUpload", + "DiskS3DeleteObjects", + "DiskS3GetObject", + "DiskS3GetObjectAttributes", + "DiskS3GetRequestThrottlerCount", + "DiskS3HeadObject", + "DiskS3ListObjects", + "DiskS3PutObject", + "DiskS3PutRequestThrottlerCount", + "DiskS3ReadRequestAttempts", + "DiskS3ReadRequestRetryableErrors", + "DiskS3ReadRequestsCount", + "DiskS3ReadRequestsErrors", + "DiskS3ReadRequestsRedirects", + "DiskS3ReadRequestsThrottling", + "DiskS3UploadPart", + "DiskS3UploadPartCopy", + "DiskS3WriteRequestAttempts", + "DiskS3WriteRequestRetryableErrors", + "DiskS3WriteRequestsCount", + "DiskS3WriteRequestsErrors", + "DiskS3WriteRequestsRedirects", + "DiskS3WriteRequestsThrottling", + "DistrCacheConnectAttempts", + "DistrCacheDataPacketsBytes", + "DistrCacheHashRingRebuilds", + "DistrCacheIgnoredBytesWhileWaitingProfileEvents", + "DistrCacheMakeRequestErrors", + "DistrCacheOpenedConnections", + "DistrCacheOpenedConnectionsBypassingPool", + "DistrCachePackets", + "DistrCachePacketsBytes", + "DistrCacheRangeChange", + "DistrCacheRangeResetBackward", + "DistrCacheRangeResetForward", + "DistrCacheReadBytesFromCache", + "DistrCacheReadBytesFromFallbackBuffer", + "DistrCacheReadErrors", + "DistrCacheReceiveResponseErrors", + "DistrCacheReconnectsAfterTimeout", + "DistrCacheRegistryUpdates", + "DistrCacheReusedConnections", + "DistrCacheServerAckRequestPackets", + "DistrCacheServerCachedReadBufferCacheHits", + "DistrCacheServerCachedReadBufferCacheMisses", + "DistrCacheServerContinueRequestPackets", + "DistrCacheServerCredentialsRefresh", + "DistrCacheServerEndRequestPackets", + "DistrCacheServerNewS3CachedClients", + "DistrCacheServerReceivedCredentialsRefreshPackets", + "DistrCacheServerReusedS3CachedClients", + "DistrCacheServerStartRequestPackets", + "DistrCacheServerSwitches", + "DistrCacheServerUpdates", + "DistrCacheUnusedDataPacketsBytes", + "DistrCacheUnusedPackets", + "DistrCacheUnusedPacketsBufferAllocations", + "DistrCacheUnusedPacketsBytes", + "DistributedAsyncInsertionFailures", + "DistributedConnectionFailAtAll", + "DistributedConnectionFailTry", + "DistributedConnectionMissingTable", + "DistributedConnectionReconnectCount", + "DistributedConnectionSkipReadOnlyReplica", + "DistributedConnectionStaleReplica", + "DistributedConnectionTries", + "DistributedConnectionUsable", + "DistributedDelayedInserts", + "DistributedRejectedInserts", + "DistributedSyncInsertionTimeoutExceeded", + "DuplicatedInsertedBlocks", + "EngineFileLikeReadFiles", + "ExecuteShellCommand", + "ExternalAggregationCompressedBytes", + "ExternalAggregationMerge", + "ExternalAggregationUncompressedBytes", + "ExternalAggregationWritePart", + "ExternalDataSourceLocalCacheReadBytes", + "ExternalJoinCompressedBytes", + "ExternalJoinMerge", + "ExternalJoinUncompressedBytes", + "ExternalJoinWritePart", + "ExternalProcessingCompressedBytesTotal", + "ExternalProcessingFilesTotal", + "ExternalProcessingUncompressedBytesTotal", + "ExternalSortCompressedBytes", + "ExternalSortMerge", + "ExternalSortUncompressedBytes", + "ExternalSortWritePart", + "FailedAsyncInsertQuery", + "FailedInsertQuery", + "FailedQuery", + "FailedSelectQuery", + "FileOpen", + "FileSegmentFailToIncreasePriority", + "FileSegmentUsedBytes", + "FileSync", + "FilesystemCacheBackgroundDownloadQueuePush", + "FilesystemCacheBackgroundEvictedBytes", + "FilesystemCacheBackgroundEvictedFileSegments", + "FilesystemCacheCreatedKeyDirectories", + "FilesystemCacheEvictedBytes", + "FilesystemCacheEvictedFileSegments", + "FilesystemCacheEvictedFileSegmentsDuringPriorityIncrease", + "FilesystemCacheEvictionReusedIterator", + "FilesystemCacheEvictionSkippedEvictingFileSegments", + "FilesystemCacheEvictionSkippedFileSegments", + "FilesystemCacheEvictionTries", + "FilesystemCacheFailToReserveSpaceBecauseOfCacheResize", + "FilesystemCacheFailToReserveSpaceBecauseOfLockContention", + "FilesystemCacheFailedEvictionCandidates", + "FilesystemCacheFreeSpaceKeepingThreadRun", + "FilesystemCacheHoldFileSegments", + "FilesystemCacheReserveAttempts", + "FilesystemCacheUnusedHoldFileSegments", + "FilterTransformPassedBytes", + "FilterTransformPassedRows", + "FunctionExecute", + "GWPAsanAllocateFailed", + "GWPAsanAllocateSuccess", + "GWPAsanFree", + "GatheredColumns", + "GlobalThreadPoolExpansions", + "GlobalThreadPoolJobs", + "GlobalThreadPoolShrinks", + "HTTPConnectionsCreated", + "HTTPConnectionsErrors", + "HTTPConnectionsExpired", + "HTTPConnectionsPreserved", + "HTTPConnectionsReset", + "HTTPConnectionsReused", + "HTTPServerConnectionsClosed", + "HTTPServerConnectionsCreated", + "HTTPServerConnectionsExpired", + "HTTPServerConnectionsPreserved", + "HTTPServerConnectionsReset", + "HTTPServerConnectionsReused", + "HardPageFaults", + "HashJoinPreallocatedElementsInHashTables", + "HedgedRequestsChangeReplica", + "IOBufferAllocBytes", + "IOBufferAllocs", + "IOUringCQEsCompleted", + "IOUringCQEsFailed", + "IOUringSQEsResubmitsAsync", + "IOUringSQEsResubmitsSync", + "IOUringSQEsSubmitted", + "IcebergMetadataFilesCacheHits", + "IcebergMetadataFilesCacheMisses", + "IcebergMetadataFilesCacheWeightLost", + "IcebergMetadataReturnedObjectInfos", + "IcebergMinMaxIndexPrunedFiles", + "IcebergPartitionPrunedFiles", + "IcebergPartitionPrunnedFiles", + "IcebergTrivialCountOptimizationApplied", + "IcebergVersionHintUsed", + "IgnoredColdParts", + "IndexBinarySearchAlgorithm", + "IndexGenericExclusionSearchAlgorithm", + "InitialQuery", + "InsertQueriesWithSubqueries", + "InsertQuery", + "InsertedBytes", + "InsertedCompactParts", + "InsertedRows", + "InsertedWideParts", + "InterfaceHTTPReceiveBytes", + "InterfaceHTTPSendBytes", + "InterfaceInterserverReceiveBytes", + "InterfaceInterserverSendBytes", + "InterfaceMySQLReceiveBytes", + "InterfaceMySQLSendBytes", + "InterfaceNativeReceiveBytes", + "InterfaceNativeSendBytes", + "InterfacePostgreSQLReceiveBytes", + "InterfacePostgreSQLSendBytes", + "InterfacePrometheusReceiveBytes", + "InterfacePrometheusSendBytes", + "JoinBuildTableRowCount", + "JoinProbeTableRowCount", + "JoinResultRowCount", + "KafkaBackgroundReads", + "KafkaCommitFailures", + "KafkaCommits", + "KafkaConsumerErrors", + "KafkaDirectReads", + "KafkaMessagesFailed", + "KafkaMessagesPolled", + "KafkaMessagesProduced", + "KafkaMessagesRead", + "KafkaProducerErrors", + "KafkaProducerFlushes", + "KafkaRebalanceAssignments", + "KafkaRebalanceErrors", + "KafkaRebalanceRevocations", + "KafkaRowsRead", + "KafkaRowsRejected", + "KafkaRowsWritten", + "KafkaWrites", + "KeeperBatchMaxCount", + "KeeperBatchMaxTotalSize", + "KeeperCheckRequest", + "KeeperCommits", + "KeeperCommitsFailed", + "KeeperCreateRequest", + "KeeperExistsRequest", + "KeeperGetRequest", + "KeeperListRequest", + "KeeperLogsEntryReadFromCommitCache", + "KeeperLogsEntryReadFromFile", + "KeeperLogsEntryReadFromLatestCache", + "KeeperLogsPrefetchedEntries", + "KeeperMultiReadRequest", + "KeeperMultiRequest", + "KeeperPacketsReceived", + "KeeperPacketsSent", + "KeeperReadSnapshot", + "KeeperReconfigRequest", + "KeeperRemoveRequest", + "KeeperRequestRejectedDueToSoftMemoryLimitCount", + "KeeperRequestTotal", + "KeeperSaveSnapshot", + "KeeperSetRequest", + "KeeperSnapshotApplys", + "KeeperSnapshotApplysFailed", + "KeeperSnapshotCreations", + "KeeperSnapshotCreationsFailed", + "LoadedDataParts", + "LoadedMarksCount", + "LoadedMarksFiles", + "LoadedMarksMemoryBytes", + "LoadedPrimaryIndexBytes", + "LoadedPrimaryIndexFiles", + "LoadedPrimaryIndexRows", + "LoadingMarksTasksCanceled", + "LocalReadThrottlerBytes", + "LocalThreadPoolExpansions", + "LocalThreadPoolShrinks", + "LocalWriteThrottlerBytes", + "LogDebug", + "LogError", + "LogFatal", + "LogInfo", + "LogTest", + "LogTrace", + "LogWarning", + "MMappedFileCacheHits", + "MMappedFileCacheMisses", + "MainConfigLoads", + "MarkCacheEvictedBytes", + "MarkCacheEvictedFiles", + "MarkCacheEvictedMarks", + "MarkCacheHits", + "MarkCacheMisses", + "MemoryAllocatorPurge", + "MemoryWorkerRun", + "Merge", + "MergeSourceParts", + "MergeTreeAllRangesAnnouncementsSent", + "MergeTreeDataProjectionWriterBlocks", + "MergeTreeDataProjectionWriterBlocksAlreadySorted", + "MergeTreeDataProjectionWriterCompressedBytes", + "MergeTreeDataProjectionWriterRows", + "MergeTreeDataProjectionWriterUncompressedBytes", + "MergeTreeDataWriterBlocks", + "MergeTreeDataWriterBlocksAlreadySorted", + "MergeTreeDataWriterCompressedBytes", + "MergeTreeDataWriterRows", + "MergeTreeDataWriterUncompressedBytes", + "MergeTreeReadTaskRequestsReceived", + "MergeTreeReadTaskRequestsSent", + "MergedColumns", + "MergedIntoCompactParts", + "MergedIntoWideParts", + "MergedRows", + "MergedUncompressedBytes", + "MergerMutatorPartsInRangesForMergeCount", + "MergerMutatorRangesForMergeCount", + "MergerMutatorSelectRangePartsCount", + "MergesThrottlerBytes", + "MetadataFromKeeperBackgroundCleanupErrors", + "MetadataFromKeeperBackgroundCleanupObjects", + "MetadataFromKeeperBackgroundCleanupTransactions", + "MetadataFromKeeperCacheHit", + "MetadataFromKeeperCacheMiss", + "MetadataFromKeeperCleanupTransactionCommit", + "MetadataFromKeeperCleanupTransactionCommitRetry", + "MetadataFromKeeperIndividualOperations", + "MetadataFromKeeperOperations", + "MetadataFromKeeperReconnects", + "MetadataFromKeeperTransactionCommit", + "MetadataFromKeeperTransactionCommitRetry", + "MetadataFromKeeperUpdateCacheOneLevel", + "MutatedRows", + "MutatedUncompressedBytes", + "MutationAffectedRowsUpperBound", + "MutationAllPartColumns", + "MutationCreatedEmptyParts", + "MutationSomePartColumns", + "MutationTotalParts", + "MutationUntouchedParts", + "MutationsAppliedOnFlyInAllReadTasks", + "MutationsThrottlerBytes", + "NetworkReceiveBytes", + "NetworkSendBytes", + "NotCreatedLogEntryForMerge", + "NotCreatedLogEntryForMutation", + "OSReadBytes", + "OSReadChars", + "OSWriteBytes", + "OSWriteChars", + "ObjectStorageQueueCancelledFiles", + "ObjectStorageQueueCommitRequests", + "ObjectStorageQueueExceptionsDuringInsert", + "ObjectStorageQueueExceptionsDuringRead", + "ObjectStorageQueueFailedFiles", + "ObjectStorageQueueFailedToBatchSetProcessing", + "ObjectStorageQueueFilteredFiles", + "ObjectStorageQueueInsertIterations", + "ObjectStorageQueueListedFiles", + "ObjectStorageQueueProcessedFiles", + "ObjectStorageQueueProcessedRows", + "ObjectStorageQueueReadBytes", + "ObjectStorageQueueReadFiles", + "ObjectStorageQueueReadRows", + "ObjectStorageQueueRemovedObjects", + "ObjectStorageQueueSuccessfulCommits", + "ObjectStorageQueueTrySetProcessingFailed", + "ObjectStorageQueueTrySetProcessingRequests", + "ObjectStorageQueueTrySetProcessingSucceeded", + "ObjectStorageQueueUnsuccessfulCommits", + "ObsoleteReplicatedParts", + "OpenedFileCacheHits", + "OpenedFileCacheMisses", + "OverflowAny", + "OverflowBreak", + "OverflowThrow", + "PageCacheHits", + "PageCacheMisses", + "PageCacheOvercommitResize", + "PageCacheReadBytes", + "PageCacheResized", + "PageCacheWeightLost", + "ParallelReplicasAvailableCount", + "ParallelReplicasDeniedRequests", + "ParallelReplicasNumRequests", + "ParallelReplicasQueryCount", + "ParallelReplicasReadAssignedForStealingMarks", + "ParallelReplicasReadAssignedMarks", + "ParallelReplicasReadMarks", + "ParallelReplicasReadUnassignedMarks", + "ParallelReplicasUnavailableCount", + "ParallelReplicasUsedCount", + "ParquetDecodingTaskBatches", + "ParquetDecodingTasks", + "ParquetPrunedRowGroups", + "ParquetReadRowGroups", + "PatchesAcquireLockMicroseconds", + "PatchesAcquireLockTries", + "PatchesAppliedInAllReadTasks", + "PatchesJoinAppliedInAllReadTasks", + "PatchesMergeAppliedInAllReadTasks", + "PatchesReadUncompressedBytes", + "PerfAlignmentFaults", + "PerfBranchInstructions", + "PerfBranchMisses", + "PerfBusCycles", + "PerfCPUClock", + "PerfCPUCycles", + "PerfCPUMigrations", + "PerfCacheMisses", + "PerfCacheReferences", + "PerfContextSwitches", + "PerfDataTLBMisses", + "PerfDataTLBReferences", + "PerfEmulationFaults", + "PerfInstructionTLBMisses", + "PerfInstructionTLBReferences", + "PerfInstructions", + "PerfLocalMemoryMisses", + "PerfLocalMemoryReferences", + "PerfMinEnabledRunningTime", + "PerfMinEnabledTime", + "PerfRefCPUCycles", + "PerfStalledCyclesBackend", + "PerfStalledCyclesFrontend", + "PerfTaskClock", + "PolygonsAddedToPool", + "PolygonsInPoolAllocatedBytes", + "PreferredWarmedUnmergedParts", + "PrimaryIndexCacheHits", + "PrimaryIndexCacheMisses", + "QueriesWithSubqueries", + "Query", + "QueryBackupThrottlerBytes", + "QueryCacheHits", + "QueryCacheMisses", + "QueryConditionCacheHits", + "QueryConditionCacheMisses", + "QueryLocalReadThrottlerBytes", + "QueryLocalWriteThrottlerBytes", + "QueryMaskingRulesMatch", + "QueryMemoryLimitExceeded", + "QueryPreempted", + "QueryProfilerConcurrencyOverruns", + "QueryProfilerErrors", + "QueryProfilerRuns", + "QueryProfilerSignalOverruns", + "QueryRemoteReadThrottlerBytes", + "QueryRemoteWriteThrottlerBytes", + "RWLockAcquiredReadLocks", + "RWLockAcquiredWriteLocks", + "ReadBackoff", + "ReadBufferFromAzureBytes", + "ReadBufferFromAzureRequestsErrors", + "ReadBufferFromFileDescriptorRead", + "ReadBufferFromFileDescriptorReadBytes", + "ReadBufferFromFileDescriptorReadFailed", + "ReadBufferFromS3Bytes", + "ReadBufferFromS3RequestsErrors", + "ReadBufferSeekCancelConnection", + "ReadCompressedBytes", + "ReadPatchesMicroseconds", + "ReadTaskRequestsReceived", + "ReadTaskRequestsSent", + "ReadTasksWithAppliedMutationsOnFly", + "ReadTasksWithAppliedPatches", + "ReadWriteBufferFromHTTPBytes", + "ReadWriteBufferFromHTTPRequestsSent", + "RefreshableViewLockTableRetry", + "RefreshableViewRefreshFailed", + "RefreshableViewRefreshSuccess", + "RefreshableViewSyncReplicaRetry", + "RefreshableViewSyncReplicaSuccess", + "RegexpLocalCacheHit", + "RegexpLocalCacheMiss", + "RegexpWithMultipleNeedlesCreated", + "RegexpWithMultipleNeedlesGlobalCacheHit", + "RegexpWithMultipleNeedlesGlobalCacheMiss", + "RejectedInserts", + "RejectedLightweightUpdates", + "RejectedMutations", + "RemoteFSBuffers", + "RemoteFSCancelledPrefetches", + "RemoteFSLazySeeks", + "RemoteFSPrefetchedBytes", + "RemoteFSPrefetchedReads", + "RemoteFSPrefetches", + "RemoteFSSeeks", + "RemoteFSSeeksWithReset", + "RemoteFSUnprefetchedBytes", + "RemoteFSUnprefetchedReads", + "RemoteFSUnusedPrefetches", + "RemoteReadThrottlerBytes", + "RemoteWriteThrottlerBytes", + "ReplicaPartialShutdown", + "ReplicatedCoveredPartsInZooKeeperOnStart", + "ReplicatedDataLoss", + "ReplicatedPartChecks", + "ReplicatedPartChecksFailed", + "ReplicatedPartFailedFetches", + "ReplicatedPartFetches", + "ReplicatedPartFetchesOfMerged", + "ReplicatedPartMerges", + "ReplicatedPartMutations", + "RestorePartsSkippedBytes", + "RestorePartsSkippedFiles", + "RowsReadByMainReader", + "RowsReadByPrewhereReaders", + "S3AbortMultipartUpload", + "S3Clients", + "S3CompleteMultipartUpload", + "S3CopyObject", + "S3CreateMultipartUpload", + "S3DeleteObjects", + "S3GetObject", + "S3GetObjectAttributes", + "S3GetRequestThrottlerCount", + "S3HeadObject", + "S3ListObjects", + "S3PutObject", + "S3PutRequestThrottlerCount", + "S3ReadRequestAttempts", + "S3ReadRequestRetryableErrors", + "S3ReadRequestsCount", + "S3ReadRequestsErrors", + "S3ReadRequestsRedirects", + "S3ReadRequestsThrottling", + "S3UploadPart", + "S3UploadPartCopy", + "S3WriteRequestAttempts", + "S3WriteRequestRetryableErrors", + "S3WriteRequestsCount", + "S3WriteRequestsErrors", + "S3WriteRequestsRedirects", + "S3WriteRequestsThrottling", + "ScalarSubqueriesCacheMiss", + "ScalarSubqueriesGlobalCacheHit", + "ScalarSubqueriesLocalCacheHit", + "SchedulerIOReadBytes", + "SchedulerIOReadRequests", + "SchedulerIOWriteBytes", + "SchedulerIOWriteRequests", + "SchemaInferenceCacheEvictions", + "SchemaInferenceCacheHits", + "SchemaInferenceCacheInvalidations", + "SchemaInferenceCacheMisses", + "SchemaInferenceCacheNumRowsHits", + "SchemaInferenceCacheNumRowsMisses", + "SchemaInferenceCacheSchemaHits", + "SchemaInferenceCacheSchemaMisses", + "Seek", + "SelectQueriesWithPrimaryKeyUsage", + "SelectQueriesWithSubqueries", + "SelectQuery", + "SelectedBytes", + "SelectedMarks", + "SelectedMarksTotal", + "SelectedParts", + "SelectedPartsTotal", + "SelectedRanges", + "SelectedRows", + "SharedDatabaseCatalogFailedToApplyState", + "SharedMergeTreeCondemnedPartsKillRequest", + "SharedMergeTreeCondemnedPartsLockConfict", + "SharedMergeTreeCondemnedPartsRemoved", + "SharedMergeTreeDataPartsFetchAttempt", + "SharedMergeTreeDataPartsFetchFromPeer", + "SharedMergeTreeDataPartsFetchFromPeerMicroseconds", + "SharedMergeTreeDataPartsFetchFromS3", + "SharedMergeTreeGetPartsBatchToLoadMicroseconds", + "SharedMergeTreeHandleBlockingParts", + "SharedMergeTreeHandleBlockingPartsMicroseconds", + "SharedMergeTreeHandleFetchPartsMicroseconds", + "SharedMergeTreeHandleOutdatedParts", + "SharedMergeTreeHandleOutdatedPartsMicroseconds", + "SharedMergeTreeLoadChecksumAndIndexesMicroseconds", + "SharedMergeTreeMergeMutationAssignmentAttempt", + "SharedMergeTreeMergeMutationAssignmentFailedWithConflict", + "SharedMergeTreeMergeMutationAssignmentFailedWithNothingToDo", + "SharedMergeTreeMergeMutationAssignmentSuccessful", + "SharedMergeTreeMergePartsMovedToCondemned", + "SharedMergeTreeMergePartsMovedToOudated", + "SharedMergeTreeMergeSelectingTaskMicroseconds", + "SharedMergeTreeMetadataCacheHintLoadedFromCache", + "SharedMergeTreeOptimizeAsync", + "SharedMergeTreeOptimizeSync", + "SharedMergeTreeOutdatedPartsConfirmationInvocations", + "SharedMergeTreeOutdatedPartsConfirmationRequest", + "SharedMergeTreeOutdatedPartsHTTPRequest", + "SharedMergeTreeOutdatedPartsHTTPResponse", + "SharedMergeTreeScheduleDataProcessingJob", + "SharedMergeTreeScheduleDataProcessingJobMicroseconds", + "SharedMergeTreeScheduleDataProcessingJobNothingToScheduled", + "SharedMergeTreeTryUpdateDiskMetadataCacheForPartMicroseconds", + "SharedMergeTreeVirtualPartsUpdates", + "SharedMergeTreeVirtualPartsUpdatesByLeader", + "SharedMergeTreeVirtualPartsUpdatesForMergesOrStatus", + "SharedMergeTreeVirtualPartsUpdatesFromPeer", + "SharedMergeTreeVirtualPartsUpdatesFromZooKeeper", + "SharedMergeTreeVirtualPartsUpdatesLeaderFailedElection", + "SharedMergeTreeVirtualPartsUpdatesLeaderSuccessfulElection", + "SharedMergeTreeVirtualPartsUpdatesPeerNotFound", + "SleepFunctionCalls", + "SlowRead", + "SoftPageFaults", + "StorageBufferErrorOnFlush", + "StorageBufferFlush", + "StorageBufferPassedAllMinThresholds", + "StorageBufferPassedBytesFlushThreshold", + "StorageBufferPassedBytesMaxThreshold", + "StorageBufferPassedRowsFlushThreshold", + "StorageBufferPassedRowsMaxThreshold", + "StorageBufferPassedTimeFlushThreshold", + "StorageBufferPassedTimeMaxThreshold", + "StorageConnectionsCreated", + "StorageConnectionsErrors", + "StorageConnectionsExpired", + "StorageConnectionsPreserved", + "StorageConnectionsReset", + "StorageConnectionsReused", + "SuspendSendingQueryToShard", + "SystemLogErrorOnFlush", + "TableFunctionExecute", + "ThreadPoolReaderPageCacheHit", + "ThreadPoolReaderPageCacheHitBytes", + "ThreadPoolReaderPageCacheMiss", + "ThreadPoolReaderPageCacheMissBytes", + "ThreadpoolReaderReadBytes", + "ThreadpoolReaderSubmit", + "ThreadpoolReaderSubmitReadSynchronously", + "ThreadpoolReaderSubmitReadSynchronouslyBytes", + "TinyS3Clients", + "USearchAddComputedDistances", + "USearchAddCount", + "USearchAddVisitedMembers", + "USearchSearchComputedDistances", + "USearchSearchCount", + "USearchSearchVisitedMembers", + "UncompressedCacheHits", + "UncompressedCacheMisses", + "UncompressedCacheWeightLost", + "VectorSimilarityIndexCacheHits", + "VectorSimilarityIndexCacheMisses", + "VectorSimilarityIndexCacheWeightLost", + "WriteBufferFromFileDescriptorWrite", + "WriteBufferFromFileDescriptorWriteBytes", + "WriteBufferFromFileDescriptorWriteFailed", + "WriteBufferFromHTTPBytes", + "WriteBufferFromHTTPRequestsSent", + "WriteBufferFromS3Bytes", + "WriteBufferFromS3RequestsErrors", + "ZooKeeperBytesReceived", + "ZooKeeperBytesSent", + "ZooKeeperCheck", + "ZooKeeperClose", + "ZooKeeperCreate", + "ZooKeeperExists", + "ZooKeeperGet", + "ZooKeeperGetACL", + "ZooKeeperHardwareExceptions", + "ZooKeeperInit", + "ZooKeeperList", + "ZooKeeperMulti", + "ZooKeeperMultiRead", + "ZooKeeperMultiWrite", + "ZooKeeperOtherExceptions", + "ZooKeeperReconfig", + "ZooKeeperRemove", + "ZooKeeperSet", + "ZooKeeperSync", + "ZooKeeperTransactions", + "ZooKeeperUserExceptions", + "ZooKeeperWatchResponse" + ], + "temporal_percent": { + "AggregatingSortedMilliseconds": "millisecond", + "AsyncLoaderWaitMicroseconds": "microsecond", + "AsynchronousReadWaitMicroseconds": "microsecond", + "AsynchronousRemoteReadWaitMicroseconds": "microsecond", + "AzureGetRequestThrottlerSleepMicroseconds": "microsecond", + "AzurePutRequestThrottlerSleepMicroseconds": "microsecond", + "AzureReadMicroseconds": "microsecond", + "AzureWriteMicroseconds": "microsecond", + "BackupEntriesCollectorForTablesDataMicroseconds": "microsecond", + "BackupEntriesCollectorMicroseconds": "microsecond", + "BackupEntriesCollectorRunPostTasksMicroseconds": "microsecond", + "BackupPreparingFileInfosMicroseconds": "microsecond", + "BackupReadMetadataMicroseconds": "microsecond", + "BackupThrottlerSleepMicroseconds": "microsecond", + "BackupWriteMetadataMicroseconds": "microsecond", + "CachedReadBufferCacheWriteMicroseconds": "microsecond", + "CachedReadBufferCreateBufferMicroseconds": "microsecond", + "CachedReadBufferReadFromCacheMicroseconds": "microsecond", + "CachedReadBufferReadFromSourceMicroseconds": "microsecond", + "CachedReadBufferWaitReadBufferMicroseconds": "microsecond", + "CachedWriteBufferCacheWriteMicroseconds": "microsecond", + "CoalescingSortedMilliseconds": "millisecond", + "CollapsingSortedMilliseconds": "millisecond", + "CommonBackgroundExecutorTaskCancelMicroseconds": "microsecond", + "CommonBackgroundExecutorTaskExecuteStepMicroseconds": "microsecond", + "CommonBackgroundExecutorTaskResetMicroseconds": "microsecond", + "CommonBackgroundExecutorWaitMicroseconds": "microsecond", + "CompileExpressionsMicroseconds": "microsecond", + "CompressedReadBufferChecksumDoesntMatchMicroseconds": "microsecond", + "ConcurrencyControlPreemptedMicroseconds": "microsecond", + "ConcurrencyControlWaitMicroseconds": "microsecond", + "ConcurrentQueryWaitMicroseconds": "microsecond", + "ConnectionPoolIsFullMicroseconds": "microsecond", + "ContextLockWaitMicroseconds": "microsecond", + "CoordinatedMergesMergeAssignmentRequestMicroseconds": "microsecond", + "CoordinatedMergesMergeAssignmentResponseMicroseconds": "microsecond", + "CoordinatedMergesMergeCoordinatorFetchMetadataMicroseconds": "microsecond", + "CoordinatedMergesMergeCoordinatorFilterMicroseconds": "microsecond", + "CoordinatedMergesMergeCoordinatorLockStateExclusivelyMicroseconds": "microsecond", + "CoordinatedMergesMergeCoordinatorLockStateForShareMicroseconds": "microsecond", + "CoordinatedMergesMergeCoordinatorSelectMergesMicroseconds": "microsecond", + "CoordinatedMergesMergeCoordinatorUpdateMicroseconds": "microsecond", + "CoordinatedMergesMergeWorkerUpdateMicroseconds": "microsecond", + "DelayedInsertsMilliseconds": "millisecond", + "DelayedMutationsMilliseconds": "millisecond", + "DictCacheLockReadNs": "nanosecond", + "DictCacheLockWriteNs": "nanosecond", + "DictCacheRequestTimeNs": "nanosecond", + "DirectorySyncElapsedMicroseconds": "microsecond", + "DiskAzureGetRequestThrottlerSleepMicroseconds": "microsecond", + "DiskAzurePutRequestThrottlerSleepMicroseconds": "microsecond", + "DiskAzureReadMicroseconds": "microsecond", + "DiskAzureWriteMicroseconds": "microsecond", + "DiskConnectionsElapsedMicroseconds": "microsecond", + "DiskReadElapsedMicroseconds": "microsecond", + "DiskS3GetRequestThrottlerSleepMicroseconds": "microsecond", + "DiskS3PutRequestThrottlerSleepMicroseconds": "microsecond", + "DiskS3ReadMicroseconds": "microsecond", + "DiskS3WriteMicroseconds": "microsecond", + "DiskWriteElapsedMicroseconds": "microsecond", + "DistrCacheConnectMicroseconds": "microsecond", + "DistrCacheFallbackReadMicroseconds": "microsecond", + "DistrCacheGetClientMicroseconds": "microsecond", + "DistrCacheGetResponseMicroseconds": "microsecond", + "DistrCacheLockRegistryMicroseconds": "microsecond", + "DistrCacheNextImplMicroseconds": "microsecond", + "DistrCachePrecomputeRangesMicroseconds": "microsecond", + "DistrCacheReadMicroseconds": "microsecond", + "DistrCacheRegistryUpdateMicroseconds": "microsecond", + "DistrCacheServerProcessRequestMicroseconds": "microsecond", + "DistrCacheStartRangeMicroseconds": "microsecond", + "DistributedDelayedInsertsMilliseconds": "millisecond", + "FetchBackgroundExecutorTaskCancelMicroseconds": "microsecond", + "FetchBackgroundExecutorTaskExecuteStepMicroseconds": "microsecond", + "FetchBackgroundExecutorTaskResetMicroseconds": "microsecond", + "FetchBackgroundExecutorWaitMicroseconds": "microsecond", + "FileSegmentCacheWriteMicroseconds": "microsecond", + "FileSegmentCompleteMicroseconds": "microsecond", + "FileSegmentHolderCompleteMicroseconds": "microsecond", + "FileSegmentLockMicroseconds": "microsecond", + "FileSegmentPredownloadMicroseconds": "microsecond", + "FileSegmentReadMicroseconds": "microsecond", + "FileSegmentRemoveMicroseconds": "microsecond", + "FileSegmentUseMicroseconds": "microsecond", + "FileSegmentWaitMicroseconds": "microsecond", + "FileSegmentWaitReadBufferMicroseconds": "microsecond", + "FileSegmentWriteMicroseconds": "microsecond", + "FileSyncElapsedMicroseconds": "microsecond", + "FilesystemCacheEvictMicroseconds": "microsecond", + "FilesystemCacheFreeSpaceKeepingThreadWorkMilliseconds": "millisecond", + "FilesystemCacheGetMicroseconds": "microsecond", + "FilesystemCacheGetOrSetMicroseconds": "microsecond", + "FilesystemCacheLoadMetadataMicroseconds": "microsecond", + "FilesystemCacheLockCacheMicroseconds": "microsecond", + "FilesystemCacheLockKeyMicroseconds": "microsecond", + "FilesystemCacheLockMetadataMicroseconds": "microsecond", + "FilesystemCacheReserveMicroseconds": "microsecond", + "FilteringMarksWithPrimaryKeyMicroseconds": "microsecond", + "FilteringMarksWithSecondaryKeysMicroseconds": "microsecond", + "GatheringColumnMilliseconds": "millisecond", + "GlobalThreadPoolJobWaitTimeMicroseconds": "microsecond", + "GlobalThreadPoolLockWaitMicroseconds": "microsecond", + "GlobalThreadPoolThreadCreationMicroseconds": "microsecond", + "HTTPConnectionsElapsedMicroseconds": "microsecond", + "IcebergIteratorInitializationMicroseconds": "microsecond", + "IcebergMetadataReadWaitTimeMicroseconds": "microsecond", + "IcebergMetadataUpdateMicroseconds": "microsecond", + "InsertQueryTimeMicroseconds": "microsecond", + "KeeperCommitWaitElapsedMicroseconds": "microsecond", + "KeeperLatency": "millisecond", + "KeeperPreprocessElapsedMicroseconds": "microsecond", + "KeeperProcessElapsedMicroseconds": "microsecond", + "KeeperStorageLockWaitMicroseconds": "microsecond", + "KeeperTotalElapsedMicroseconds": "microsecond", + "LoadedDataPartsMicroseconds": "microsecond", + "LocalReadThrottlerSleepMicroseconds": "microsecond", + "LocalThreadPoolBusyMicroseconds": "microsecond", + "LocalThreadPoolJobWaitTimeMicroseconds": "microsecond", + "LocalThreadPoolJobs": "microsecond", + "LocalThreadPoolLockWaitMicroseconds": "microsecond", + "LocalThreadPoolThreadCreationMicroseconds": "microsecond", + "LocalWriteThrottlerSleepMicroseconds": "microsecond", + "LoggerElapsedNanoseconds": "nanosecond", + "MemoryAllocatorPurgeTimeMicroseconds": "microsecond", + "MemoryOvercommitWaitTimeMicroseconds": "microsecond", + "MemoryWorkerRunElapsedMicroseconds": "microsecond", + "MergeExecuteMilliseconds": "millisecond", + "MergeHorizontalStageExecuteMilliseconds": "millisecond", + "MergeHorizontalStageTotalMilliseconds": "millisecond", + "MergeMutateBackgroundExecutorTaskCancelMicroseconds": "microsecond", + "MergeMutateBackgroundExecutorTaskExecuteStepMicroseconds": "microsecond", + "MergeMutateBackgroundExecutorTaskResetMicroseconds": "microsecond", + "MergeMutateBackgroundExecutorWaitMicroseconds": "microsecond", + "MergePrewarmStageExecuteMilliseconds": "millisecond", + "MergePrewarmStageTotalMilliseconds": "millisecond", + "MergeProjectionStageExecuteMilliseconds": "millisecond", + "MergeProjectionStageTotalMilliseconds": "millisecond", + "MergeTotalMilliseconds": "millisecond", + "MergeTreeAllRangesAnnouncementsSentElapsedMicroseconds": "microsecond", + "MergeTreeDataProjectionWriterMergingBlocksMicroseconds": "microsecond", + "MergeTreeDataProjectionWriterSortingBlocksMicroseconds": "microsecond", + "MergeTreeDataWriterMergingBlocksMicroseconds": "microsecond", + "MergeTreeDataWriterProjectionsCalculationMicroseconds": "microsecond", + "MergeTreeDataWriterSkipIndicesCalculationMicroseconds": "microsecond", + "MergeTreeDataWriterSortingBlocksMicroseconds": "microsecond", + "MergeTreeDataWriterStatisticsCalculationMicroseconds": "microsecond", + "MergeTreePrefetchedReadPoolInit": "microsecond", + "MergeTreeReadTaskRequestsSentElapsedMicroseconds": "microsecond", + "MergeVerticalStageExecuteMilliseconds": "millisecond", + "MergeVerticalStageTotalMilliseconds": "millisecond", + "MergerMutatorPrepareRangesForMergeElapsedMicroseconds": "microsecond", + "MergerMutatorSelectPartsForMergeElapsedMicroseconds": "microsecond", + "MergerMutatorsGetPartsForMergeElapsedMicroseconds": "microsecond", + "MergesThrottlerSleepMicroseconds": "microsecond", + "MergingSortedMilliseconds": "millisecond", + "MetadataFromKeeperCacheUpdateMicroseconds": "microsecond", + "MetadataFromKeeperIndividualOperationsMicroseconds": "microsecond", + "MoveBackgroundExecutorTaskCancelMicroseconds": "microsecond", + "MoveBackgroundExecutorTaskExecuteStepMicroseconds": "microsecond", + "MoveBackgroundExecutorTaskResetMicroseconds": "microsecond", + "MoveBackgroundExecutorWaitMicroseconds": "microsecond", + "MutateTaskProjectionsCalculationMicroseconds": "microsecond", + "MutationExecuteMilliseconds": "millisecond", + "MutationTotalMilliseconds": "millisecond", + "MutationsThrottlerSleepMicroseconds": "microsecond", + "NetworkReceiveElapsedMicroseconds": "microsecond", + "NetworkSendElapsedMicroseconds": "microsecond", + "OSCPUVirtualTimeMicroseconds": "microsecond", + "OSCPUWaitMicroseconds": "microsecond", + "OSIOWaitMicroseconds": "microsecond", + "ObjectStorageQueueCleanupMaxSetSizeOrTTLMicroseconds": "microsecond", + "ObjectStorageQueueLockLocalFileStatusesMicroseconds": "microsecond", + "ObjectStorageQueuePullMicroseconds": "microsecond", + "OpenedFileCacheMicroseconds": "microsecond", + "OtherQueryTimeMicroseconds": "microsecond", + "ParallelReplicasAnnouncementMicroseconds": "microsecond", + "ParallelReplicasCollectingOwnedSegmentsMicroseconds": "microsecond", + "ParallelReplicasHandleAnnouncementMicroseconds": "microsecond", + "ParallelReplicasHandleRequestMicroseconds": "microsecond", + "ParallelReplicasProcessingPartsMicroseconds": "microsecond", + "ParallelReplicasReadRequestMicroseconds": "microsecond", + "ParallelReplicasStealingByHashMicroseconds": "microsecond", + "ParallelReplicasStealingLeftoversMicroseconds": "microsecond", + "ParquetFetchWaitTimeMicroseconds": "microsecond", + "PartsLockHoldMicroseconds": "microsecond", + "PartsLockWaitMicroseconds": "microsecond", + "QueryBackupThrottlerSleepMicroseconds": "microsecond", + "QueryLocalReadThrottlerSleepMicroseconds": "microsecond", + "QueryLocalWriteThrottlerSleepMicroseconds": "microsecond", + "QueryRemoteReadThrottlerSleepMicroseconds": "microsecond", + "QueryRemoteWriteThrottlerSleepMicroseconds": "microsecond", + "QueryTimeMicroseconds": "microsecond", + "RWLockReadersWaitMilliseconds": "millisecond", + "RWLockWritersWaitMilliseconds": "millisecond", + "ReadBufferFromAzureInitMicroseconds": "microsecond", + "ReadBufferFromAzureMicroseconds": "microsecond", + "ReadBufferFromS3InitMicroseconds": "microsecond", + "ReadBufferFromS3Microseconds": "microsecond", + "ReadTaskRequestsSentElapsedMicroseconds": "microsecond", + "RealTimeMicroseconds": "microsecond", + "RemoteReadThrottlerSleepMicroseconds": "microsecond", + "RemoteWriteThrottlerSleepMicroseconds": "microsecond", + "ReplacingSortedMilliseconds": "millisecond", + "S3GetRequestThrottlerSleepMicroseconds": "microsecond", + "S3PutRequestThrottlerSleepMicroseconds": "microsecond", + "S3QueueSetFileFailedMicroseconds": "microsecond", + "S3QueueSetFileProcessedMicroseconds": "microsecond", + "S3QueueSetFileProcessingMicroseconds": "microsecond", + "S3ReadMicroseconds": "microsecond", + "S3WriteMicroseconds": "microsecond", + "SchedulerIOReadWaitMicroseconds": "microsecond", + "SchedulerIOWriteWaitMicroseconds": "microsecond", + "SelectQueryTimeMicroseconds": "microsecond", + "ServerStartupMilliseconds": "millisecond", + "SharedDatabaseCatalogStateApplicationMicroseconds": "microsecond", + "SharedMergeTreeVirtualPartsUpdateMicroseconds": "microsecond", + "SharedMergeTreeVirtualPartsUpdatesFromPeerMicroseconds": "microsecond", + "SharedMergeTreeVirtualPartsUpdatesFromZooKeeperMicroseconds": "microsecond", + "SleepFunctionElapsedMicroseconds": "microsecond", + "SleepFunctionMicroseconds": "microsecond", + "StorageBufferLayerLockReadersWaitMilliseconds": "millisecond", + "StorageBufferLayerLockWritersWaitMilliseconds": "millisecond", + "StorageConnectionsElapsedMicroseconds": "microsecond", + "SummingSortedMilliseconds": "millisecond", + "SynchronousReadWaitMicroseconds": "microsecond", + "SynchronousRemoteReadWaitMicroseconds": "microsecond", + "SystemTimeMicroseconds": "microsecond", + "ThreadPoolReaderPageCacheHitElapsedMicroseconds": "microsecond", + "ThreadPoolReaderPageCacheMissElapsedMicroseconds": "microsecond", + "ThreadpoolReaderPrepareMicroseconds": "microsecond", + "ThreadpoolReaderSubmitLookupInCacheMicroseconds": "microsecond", + "ThreadpoolReaderSubmitReadSynchronouslyMicroseconds": "microsecond", + "ThreadpoolReaderTaskMicroseconds": "microsecond", + "ThrottlerSleepMicroseconds": "microsecond", + "UserTimeMicroseconds": "microsecond", + "VersionedCollapsingSortedMilliseconds": "millisecond", + "WaitMarksLoadMicroseconds": "microsecond", + "WaitPrefetchTaskMicroseconds": "microsecond", + "WriteBufferFromS3Microseconds": "microsecond", + "WriteBufferFromS3WaitInflightLimitMicroseconds": "microsecond", + "ZooKeeperWaitMicroseconds": "microsecond" + } + } +} diff --git a/clickhouse/datadog_checks/clickhouse/data/system_metrics.json b/clickhouse/datadog_checks/clickhouse/data/system_metrics.json new file mode 100644 index 0000000000000..116fc08c266d0 --- /dev/null +++ b/clickhouse/datadog_checks/clickhouse/data/system_metrics.json @@ -0,0 +1,441 @@ +{ + "name": "system_metrics", + "query": "SELECT value, metric FROM system.metrics", + "value_column": "metric_value", + "match_column": "metric_name", + "prefix": "metrics", + "items": { + "gauge": [ + "ActiveTimersInQueryProfiler", + "AddressesActive", + "AddressesBanned", + "AggregatorThreads", + "AggregatorThreadsActive", + "AggregatorThreadsScheduled", + "AsyncInsertCacheSize", + "AsynchronousInsertQueueBytes", + "AsynchronousInsertQueueSize", + "AsynchronousInsertThreads", + "AsynchronousInsertThreadsActive", + "AsynchronousInsertThreadsScheduled", + "AsynchronousReadWait", + "AttachedDatabase", + "AttachedDictionary", + "AttachedReplicatedTable", + "AttachedTable", + "AttachedView", + "AvroSchemaCacheBytes", + "AvroSchemaCacheCells", + "AvroSchemaRegistryCacheBytes", + "AvroSchemaRegistryCacheCells", + "AzureRequests", + "BackgroundBufferFlushSchedulePoolSize", + "BackgroundBufferFlushSchedulePoolTask", + "BackgroundCommonPoolSize", + "BackgroundCommonPoolTask", + "BackgroundDistributedSchedulePoolSize", + "BackgroundDistributedSchedulePoolTask", + "BackgroundFetchesPoolSize", + "BackgroundFetchesPoolTask", + "BackgroundMergesAndMutationsPoolSize", + "BackgroundMergesAndMutationsPoolTask", + "BackgroundMessageBrokerSchedulePoolSize", + "BackgroundMessageBrokerSchedulePoolTask", + "BackgroundMovePoolSize", + "BackgroundMovePoolTask", + "BackgroundSchedulePoolSize", + "BackgroundSchedulePoolTask", + "BackupsIOThreads", + "BackupsIOThreadsActive", + "BackupsIOThreadsScheduled", + "BackupsThreads", + "BackupsThreadsActive", + "BackupsThreadsScheduled", + "BrokenDisks", + "BrokenDistributedBytesToInsert", + "BrokenDistributedFilesToInsert", + "BuildVectorSimilarityIndexThreads", + "BuildVectorSimilarityIndexThreadsActive", + "BuildVectorSimilarityIndexThreadsScheduled", + "CacheDetachedFileSegments", + "CacheDictionaryThreads", + "CacheDictionaryThreadsActive", + "CacheDictionaryThreadsScheduled", + "CacheDictionaryUpdateQueueBatches", + "CacheDictionaryUpdateQueueKeys", + "CacheFileSegments", + "CacheWarmerBytesInProgress", + "CompiledExpressionCacheBytes", + "CompiledExpressionCacheCount", + "Compressing", + "CompressionThread", + "CompressionThreadActive", + "CompressionThreadScheduled", + "ConcurrencyControlAcquired", + "ConcurrencyControlAcquiredNonCompeting", + "ConcurrencyControlPreempted", + "ConcurrencyControlScheduled", + "ConcurrencyControlSoftLimit", + "ConcurrentHashJoinPoolThreads", + "ConcurrentHashJoinPoolThreadsActive", + "ConcurrentHashJoinPoolThreadsScheduled", + "ConcurrentQueryAcquired", + "ConcurrentQueryScheduled", + "ContextLockWait", + "CoordinatedMergesCoordinatorAssignedMerges", + "CoordinatedMergesCoordinatorRunningMerges", + "CoordinatedMergesWorkerAssignedMerges", + "CreatedTimersInQueryProfiler", + "DDLWorkerThreads", + "DDLWorkerThreadsActive", + "DDLWorkerThreadsScheduled", + "DNSAddressesCacheBytes", + "DNSAddressesCacheSize", + "DNSHostsCacheBytes", + "DNSHostsCacheSize", + "DWARFReaderThreads", + "DWARFReaderThreadsActive", + "DWARFReaderThreadsScheduled", + "DatabaseBackupThreads", + "DatabaseBackupThreadsActive", + "DatabaseBackupThreadsScheduled", + "DatabaseCatalogThreads", + "DatabaseCatalogThreadsActive", + "DatabaseCatalogThreadsScheduled", + "DatabaseOnDiskThreads", + "DatabaseOnDiskThreadsActive", + "DatabaseOnDiskThreadsScheduled", + "DatabaseReplicatedCreateTablesThreads", + "DatabaseReplicatedCreateTablesThreadsActive", + "DatabaseReplicatedCreateTablesThreadsScheduled", + "Decompressing", + "DelayedInserts", + "DestroyAggregatesThreads", + "DestroyAggregatesThreadsActive", + "DestroyAggregatesThreadsScheduled", + "DictCacheRequests", + "DiskConnectionsStored", + "DiskConnectionsTotal", + "DiskObjectStorageAsyncThreads", + "DiskObjectStorageAsyncThreadsActive", + "DiskPlainRewritableAzureDirectoryMapSize", + "DiskPlainRewritableAzureFileCount", + "DiskPlainRewritableAzureUniqueFileNamesCount", + "DiskPlainRewritableLocalDirectoryMapSize", + "DiskPlainRewritableLocalFileCount", + "DiskPlainRewritableLocalUniqueFileNamesCount", + "DiskPlainRewritableS3DirectoryMapSize", + "DiskPlainRewritableS3FileCount", + "DiskPlainRewritableS3UniqueFileNamesCount", + "DiskS3NoSuchKeyErrors", + "DiskSpaceReservedForMerge", + "DistrCacheAllocatedConnections", + "DistrCacheBorrowedConnections", + "DistrCacheOpenedConnections", + "DistrCacheReadRequests", + "DistrCacheRegisteredServers", + "DistrCacheRegisteredServersCurrentAZ", + "DistrCacheServerConnections", + "DistrCacheServerRegistryConnections", + "DistrCacheServerS3CachedClients", + "DistrCacheUsedConnections", + "DistrCacheWriteRequests", + "DistributedBytesToInsert", + "DistributedFilesToInsert", + "DistributedInsertThreads", + "DistributedInsertThreadsActive", + "DistributedInsertThreadsScheduled", + "DistributedSend", + "DropDistributedCacheThreads", + "DropDistributedCacheThreadsActive", + "DropDistributedCacheThreadsScheduled", + "EphemeralNode", + "FilesystemCacheDelayedCleanupElements", + "FilesystemCacheDownloadQueueElements", + "FilesystemCacheElements", + "FilesystemCacheHoldFileSegments", + "FilesystemCacheKeys", + "FilesystemCacheReadBuffers", + "FilesystemCacheReserveThreads", + "FilesystemCacheSize", + "FilesystemCacheSizeLimit", + "FilteringMarksWithPrimaryKey", + "FilteringMarksWithSecondaryKeys", + "FormatParsingThreads", + "FormatParsingThreadsActive", + "FormatParsingThreadsScheduled", + "GlobalThread", + "GlobalThreadActive", + "GlobalThreadScheduled", + "HTTPConnection", + "HTTPConnectionsStored", + "HTTPConnectionsTotal", + "HashedDictionaryThreads", + "HashedDictionaryThreadsActive", + "HashedDictionaryThreadsScheduled", + "HiveFilesCacheBytes", + "HiveFilesCacheFiles", + "HiveMetadataFilesCacheBytes", + "HiveMetadataFilesCacheFiles", + "IDiskCopierThreads", + "IDiskCopierThreadsActive", + "IDiskCopierThreadsScheduled", + "IOPrefetchThreads", + "IOPrefetchThreadsActive", + "IOPrefetchThreadsScheduled", + "IOThreads", + "IOThreadsActive", + "IOThreadsScheduled", + "IOUringInFlightEvents", + "IOUringPendingEvents", + "IOWriterThreads", + "IOWriterThreadsActive", + "IOWriterThreadsScheduled", + "IcebergCatalogThreads", + "IcebergCatalogThreadsActive", + "IcebergCatalogThreadsScheduled", + "IcebergMetadataFilesCacheBytes", + "IcebergMetadataFilesCacheFiles", + "IndexMarkCacheBytes", + "IndexMarkCacheFiles", + "IndexUncompressedCacheBytes", + "IndexUncompressedCacheCells", + "InterserverConnection", + "IsServerShuttingDown", + "KafkaAssignedPartitions", + "KafkaBackgroundReads", + "KafkaConsumers", + "KafkaConsumersInUse", + "KafkaConsumersWithAssignment", + "KafkaLibrdkafkaThreads", + "KafkaProducers", + "KafkaWrites", + "KeeperAliveConnections", + "KeeperOutstandingRequests", + "LicenseRemainingSeconds", + "LocalThread", + "LocalThreadActive", + "LocalThreadScheduled", + "MMapCacheCells", + "MMappedFileBytes", + "MMappedFiles", + "MarkCacheBytes", + "MarkCacheFiles", + "MarksLoaderThreads", + "MarksLoaderThreadsActive", + "MarksLoaderThreadsScheduled", + "MaxDDLEntryID", + "MaxPushedDDLEntryID", + "MemoryTracking", + "MemoryTrackingUncorrected", + "Merge", + "MergeJoinBlocksCacheBytes", + "MergeJoinBlocksCacheCount", + "MergeParts", + "MergeTreeAllRangesAnnouncementsSent", + "MergeTreeBackgroundExecutorThreads", + "MergeTreeBackgroundExecutorThreadsActive", + "MergeTreeBackgroundExecutorThreadsScheduled", + "MergeTreeDataSelectExecutorThreads", + "MergeTreeDataSelectExecutorThreadsActive", + "MergeTreeDataSelectExecutorThreadsScheduled", + "MergeTreeFetchPartitionThreads", + "MergeTreeFetchPartitionThreadsActive", + "MergeTreeFetchPartitionThreadsScheduled", + "MergeTreeOutdatedPartsLoaderThreads", + "MergeTreeOutdatedPartsLoaderThreadsActive", + "MergeTreeOutdatedPartsLoaderThreadsScheduled", + "MergeTreePartsCleanerThreads", + "MergeTreePartsCleanerThreadsActive", + "MergeTreePartsCleanerThreadsScheduled", + "MergeTreePartsLoaderThreads", + "MergeTreePartsLoaderThreadsActive", + "MergeTreePartsLoaderThreadsScheduled", + "MergeTreeReadTaskRequestsSent", + "MergeTreeSubcolumnsReaderThreads", + "MergeTreeSubcolumnsReaderThreadsActive", + "MergeTreeSubcolumnsReaderThreadsScheduled", + "MergeTreeUnexpectedPartsLoaderThreads", + "MergeTreeUnexpectedPartsLoaderThreadsActive", + "MergeTreeUnexpectedPartsLoaderThreadsScheduled", + "MergesMutationsMemoryTracking", + "MetadataFromKeeperCacheObjects", + "Move", + "MySQLConnection", + "NetworkReceive", + "NetworkSend", + "ObjectStorageAzureThreads", + "ObjectStorageAzureThreadsActive", + "ObjectStorageAzureThreadsScheduled", + "ObjectStorageQueueRegisteredServers", + "ObjectStorageQueueShutdownThreads", + "ObjectStorageQueueShutdownThreadsActive", + "ObjectStorageQueueShutdownThreadsScheduled", + "ObjectStorageS3Threads", + "ObjectStorageS3ThreadsActive", + "ObjectStorageS3ThreadsScheduled", + "OpenFileForRead", + "OpenFileForWrite", + "OutdatedPartsLoadingThreads", + "OutdatedPartsLoadingThreadsActive", + "OutdatedPartsLoadingThreadsScheduled", + "PageCacheBytes", + "PageCacheCells", + "ParallelCompressedWriteBufferThreads", + "ParallelCompressedWriteBufferWait", + "ParallelFormattingOutputFormatThreads", + "ParallelFormattingOutputFormatThreadsActive", + "ParallelFormattingOutputFormatThreadsScheduled", + "ParallelParsingInputFormatThreads", + "ParallelParsingInputFormatThreadsActive", + "ParallelParsingInputFormatThreadsScheduled", + "ParallelWithQueryActiveThreads", + "ParallelWithQueryScheduledThreads", + "ParallelWithQueryThreads", + "ParquetDecoderIOThreads", + "ParquetDecoderIOThreadsActive", + "ParquetDecoderIOThreadsScheduled", + "ParquetDecoderThreads", + "ParquetDecoderThreadsActive", + "ParquetDecoderThreadsScheduled", + "ParquetEncoderThreads", + "ParquetEncoderThreadsActive", + "ParquetEncoderThreadsScheduled", + "PartMutation", + "PartsActive", + "PartsCommitted", + "PartsCompact", + "PartsDeleteOnDestroy", + "PartsDeleting", + "PartsOutdated", + "PartsPreActive", + "PartsPreCommitted", + "PartsTemporary", + "PartsWide", + "PendingAsyncInsert", + "PolygonDictionaryThreads", + "PolygonDictionaryThreadsActive", + "PolygonDictionaryThreadsScheduled", + "PostgreSQLConnection", + "PrimaryIndexCacheBytes", + "PrimaryIndexCacheFiles", + "Query", + "QueryCacheBytes", + "QueryCacheEntries", + "QueryConditionCacheBytes", + "QueryConditionCacheEntries", + "QueryPipelineExecutorThreads", + "QueryPipelineExecutorThreadsActive", + "QueryPipelineExecutorThreadsScheduled", + "QueryPreempted", + "QueryThread", + "RWLockActiveReaders", + "RWLockActiveWriters", + "RWLockWaitingReaders", + "RWLockWaitingWriters", + "Read", + "ReadTaskRequestsSent", + "ReadonlyDisks", + "ReadonlyReplica", + "RefreshableViews", + "RefreshingViews", + "RemoteRead", + "ReplicatedChecks", + "ReplicatedFetch", + "ReplicatedSend", + "RestartReplicaThreads", + "RestartReplicaThreadsActive", + "RestartReplicaThreadsScheduled", + "RestoreThreads", + "RestoreThreadsActive", + "RestoreThreadsScheduled", + "Revision", + "S3Requests", + "SchedulerIOReadScheduled", + "SchedulerIOWriteScheduled", + "SendExternalTables", + "SendScalars", + "SharedCatalogDropDetachLocalTablesErrors", + "SharedCatalogDropLocalThreads", + "SharedCatalogDropLocalThreadsActive", + "SharedCatalogDropLocalThreadsScheduled", + "SharedCatalogDropZooKeeperThreads", + "SharedCatalogDropZooKeeperThreadsActive", + "SharedCatalogDropZooKeeperThreadsScheduled", + "SharedCatalogNumberOfObjectsInState", + "SharedCatalogStateApplicationThreads", + "SharedCatalogStateApplicationThreadsActive", + "SharedCatalogStateApplicationThreadsScheduled", + "SharedDatabaseCatalogTablesInLocalDropDetachQueue", + "SharedMergeTreeAssignedCurrentParts", + "SharedMergeTreeCondemnedPartsInKeeper", + "SharedMergeTreeFetch", + "SharedMergeTreeOutdatedPartsInKeeper", + "SharedMergeTreeThreads", + "SharedMergeTreeThreadsActive", + "SharedMergeTreeThreadsScheduled", + "StartupScriptsExecutionState", + "StartupSystemTablesThreads", + "StartupSystemTablesThreadsActive", + "StartupSystemTablesThreadsScheduled", + "StatelessWorkerThreads", + "StatelessWorkerThreadsActive", + "StatelessWorkerThreadsScheduled", + "StorageBufferBytes", + "StorageBufferFlushThreads", + "StorageBufferFlushThreadsActive", + "StorageBufferFlushThreadsScheduled", + "StorageBufferRows", + "StorageConnectionsStored", + "StorageConnectionsTotal", + "StorageDistributedThreads", + "StorageDistributedThreadsActive", + "StorageDistributedThreadsScheduled", + "StorageHiveThreads", + "StorageHiveThreadsActive", + "StorageHiveThreadsScheduled", + "StorageObjectStorageThreads", + "StorageObjectStorageThreadsActive", + "StorageObjectStorageThreadsScheduled", + "StorageS3Threads", + "StorageS3ThreadsActive", + "StorageS3ThreadsScheduled", + "SystemReplicasThreads", + "SystemReplicasThreadsActive", + "SystemReplicasThreadsScheduled", + "TCPConnection", + "TablesLoaderBackgroundThreads", + "TablesLoaderBackgroundThreadsActive", + "TablesLoaderBackgroundThreadsScheduled", + "TablesLoaderForegroundThreads", + "TablesLoaderForegroundThreadsActive", + "TablesLoaderForegroundThreadsScheduled", + "TablesToDropQueueSize", + "TaskTrackerThreads", + "TaskTrackerThreadsActive", + "TaskTrackerThreadsScheduled", + "TemporaryFilesForAggregation", + "TemporaryFilesForJoin", + "TemporaryFilesForMerge", + "TemporaryFilesForSort", + "TemporaryFilesUnknown", + "ThreadPoolFSReaderThreads", + "ThreadPoolFSReaderThreadsActive", + "ThreadPoolFSReaderThreadsScheduled", + "ThreadPoolRemoteFSReaderThreads", + "ThreadPoolRemoteFSReaderThreadsActive", + "ThreadPoolRemoteFSReaderThreadsScheduled", + "ThreadsInOvercommitTracker", + "TotalTemporaryFiles", + "UncompressedCacheBytes", + "UncompressedCacheCells", + "VectorSimilarityIndexCacheBytes", + "VectorSimilarityIndexCacheCells", + "VersionInteger", + "Write", + "ZooKeeperRequest", + "ZooKeeperSession", + "ZooKeeperWatch" + ] + } +} diff --git a/clickhouse/scripts/generate_metrics.py b/clickhouse/scripts/generate_metrics.py index d9557f5ab6635..91a688ad6c588 100644 --- a/clickhouse/scripts/generate_metrics.py +++ b/clickhouse/scripts/generate_metrics.py @@ -7,11 +7,12 @@ import collections import csv import itertools +import json import os import pprint import re from dataclasses import dataclass -from enum import Enum, StrEnum +from enum import StrEnum from typing import Iterable import requests @@ -21,7 +22,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) TEMPLATES_DIR = os.path.join(HERE, 'templates') INTEGRATION_DIR = os.path.join(HERE, '..') -QUERIES_DIR = os.path.join(INTEGRATION_DIR, 'datadog_checks', 'clickhouse', 'advanced_queries') +DATA_DIR = os.path.join(INTEGRATION_DIR, 'datadog_checks', 'clickhouse', 'data') TESTS_DIR = os.path.join(INTEGRATION_DIR, 'tests') METADATAFILE_PATH = os.path.join(INTEGRATION_DIR, 'metadata.csv') METADATAFILE_LEGACY_PATH = os.path.join(INTEGRATION_DIR, 'metadata-legacy.csv') @@ -74,35 +75,56 @@ class MetricKind(StrEnum): @dataclass -class Template: +class FileTemplate: source_path: str target_path: str +@dataclass +class QuerySpec: + """Per-system-table parameters for the compact JSON output.""" + + name: str + query: str + prefix: str + target_path: str + match_column: str = 'metric_name' + value_column: str = 'metric_value' + + @dataclass class MetricsGenerator: kind: MetricKind - template: Template + query_spec: QuerySpec is_optional: bool = False -class Templates(Enum): - QUERY_ASYNC_METRICS = Template( - source_path='system_async_metrics.tpl', - target_path=os.path.join(QUERIES_DIR, 'system_async_metrics.py'), - ) - QUERY_EVENTS = Template( - source_path='system_events.tpl', - target_path=os.path.join(QUERIES_DIR, 'system_events.py'), - ) - QUERY_METRICS = Template( - source_path='system_metrics.tpl', - target_path=os.path.join(QUERIES_DIR, 'system_metrics.py'), - ) - TESTS_METRICS = Template( - source_path='tests_metrics.tpl', - target_path=os.path.join(TESTS_DIR, 'advanced_metrics.py'), - ) +QUERY_SPECS = { + MetricKind.ASYNC_METRICS: QuerySpec( + name='system_asynchronous_metrics', + query='SELECT value, metric FROM system.asynchronous_metrics', + prefix='asynchronous_metrics', + target_path=os.path.join(DATA_DIR, 'system_async_metrics.json'), + ), + MetricKind.EVENTS: QuerySpec( + name='system_events', + query='SELECT value, event FROM system.events', + prefix='events', + target_path=os.path.join(DATA_DIR, 'system_events.json'), + ), + MetricKind.METRICS: QuerySpec( + name='system_metrics', + query='SELECT value, metric FROM system.metrics', + prefix='metrics', + target_path=os.path.join(DATA_DIR, 'system_metrics.json'), + ), +} + + +TESTS_METRICS_TEMPLATE = FileTemplate( + source_path='tests_metrics.tpl', + target_path=os.path.join(TESTS_DIR, 'advanced_metrics.py'), +) def versions() -> list[str]: @@ -128,7 +150,7 @@ def write_file(file, contents, encoding='utf-8'): f.write(contents) -def generate_queries_file(template: Template, config: dict): +def generate_queries_file(template: FileTemplate, config: dict): source_path = os.path.join(TEMPLATES_DIR, template.source_path) if not os.path.exists(source_path): print(f'Unknown template file: {source_path}') @@ -177,20 +199,6 @@ def type(self) -> str: def scale(self) -> str | None: return self.metric_type_info()[1] - def get_query_item(self) -> str: - metric_type, scale = self.metric_type_info() - - metric_scale = '' - if scale is not None: - metric_scale = ", 'scale': '{scale}'".format(scale=scale) - - return "'{metric}': {{'name': '{metric_name}', 'type': '{metric_type}'{metric_scale}}}".format( - metric=self.name, - metric_name=self.metric_name(), - metric_type=metric_type, - metric_scale=metric_scale, - ) - def fetch_current_metrics(version: str) -> dict[str, ClickhouseMetric]: raw_metrics = requests.get(SOURCE_URL_CURRENT_METRICS.format(branch=version), timeout=10).text @@ -262,11 +270,55 @@ def clean_description(description: str) -> str: return result -def generate_queries(template: Template, metrics: Iterable[ClickhouseMetric]): - config = { - 'items': ',\n'.join(indent_line(metric.get_query_item(), 16) for metric in sorted(metrics)), +def generate_queries(query_spec: QuerySpec, metrics: Iterable[ClickhouseMetric]): + """Emit the compact JSON for ``query_spec`` from the sorted ``metrics``.""" + items: dict[str, list[str] | dict[str, str]] = {} + for metric in sorted(metrics): + metric_type, scale = metric.metric_type_info() + existing = items.get(metric_type) + if scale is None: + if existing is None: + items[metric_type] = [metric.name] + elif isinstance(existing, list): + existing.append(metric.name) + else: + raise ValueError( + f"metric type {metric_type!r} mixes scaled and unscaled entries; " + f"{metric.name!r} has no scale but earlier entries did" + ) + else: + if existing is None: + items[metric_type] = {metric.name: scale} + elif isinstance(existing, dict): + existing[metric.name] = scale + else: + raise ValueError( + f"metric type {metric_type!r} mixes scaled and unscaled entries; " + f"{metric.name!r} has scale {scale!r} but earlier entries had none" + ) + items_sorted: dict[str, list[str] | dict[str, str]] = {} + for type_name in sorted(items): + group = items[type_name] + if isinstance(group, dict): + items_sorted[type_name] = {key: group[key] for key in sorted(group)} + else: + items_sorted[type_name] = sorted(group) + spec = { + 'name': query_spec.name, + 'query': query_spec.query, + 'value_column': query_spec.value_column, + 'match_column': query_spec.match_column, + 'prefix': query_spec.prefix, + 'items': items_sorted, } - generate_queries_file(template, config) + write_json(query_spec.target_path, spec) + + +def write_json(target_path: str, spec: dict) -> None: + target_dir = os.path.dirname(target_path) + if not os.path.exists(target_dir): + os.makedirs(target_dir) + write_file(target_path, json.dumps(spec, indent=2) + '\n') def generate_metadata_file(metrics: Iterable[ClickhouseMetric]): @@ -458,24 +510,24 @@ def deep_merge(left: dict[str, set[str]], right: dict[str, set[str]]) -> dict[st 'base_version_mapper': printable_consts_mapper(versioned_base_metrics), 'optional_version_mapper': printable_consts_mapper(versioned_optional_metrics, optional=True), } - generate_queries_file(Templates.TESTS_METRICS.value, config) + generate_queries_file(TESTS_METRICS_TEMPLATE, config) def generate(): METRIC_GENERATORS = [ MetricsGenerator( kind=MetricKind.ASYNC_METRICS, - template=Templates.QUERY_ASYNC_METRICS.value, + query_spec=QUERY_SPECS[MetricKind.ASYNC_METRICS], is_optional=True, ), MetricsGenerator( kind=MetricKind.EVENTS, - template=Templates.QUERY_EVENTS.value, + query_spec=QUERY_SPECS[MetricKind.EVENTS], is_optional=True, ), MetricsGenerator( kind=MetricKind.METRICS, - template=Templates.QUERY_METRICS.value, + query_spec=QUERY_SPECS[MetricKind.METRICS], is_optional=False, ), ] @@ -483,11 +535,11 @@ def generate(): all: dict[str, ClickhouseMetric] = {} calculated: list[CalculatedMetrics] = [] - # generate query modules + # generate per-system-table JSON data files for generator in METRIC_GENERATORS: metrics = calculate_metrics(generator) stats[generator.kind] = len(metrics.all) - generate_queries(generator.template, metrics.all.values()) + generate_queries(generator.query_spec, metrics.all.values()) all.update(metrics.all) calculated.append(metrics) diff --git a/clickhouse/scripts/templates/system_async_metrics.tpl b/clickhouse/scripts/templates/system_async_metrics.tpl deleted file mode 100644 index e7d568cc7711b..0000000000000 --- a/clickhouse/scripts/templates/system_async_metrics.tpl +++ /dev/null @@ -1,27 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# This file is autogenerated. -# To change this file you should edit scripts/templates/system_async_metrics.tpl and then run the following command: -# hatch run metrics:generate - -# https://clickhouse.com/docs/operations/system-tables/asynchronous_metrics -SystemAsynchronousMetrics = {{ - 'name': 'system_asynchronous_metrics', - 'query': 'SELECT value, metric FROM system.asynchronous_metrics', - 'columns': [ - {{ - 'name': 'metric_value', - 'type': 'source' - }}, - {{ - 'name': 'metric_name', - 'type': 'match', - 'source': 'metric_value', - 'items': {{ -{items} - }}, - }}, - ], -}} diff --git a/clickhouse/scripts/templates/system_events.tpl b/clickhouse/scripts/templates/system_events.tpl deleted file mode 100644 index 6520897aed02d..0000000000000 --- a/clickhouse/scripts/templates/system_events.tpl +++ /dev/null @@ -1,27 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# This file is autogenerated. -# To change this file you should edit scripts/templates/system_events.tpl and then run the following command: -# hatch run metrics:generate - -# https://clickhouse.com/docs/operations/system-tables/events -SystemEvents = {{ - 'name': 'system_events', - 'query': 'SELECT value, event FROM system.events', - 'columns': [ - {{ - 'name': 'metric_value', - 'type': 'source' - }}, - {{ - 'name': 'metric_name', - 'type': 'match', - 'source': 'metric_value', - 'items': {{ -{items} - }}, - }}, - ], -}} diff --git a/clickhouse/scripts/templates/system_metrics.tpl b/clickhouse/scripts/templates/system_metrics.tpl deleted file mode 100644 index 652562873d7e7..0000000000000 --- a/clickhouse/scripts/templates/system_metrics.tpl +++ /dev/null @@ -1,27 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# This file is autogenerated. -# To change this file you should edit scripts/templates/system_metrics.tpl and then run the following command: -# hatch run metrics:generate - -# https://clickhouse.com/docs/operations/system-tables/metrics -SystemMetrics = {{ - 'name': 'system_metrics', - 'query': 'SELECT value, metric FROM system.metrics', - 'columns': [ - {{ - 'name': 'metric_value', - 'type': 'source' - }}, - {{ - 'name': 'metric_name', - 'type': 'match', - 'source': 'metric_value', - 'items': {{ -{items} - }}, - }}, - ], -}} diff --git a/clickhouse/tests/test_advanced_queries.py b/clickhouse/tests/test_advanced_queries.py new file mode 100644 index 0000000000000..8cc1a2a70268e --- /dev/null +++ b/clickhouse/tests/test_advanced_queries.py @@ -0,0 +1,171 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +"""Tests for the ``advanced_queries`` package.""" + +from __future__ import annotations + +import json + +import pytest + +from datadog_checks.clickhouse import advanced_queries + +MATCH_QUERY_NAMES = ('SystemEvents', 'SystemMetrics', 'SystemAsynchronousMetrics') +ALL_NAMES = (*MATCH_QUERY_NAMES, 'SystemErrors') + + +@pytest.fixture(autouse=True) +def _reset_match_query_cache(): + """Clear the module-level match-query cache so each test sees a fresh load.""" + advanced_queries._match_query_cache.clear() + yield + advanced_queries._match_query_cache.clear() + + +@pytest.fixture +def isolated_data_dir(tmp_path, monkeypatch): + """Redirect ``load_match_query`` to a temporary directory.""" + monkeypatch.setattr(advanced_queries, 'DATA_DIR', str(tmp_path)) + return tmp_path + + +# --------------------------------------------------------------------------- +# Module attribute access (__getattr__ for match queries; literal for SystemErrors) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize('name', ALL_NAMES) +def test_module_attribute_returns_querymanager_shape(name): + spec = getattr(advanced_queries, name) + assert isinstance(spec['name'], str) and spec['name'] + assert isinstance(spec['query'], str) and spec['query'] + assert isinstance(spec['columns'], list) and spec['columns'] + + +def test_module_attribute_caches_match_query_result(): + first = advanced_queries.SystemEvents + second = advanced_queries.SystemEvents + assert first is second + + +def test_unknown_attribute_raises_attribute_error(): + with pytest.raises(AttributeError, match=r"module .* has no attribute 'SystemNonsense'"): + advanced_queries.SystemNonsense # noqa: B018 + + +# --------------------------------------------------------------------------- +# Bulk match queries: load_match_query() + _expand_match_items() +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize('name', MATCH_QUERY_NAMES) +def test_match_query_has_source_and_match_columns(name): + spec = getattr(advanced_queries, name) + source_col, match_col = spec['columns'] + assert source_col == {'name': 'metric_value', 'type': 'source'} + assert match_col['name'] == 'metric_name' + assert match_col['type'] == 'match' + assert match_col['source'] == 'metric_value' + assert isinstance(match_col['items'], dict) + + +@pytest.mark.parametrize('name', MATCH_QUERY_NAMES) +def test_match_query_items_are_alphabetically_sorted(name): + items = getattr(advanced_queries, name)['columns'][1]['items'] + assert list(items) == sorted(items) + + +@pytest.mark.parametrize('name', MATCH_QUERY_NAMES) +def test_match_query_items_carry_name_and_type(name): + items = getattr(advanced_queries, name)['columns'][1]['items'] + for key, entry in items.items(): + assert entry['type'] + assert entry['name'].endswith('.' + key) or entry['name'] == f"{entry['name'].split('.', 1)[0]}.{key}" + + +def test_temporal_percent_entries_carry_scale(): + items = advanced_queries.SystemEvents['columns'][1]['items'] + scaled = [(key, entry) for key, entry in items.items() if entry['type'] == 'temporal_percent'] + assert scaled, "system_events should ship at least one temporal_percent entry" + for _, entry in scaled: + assert entry['scale'] in {'second', 'millisecond', 'microsecond', 'nanosecond'} + + +def test_dotted_key_is_preserved_in_name(): + items = advanced_queries.SystemAsynchronousMetrics['columns'][1]['items'] + assert items['jemalloc.epoch']['name'] == 'asynchronous_metrics.jemalloc.epoch' + + +# --------------------------------------------------------------------------- +# SystemErrors (inline Python literal, not a match query) +# --------------------------------------------------------------------------- + + +def test_system_errors_is_inline_literal_with_expected_columns(): + spec = advanced_queries.SystemErrors + assert spec['name'] == 'system.errors' + assert spec['columns'][0] == {'name': 'errors.raised', 'type': 'monotonic_count'} + assert spec['columns'][-1] == {'name': 'remote', 'type': 'tag', 'boolean': True} + + +def test_system_errors_is_not_resolved_through_getattr(): + advanced_queries._match_query_cache.clear() + _ = advanced_queries.SystemErrors + assert 'SystemErrors' not in advanced_queries._match_query_cache + + +# --------------------------------------------------------------------------- +# Error wrapping for malformed JSON +# --------------------------------------------------------------------------- + + +def _write_spec(tmp_path, name, payload): + (tmp_path / f'{name}.json').write_text(json.dumps(payload), encoding='utf-8') + + +@pytest.mark.parametrize( + 'payload', + [ + pytest.param('not valid json', id='invalid-json'), + pytest.param('{"name": "x"}', id='missing-items-and-prefix'), + pytest.param('{"name": "x", "query": "y", "items": ["should-be-dict"]}', id='items-as-list'), + pytest.param('{"name": "x", "query": "y", "items": 5, "prefix": "p"}', id='items-as-scalar'), + ], +) +def test_load_match_query_wraps_malformed_payloads_in_runtime_error(isolated_data_dir, payload): + (isolated_data_dir / 'broken.json').write_text(payload, encoding='utf-8') + with pytest.raises(RuntimeError, match=r"failed to load advanced query 'broken'"): + advanced_queries.load_match_query('broken') + + +def test_load_match_query_wraps_missing_file_in_runtime_error(isolated_data_dir): + with pytest.raises(RuntimeError, match=r"failed to load advanced query 'missing'") as excinfo: + advanced_queries.load_match_query('missing') + assert isinstance(excinfo.value.__cause__, FileNotFoundError) + + +def test_load_match_query_preserves_cause_chain(isolated_data_dir): + _write_spec(isolated_data_dir, 'no_query', {'name': 'x'}) + with pytest.raises(RuntimeError) as excinfo: + advanced_queries.load_match_query('no_query') + assert isinstance(excinfo.value.__cause__, KeyError) + + +# --------------------------------------------------------------------------- +# warm_cache idempotency +# --------------------------------------------------------------------------- + + +def test_warm_cache_populates_every_match_query_name(): + assert advanced_queries._match_query_cache == {} + advanced_queries.warm_cache() + assert set(advanced_queries._match_query_cache) == set(MATCH_QUERY_NAMES) + + +def test_warm_cache_does_not_overwrite_existing_entries(): + sentinel = object() + advanced_queries._match_query_cache['SystemEvents'] = sentinel + advanced_queries.warm_cache() + assert advanced_queries._match_query_cache['SystemEvents'] is sentinel + assert set(advanced_queries._match_query_cache) == set(MATCH_QUERY_NAMES) diff --git a/cloudgen_firewall/assets/logs/cloudgen_firewall_tests.yaml b/cloudgen_firewall/assets/logs/cloudgen_firewall_tests.yaml index 879809ecc2a37..d18eea4e1bc88 100644 --- a/cloudgen_firewall/assets/logs/cloudgen_firewall_tests.yaml +++ b/cloudgen_firewall/assets/logs/cloudgen_firewall_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: cloudgen_firewall tests: - diff --git a/code-coverage.datadog.yml b/code-coverage.datadog.yml index cb9fe798dbc94..0de01c2ce72c4 100644 --- a/code-coverage.datadog.yml +++ b/code-coverage.datadog.yml @@ -1,2 +1,717 @@ schema-version: v1 carryforward: true +gates: +- type: total_coverage_percentage + config: + threshold: 75 + services: + - '*' + - '!couchbase' + - '!datadog_checks_base' + - '!datadog_checks_dev' + - '!fluentd' + - '!foundationdb' + - '!gearmand' + - '!glusterfs' + - '!hdfs_namenode' + - '!ibm_i' + - '!istio' + - '!kafka_actions' + - '!mapr' + - '!mapreduce' + - '!openstack' + - '!silverstripe_cms' + - '!snmp' + - '!varnish' +# New custom gates for services whose thresholds had to be lowered when migrating to Datadog Code Coverage. +- type: total_coverage_percentage + config: + threshold: 74 + services: + - couchbase +- type: total_coverage_percentage + config: + threshold: 73 + services: + - foundationdb + - varnish +- type: total_coverage_percentage + config: + threshold: 72 + services: + - datadog_checks_base +- type: total_coverage_percentage + config: + threshold: 71 + services: + - fluentd + - hdfs_namenode + - mapr +- type: total_coverage_percentage + config: + threshold: 70 + services: + - glusterfs + - istio +- type: total_coverage_percentage + config: + threshold: 69 + services: + - mapreduce +- type: total_coverage_percentage + config: + threshold: 67 + services: + - ibm_i + - silverstripe_cms +- type: total_coverage_percentage + config: + threshold: 61 + services: + - gearmand +- type: total_coverage_percentage + config: + threshold: 58 + services: + - kafka_actions +- type: total_coverage_percentage + config: + threshold: 52 + services: + - datadog_checks_dev +# Existing custom gates. +- type: total_coverage_percentage + config: + threshold: 45 + services: + - openstack +- type: total_coverage_percentage + config: + threshold: 30 + services: + - snmp +services: +- id: active_directory + paths: + - active_directory/datadog_checks/active_directory/ +- id: activemq_xml + paths: + - activemq_xml/datadog_checks/activemq_xml/ +- id: aerospike + paths: + - aerospike/datadog_checks/aerospike/ +- id: airflow + paths: + - airflow/datadog_checks/airflow/ +- id: amazon_msk + paths: + - amazon_msk/datadog_checks/amazon_msk/ +- id: ambari + paths: + - ambari/datadog_checks/ambari/ +- id: apache + paths: + - apache/datadog_checks/apache/ +- id: appgate_sdp + paths: + - appgate_sdp/datadog_checks/appgate_sdp/ +- id: arangodb + paths: + - arangodb/datadog_checks/arangodb/ +- id: argo_rollouts + paths: + - argo_rollouts/datadog_checks/argo_rollouts/ +- id: argo_workflows + paths: + - argo_workflows/datadog_checks/argo_workflows/ +- id: argocd + paths: + - argocd/datadog_checks/argocd/ +- id: aspdotnet + paths: + - aspdotnet/datadog_checks/aspdotnet/ +- id: avi_vantage + paths: + - avi_vantage/datadog_checks/avi_vantage/ +- id: aws_neuron + paths: + - aws_neuron/datadog_checks/aws_neuron/ +- id: azure_iot_edge + paths: + - azure_iot_edge/datadog_checks/azure_iot_edge/ +- id: bentoml + paths: + - bentoml/datadog_checks/bentoml/ +- id: boundary + paths: + - boundary/datadog_checks/boundary/ +- id: btrfs + paths: + - btrfs/datadog_checks/btrfs/ +- id: cacti + paths: + - cacti/datadog_checks/cacti/ +- id: calico + paths: + - calico/datadog_checks/calico/ +- id: cassandra_nodetool + paths: + - cassandra_nodetool/datadog_checks/cassandra_nodetool/ +- id: celery + paths: + - celery/datadog_checks/celery/ +- id: ceph + paths: + - ceph/datadog_checks/ceph/ +- id: cert_manager + paths: + - cert_manager/datadog_checks/cert_manager/ +- id: checkpoint_harmony_endpoint + paths: + - checkpoint_harmony_endpoint/datadog_checks/checkpoint_harmony_endpoint/ +- id: cilium + paths: + - cilium/datadog_checks/cilium/ +- id: cisco_aci + paths: + - cisco_aci/datadog_checks/cisco_aci/ +- id: citrix_hypervisor + paths: + - citrix_hypervisor/datadog_checks/citrix_hypervisor/ +- id: clickhouse + paths: + - clickhouse/datadog_checks/clickhouse/ +- id: cloud_foundry_api + paths: + - cloud_foundry_api/datadog_checks/cloud_foundry_api/ +- id: cloudera + paths: + - cloudera/datadog_checks/cloudera/ +- id: cockroachdb + paths: + - cockroachdb/datadog_checks/cockroachdb/ +- id: consul + paths: + - consul/datadog_checks/consul/ +- id: control_m + paths: + - control_m/datadog_checks/control_m/ +- id: coredns + paths: + - coredns/datadog_checks/coredns/ +- id: couch + paths: + - couch/datadog_checks/couch/ +- id: couchbase + paths: + - couchbase/datadog_checks/couchbase/ +- id: crio + paths: + - crio/datadog_checks/crio/ +- id: datadog_checks_base + paths: + - datadog_checks_base/datadog_checks/base/ +- id: datadog_checks_dev + paths: + - datadog_checks_dev/datadog_checks/dev/ +- id: datadog_checks_downloader + paths: + - datadog_checks_downloader/datadog_checks/downloader/ +- id: datadog_cluster_agent + paths: + - datadog_cluster_agent/datadog_checks/datadog_cluster_agent/ +- id: dcgm + paths: + - dcgm/datadog_checks/dcgm/ +- id: ddev + paths: + - ddev/src/ddev/ +- id: directory + paths: + - directory/datadog_checks/directory/ +- id: disk + paths: + - disk/datadog_checks/disk/ +- id: dns_check + paths: + - dns_check/datadog_checks/dns_check/ +- id: do_query_actions + paths: + - do_query_actions/datadog_checks/do_query_actions/ +- id: dotnetclr + paths: + - dotnetclr/datadog_checks/dotnetclr/ +- id: druid + paths: + - druid/datadog_checks/druid/ +- id: duckdb + paths: + - duckdb/datadog_checks/duckdb/ +- id: ecs_fargate + paths: + - ecs_fargate/datadog_checks/ecs_fargate/ +- id: eks_fargate + paths: + - eks_fargate/datadog_checks/eks_fargate/ +- id: elastic + paths: + - elastic/datadog_checks/elastic/ +- id: envoy + paths: + - envoy/datadog_checks/envoy/ +- id: esxi + paths: + - esxi/datadog_checks/esxi/ +- id: etcd + paths: + - etcd/datadog_checks/etcd/ +- id: exchange_server + paths: + - exchange_server/datadog_checks/exchange_server/ +- id: external_dns + paths: + - external_dns/datadog_checks/external_dns/ +- id: falco + paths: + - falco/datadog_checks/falco/ +- id: fluentd + paths: + - fluentd/datadog_checks/fluentd/ +- id: fluxcd + paths: + - fluxcd/datadog_checks/fluxcd/ +- id: fly_io + paths: + - fly_io/datadog_checks/fly_io/ +- id: foundationdb + paths: + - foundationdb/datadog_checks/foundationdb/ +- id: gearmand + paths: + - gearmand/datadog_checks/gearmand/ +- id: gitlab + paths: + - gitlab/datadog_checks/gitlab/ +- id: gitlab_runner + paths: + - gitlab_runner/datadog_checks/gitlab_runner/ +- id: glusterfs + paths: + - glusterfs/datadog_checks/glusterfs/ +- id: go_expvar + paths: + - go_expvar/datadog_checks/go_expvar/ +- id: guarddog + paths: + - guarddog/datadog_checks/guarddog/ +- id: gunicorn + paths: + - gunicorn/datadog_checks/gunicorn/ +- id: haproxy + paths: + - haproxy/datadog_checks/haproxy/ +- id: harbor + paths: + - harbor/datadog_checks/harbor/ +- id: hazelcast + paths: + - hazelcast/datadog_checks/hazelcast/ +- id: hdfs_datanode + paths: + - hdfs_datanode/datadog_checks/hdfs_datanode/ +- id: hdfs_namenode + paths: + - hdfs_namenode/datadog_checks/hdfs_namenode/ +- id: http_check + paths: + - http_check/datadog_checks/http_check/ +- id: hugging_face_tgi + paths: + - hugging_face_tgi/datadog_checks/hugging_face_tgi/ +- id: ibm_ace + paths: + - ibm_ace/datadog_checks/ibm_ace/ +- id: ibm_db2 + paths: + - ibm_db2/datadog_checks/ibm_db2/ +- id: ibm_i + paths: + - ibm_i/datadog_checks/ibm_i/ +- id: ibm_mq + paths: + - ibm_mq/datadog_checks/ibm_mq/ +- id: ibm_spectrum_lsf + paths: + - ibm_spectrum_lsf/datadog_checks/ibm_spectrum_lsf/ +- id: ibm_was + paths: + - ibm_was/datadog_checks/ibm_was/ +- id: iis + paths: + - iis/datadog_checks/iis/ +- id: impala + paths: + - impala/datadog_checks/impala/ +- id: infiniband + paths: + - infiniband/datadog_checks/infiniband/ +- id: istio + paths: + - istio/datadog_checks/istio/ +- id: kafka_actions + paths: + - kafka_actions/datadog_checks/kafka_actions/ +- id: kafka_consumer + paths: + - kafka_consumer/datadog_checks/kafka_consumer/ +- id: karpenter + paths: + - karpenter/datadog_checks/karpenter/ +- id: keda + paths: + - keda/datadog_checks/keda/ +- id: kong + paths: + - kong/datadog_checks/kong/ +- id: krakend + paths: + - krakend/datadog_checks/krakend/ +- id: kube_apiserver_metrics + paths: + - kube_apiserver_metrics/datadog_checks/kube_apiserver_metrics/ +- id: kube_controller_manager + paths: + - kube_controller_manager/datadog_checks/kube_controller_manager/ +- id: kube_dns + paths: + - kube_dns/datadog_checks/kube_dns/ +- id: kube_metrics_server + paths: + - kube_metrics_server/datadog_checks/kube_metrics_server/ +- id: kube_proxy + paths: + - kube_proxy/datadog_checks/kube_proxy/ +- id: kube_scheduler + paths: + - kube_scheduler/datadog_checks/kube_scheduler/ +- id: kubeflow + paths: + - kubeflow/datadog_checks/kubeflow/ +- id: kubelet + paths: + - kubelet/datadog_checks/kubelet/ +- id: kubernetes_cluster_autoscaler + paths: + - kubernetes_cluster_autoscaler/datadog_checks/kubernetes_cluster_autoscaler/ +- id: kubernetes_state + paths: + - kubernetes_state/datadog_checks/kubernetes_state/ +- id: kubevirt_api + paths: + - kubevirt_api/datadog_checks/kubevirt_api/ +- id: kubevirt_controller + paths: + - kubevirt_controller/datadog_checks/kubevirt_controller/ +- id: kubevirt_handler + paths: + - kubevirt_handler/datadog_checks/kubevirt_handler/ +- id: kuma + paths: + - kuma/datadog_checks/kuma/ +- id: kyototycoon + paths: + - kyototycoon/datadog_checks/kyototycoon/ +- id: kyverno + paths: + - kyverno/datadog_checks/kyverno/ +- id: lighttpd + paths: + - lighttpd/datadog_checks/lighttpd/ +- id: linkerd + paths: + - linkerd/datadog_checks/linkerd/ +- id: linux_proc_extras + paths: + - linux_proc_extras/datadog_checks/linux_proc_extras/ +- id: litellm + paths: + - litellm/datadog_checks/litellm/ +- id: lparstats + paths: + - lparstats/datadog_checks/lparstats/ +- id: lustre + paths: + - lustre/datadog_checks/lustre/ +- id: mac_audit_logs + paths: + - mac_audit_logs/datadog_checks/mac_audit_logs/ +- id: mapr + paths: + - mapr/datadog_checks/mapr/ +- id: mapreduce + paths: + - mapreduce/datadog_checks/mapreduce/ +- id: marathon + paths: + - marathon/datadog_checks/marathon/ +- id: marklogic + paths: + - marklogic/datadog_checks/marklogic/ +- id: mcache + paths: + - mcache/datadog_checks/mcache/ +- id: mesos_master + paths: + - mesos_master/datadog_checks/mesos_master/ +- id: mesos_slave + paths: + - mesos_slave/datadog_checks/mesos_slave/ +- id: milvus + paths: + - milvus/datadog_checks/milvus/ +- id: mongo + paths: + - mongo/datadog_checks/mongo/ +- id: mysql + paths: + - mysql/datadog_checks/mysql/ +- id: n8n + paths: + - n8n/datadog_checks/n8n/ +- id: nagios + paths: + - nagios/datadog_checks/nagios/ +- id: network + paths: + - network/datadog_checks/network/ +- id: nfsstat + paths: + - nfsstat/datadog_checks/nfsstat/ +- id: nginx + paths: + - nginx/datadog_checks/nginx/ +- id: nginx_ingress_controller + paths: + - nginx_ingress_controller/datadog_checks/nginx_ingress_controller/ +- id: nifi + paths: + - nifi/datadog_checks/nifi/ +- id: nutanix + paths: + - nutanix/datadog_checks/nutanix/ +- id: nvidia_nim + paths: + - nvidia_nim/datadog_checks/nvidia_nim/ +- id: nvidia_triton + paths: + - nvidia_triton/datadog_checks/nvidia_triton/ +- id: octopus_deploy + paths: + - octopus_deploy/datadog_checks/octopus_deploy/ +- id: openldap + paths: + - openldap/datadog_checks/openldap/ +- id: openmetrics + paths: + - openmetrics/datadog_checks/openmetrics/ +- id: openstack + paths: + - openstack/datadog_checks/openstack/ +- id: openstack_controller + paths: + - openstack_controller/datadog_checks/openstack_controller/ +- id: pdh_check + paths: + - pdh_check/datadog_checks/pdh_check/ +- id: pgbouncer + paths: + - pgbouncer/datadog_checks/pgbouncer/ +- id: php_fpm + paths: + - php_fpm/datadog_checks/php_fpm/ +- id: postfix + paths: + - postfix/datadog_checks/postfix/ +- id: postgres + paths: + - postgres/datadog_checks/postgres/ +- id: powerdns_recursor + paths: + - powerdns_recursor/datadog_checks/powerdns_recursor/ +- id: prefect + paths: + - prefect/datadog_checks/prefect/ +- id: process + paths: + - process/datadog_checks/process/ +- id: prometheus + paths: + - prometheus/datadog_checks/prometheus/ +- id: proxmox + paths: + - proxmox/datadog_checks/proxmox/ +- id: proxysql + paths: + - proxysql/datadog_checks/proxysql/ +- id: pulsar + paths: + - pulsar/datadog_checks/pulsar/ +- id: quarkus + paths: + - quarkus/datadog_checks/quarkus/ +- id: rabbitmq + paths: + - rabbitmq/datadog_checks/rabbitmq/ +- id: ray + paths: + - ray/datadog_checks/ray/ +- id: redisdb + paths: + - redisdb/datadog_checks/redisdb/ +- id: rethinkdb + paths: + - rethinkdb/datadog_checks/rethinkdb/ +- id: riak + paths: + - riak/datadog_checks/riak/ +- id: riakcs + paths: + - riakcs/datadog_checks/riakcs/ +- id: sap_hana + paths: + - sap_hana/datadog_checks/sap_hana/ +- id: scylla + paths: + - scylla/datadog_checks/scylla/ +- id: silk + paths: + - silk/datadog_checks/silk/ +- id: silverstripe_cms + paths: + - silverstripe_cms/datadog_checks/silverstripe_cms/ +- id: singlestore + paths: + - singlestore/datadog_checks/singlestore/ +- id: slurm + paths: + - slurm/datadog_checks/slurm/ +- id: snmp + paths: + - snmp/datadog_checks/snmp/ +- id: sonarqube + paths: + - sonarqube/datadog_checks/sonarqube/ +- id: sonatype_nexus + paths: + - sonatype_nexus/datadog_checks/sonatype_nexus/ +- id: spark + paths: + - spark/datadog_checks/spark/ +- id: sqlserver + paths: + - sqlserver/datadog_checks/sqlserver/ +- id: squid + paths: + - squid/datadog_checks/squid/ +- id: ssh_check + paths: + - ssh_check/datadog_checks/ssh_check/ +- id: statsd + paths: + - statsd/datadog_checks/statsd/ +- id: strimzi + paths: + - strimzi/datadog_checks/strimzi/ +- id: supabase + paths: + - supabase/datadog_checks/supabase/ +- id: supervisord + paths: + - supervisord/datadog_checks/supervisord/ +- id: system_core + paths: + - system_core/datadog_checks/system_core/ +- id: system_swap + paths: + - system_swap/datadog_checks/system_swap/ +- id: tcp_check + paths: + - tcp_check/datadog_checks/tcp_check/ +- id: teamcity + paths: + - teamcity/datadog_checks/teamcity/ +- id: tekton + paths: + - tekton/datadog_checks/tekton/ +- id: teleport + paths: + - teleport/datadog_checks/teleport/ +- id: temporal + paths: + - temporal/datadog_checks/temporal/ +- id: teradata + paths: + - teradata/datadog_checks/teradata/ +- id: tibco_ems + paths: + - tibco_ems/datadog_checks/tibco_ems/ +- id: tls + paths: + - tls/datadog_checks/tls/ +- id: torchserve + paths: + - torchserve/datadog_checks/torchserve/ +- id: traefik_mesh + paths: + - traefik_mesh/datadog_checks/traefik_mesh/ +- id: traffic_server + paths: + - traffic_server/datadog_checks/traffic_server/ +- id: twemproxy + paths: + - twemproxy/datadog_checks/twemproxy/ +- id: twistlock + paths: + - twistlock/datadog_checks/twistlock/ +- id: varnish + paths: + - varnish/datadog_checks/varnish/ +- id: vault + paths: + - vault/datadog_checks/vault/ +- id: velero + paths: + - velero/datadog_checks/velero/ +- id: vertica + paths: + - vertica/datadog_checks/vertica/ +- id: vllm + paths: + - vllm/datadog_checks/vllm/ +- id: voltdb + paths: + - voltdb/datadog_checks/voltdb/ +- id: vsphere + paths: + - vsphere/datadog_checks/vsphere/ +- id: weaviate + paths: + - weaviate/datadog_checks/weaviate/ +- id: win32_event_log + paths: + - win32_event_log/datadog_checks/win32_event_log/ +- id: windows_performance_counters + paths: + - windows_performance_counters/datadog_checks/windows_performance_counters/ +- id: windows_service + paths: + - windows_service/datadog_checks/windows_service/ +- id: wmi_check + paths: + - wmi_check/datadog_checks/wmi_check/ +- id: yarn + paths: + - yarn/datadog_checks/yarn/ +- id: zk + paths: + - zk/datadog_checks/zk/ diff --git a/consul/assets/logs/consul_tests.yaml b/consul/assets/logs/consul_tests.yaml index 2ca9f74998753..1299069dd1ac7 100644 --- a/consul/assets/logs/consul_tests.yaml +++ b/consul/assets/logs/consul_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "consul" tests: diff --git a/coredns/assets/logs/coredns.yaml b/coredns/assets/logs/coredns.yaml index 57011c74cbd64..4bb9c1ed7d0bc 100644 --- a/coredns/assets/logs/coredns.yaml +++ b/coredns/assets/logs/coredns.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: coredns metric_id: coredns backend_only: false diff --git a/coredns/assets/logs/coredns_tests.yaml b/coredns/assets/logs/coredns_tests.yaml index 9eb3610f85326..7ed86e3c86139 100644 --- a/coredns/assets/logs/coredns_tests.yaml +++ b/coredns/assets/logs/coredns_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "coredns" tests: - diff --git a/couchbase/tests/conftest.py b/couchbase/tests/conftest.py index 5d5136982e14c..002ee072a145d 100644 --- a/couchbase/tests/conftest.py +++ b/couchbase/tests/conftest.py @@ -75,6 +75,7 @@ def dd_environment(): WaitFor(bucket_stats), WaitFor(load_sample_bucket), WaitFor(create_syncgw_database), + WaitFor(gamesim_primary_index_ready), ] with docker_run( compose_file=os.path.join(HERE, 'compose', 'docker-compose.yaml'), @@ -215,10 +216,15 @@ def load_sample_bucket(): if task["sample"] == "gamesim-sample": task_id = task["taskId"] + # No matching task in the install response β€” the bucket is likely loading + # under a task we can't observe. WaitFor will retry; on the retry the install + # POST takes the already-loaded shortcut, and gamesim_primary_index_ready is + # the authoritative gate that blocks until the bundled GSI is online. + if task_id is None: + return False + while True: # Loop until the task ID is gone, meaning the task is done. - task_is_done = False - r = requests.get( '{}/pools/default/tasks'.format(URL), auth=(USER, PASSWORD), @@ -226,11 +232,8 @@ def load_sample_bucket(): r.raise_for_status() result = r.json() - for task in result: - if task.get("task_id", "") == task_id: - task_is_done = True - - if task_is_done: + task_still_running = any(task.get("task_id") == task_id for task in result) + if not task_still_running: break time.sleep(1) @@ -238,6 +241,28 @@ def load_sample_bucket(): return True +def gamesim_primary_index_ready(): + """Wait until every gamesim_primary keyspace reports initial_build_progress == 100.""" + r = requests.get( + '{}/api/v1/stats'.format(INDEX_STATS_URL), + auth=(USER, PASSWORD), + ) + r.raise_for_status() + + data = r.json() + matches = [ + stats + for keyspace, stats in data.items() + if keyspace != "indexer" + and keyspace.split(":")[0] == "gamesim-sample" + and keyspace.split(":")[-1] == "gamesim_primary" + ] + if not matches: + print("gamesim_primary not yet visible; keyspaces seen: {}".format(list(data.keys()))) + return False + return all(s.get("initial_build_progress") == 100 for s in matches) + + def create_syncgw_database(): """ Create sample database diff --git a/datadog_checks_base/changelog.d/22750.added b/datadog_checks_base/changelog.d/22750.added new file mode 100644 index 0000000000000..e8a49db170139 --- /dev/null +++ b/datadog_checks_base/changelog.d/22750.added @@ -0,0 +1 @@ +Add file-based YAML metrics loading for OpenMetrics V2 checks with composable predicates \ No newline at end of file diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/__init__.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/__init__.py index 46dd167dcde48..cc1fa89df93d3 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/__init__.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/__init__.py @@ -1,3 +1,6 @@ # (C) Datadog, Inc. 2020-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import lazy_loader + +__getattr__, __dir__, __all__ = lazy_loader.attach_stub(__name__, __file__) diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/__init__.pyi b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/__init__.pyi new file mode 100644 index 0000000000000..a2c49c408d85c --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/__init__.pyi @@ -0,0 +1,22 @@ +# (C) Datadog, Inc. 2025-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from .base import OpenMetricsBaseCheckV2 +from .metrics_mapping import ( + AllOf, + AnyOf, + ConfigOptionEquals, + ConfigOptionTruthy, + MetricsMapping, + MetricsPredicate, +) + +__all__ = [ + 'AllOf', + 'AnyOf', + 'ConfigOptionEquals', + 'ConfigOptionTruthy', + 'MetricsMapping', + 'MetricsPredicate', + 'OpenMetricsBaseCheckV2', +] diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py index 1a476138157de..2fb5062e146bf 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py @@ -1,9 +1,14 @@ # (C) Datadog, Inc. 2020-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + from collections import ChainMap from contextlib import contextmanager +from pathlib import Path +from typing import TYPE_CHECKING +import yaml from requests.exceptions import RequestException from datadog_checks.base.checks import AgentCheck @@ -12,6 +17,11 @@ from .scraper import OpenMetricsScraper +if TYPE_CHECKING: + from collections.abc import Mapping + + from .metrics_mapping import MetricsMapping, _RawMetricsConfig + class OpenMetricsBaseCheckV2(AgentCheck): """ @@ -32,6 +42,14 @@ class OpenMetricsBaseCheckV2(AgentCheck): DEFAULT_METRIC_LIMIT = 2000 + METRICS_MAP: tuple[MetricsMapping, ...] = () + """YAML files with metric name mappings to load automatically. + + When empty (default), looks for ``metrics.yaml`` next to the check module, + falling back to ``metrics.yml`` if the former is absent. When set, only the + declared files are loaded (with predicates controlling conditional loading). + """ + # Allow tracing for openmetrics integrations def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) @@ -52,6 +70,9 @@ def __init__(self, name, init_config, instances): # All configured scrapers keyed by the endpoint self.scrapers = {} + # Cache for file-based metrics loaded from METRICS_MAP; None means not yet loaded + self._file_metrics: list[_RawMetricsConfig] | None = None + self.check_initializations.append(self.configure_scrapers) def check(self, _): @@ -105,14 +126,80 @@ def set_dynamic_tags(self, *tags): scraper.set_dynamic_tags(*tags) def get_config_with_defaults(self, config): - return ChainMap(config, self.get_default_config()) + """Combine instance config with class defaults and file-based metric mappings. + + Subclasses that override this method must call ``super().get_config_with_defaults(config)``; + otherwise the YAML mappings declared via ``METRICS_MAP`` (or discovered by convention) are + silently skipped. + """ + defaults = dict(self.get_default_config()) + if file_metrics := self._load_file_based_metrics(config): + defaults['metrics'] = list(defaults.get('metrics', [])) + file_metrics + return ChainMap(config, defaults) - def get_default_config(self): + def get_default_config(self) -> dict: + """Return instance-level default scraper configuration values. + + The returned dict can be mutated by the framework before being wrapped + in a ``ChainMap``. Avoid returning a shared or instance-level object to avoid + state leakage between check executions. + """ return {} def refresh_scrapers(self): pass + def _load_file_based_metrics(self, config: Mapping) -> list[_RawMetricsConfig]: + """Load metric mappings from YAML files declared in ``METRICS_MAP``. + + Results are cached for the lifetime of the check instance. Predicates + are evaluated once against the first ``config`` supplied; ``METRICS_MAP`` + is a class-level declaration and the instance config does not change + between runs, so subsequent calls always receive the same effective + configuration. + + Falls back to convention-based discovery of ``metrics.yaml`` or + ``metrics.yml`` (in that order) when ``METRICS_MAP`` is empty. + + Permanent load failures (malformed YAML, unreadable files) are raised + once on the first call; the cache is sealed beforehand so subsequent + scrapes do not retry and re-raise the same error. A failure on any + single file in a multi-file ``METRICS_MAP`` discards results from + files loaded earlier in the same call: the cache lands as ``[]``, not + as a partial mapping. + """ + if self._file_metrics is not None: + return self._file_metrics + + self._file_metrics = [] + package_dir = self._get_package_dir() + if not self.METRICS_MAP: + for candidate in (Path("metrics.yaml"), Path("metrics.yml")): + resolved = package_dir / candidate + if resolved.is_file(): + self._file_metrics = [self._load_metrics_file(resolved)] + break + else: + self._file_metrics = [ + self._load_metrics_file(package_dir / source.path) + for source in self.METRICS_MAP + if source.should_load(config) + ] + + return self._file_metrics + + def _load_metrics_file(self, file_path: Path) -> _RawMetricsConfig: + try: + with open(file_path) as f: + data = yaml.safe_load(f) + except yaml.YAMLError as e: + raise ConfigurationError(f"Failed to parse metrics file {file_path}: {e}") from e + except OSError as e: + raise ConfigurationError(f"Failed to read metrics file {file_path}: {e}") from e + if not isinstance(data, dict): + raise ConfigurationError(f"Metrics file {file_path} must contain a YAML mapping, got {type(data).__name__}") + return data + @contextmanager def adopt_namespace(self, namespace): old_namespace = self.__NAMESPACE__ diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/metrics_mapping.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/metrics_mapping.py new file mode 100644 index 0000000000000..3a4582b849dab --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/metrics_mapping.py @@ -0,0 +1,118 @@ +# (C) Datadog, Inc. 2025-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, Protocol + +from datadog_checks.base.config import is_affirmative + +if TYPE_CHECKING: + from datadog_checks.base.types import InstanceType + +_RawMetricsConfig = dict[str, str | dict[str, Any]] +"""Metric name mapping loaded from a YAML file. + +Keys are raw Prometheus metric names; values are either Datadog metric names +(simple renaming) or dicts describing transformer configuration. The inner +dict carries the full ``OpenMetricsScraper`` transformer shape (type, label +maps, nested options), so it is intentionally widened to ``dict[str, Any]``. + +Internal: the type the loader returns; integration authors should not need to +reference this directly. +""" + + +class MetricsPredicate(Protocol): + """ + Protocol for predicates that control whether a metrics mapping should be loaded. + + Implement ``should_load`` to create custom loading conditions. + """ + + def should_load(self, config: InstanceType) -> bool: ... + + +class ConfigOptionTruthy: + """ + Load metrics only if a configuration option is truthy. + + Uses ``is_affirmative`` to evaluate the value. Defaults to ``True`` + (include metrics unless explicitly disabled). + """ + + def __init__(self, option: str, default: bool = True) -> None: + self.option = option + self.default = default + + def should_load(self, config: InstanceType) -> bool: + return is_affirmative(config.get(self.option, self.default)) + + +class ConfigOptionEquals: + """ + Load metrics only if a configuration option equals a specific value. + + A missing key compares equal to ``None``: ``ConfigOptionEquals("flag", None)`` + matches both ``{"flag": None}`` and instances that omit the key entirely. + """ + + def __init__(self, option: str, value: Any) -> None: + self.option = option + self.value = value + + def should_load(self, config: InstanceType) -> bool: + return config.get(self.option) == self.value + + +class AllOf: + """ + Compose predicates: all must pass for the metrics to be loaded. + + Follows Python's ``all()`` semantics: returns ``True`` when empty. + """ + + def __init__(self, *predicates: MetricsPredicate) -> None: + self.predicates = predicates + + def should_load(self, config: InstanceType) -> bool: + return all(p.should_load(config) for p in self.predicates) + + +class AnyOf: + """ + Compose predicates: any passing is sufficient to load the metrics. + + Follows Python's ``any()`` semantics: returns ``False`` when empty. + """ + + def __init__(self, *predicates: MetricsPredicate) -> None: + self.predicates = predicates + + def should_load(self, config: InstanceType) -> bool: + return any(p.should_load(config) for p in self.predicates) + + +@dataclass(frozen=True) +class MetricsMapping: + """ + Declares a YAML file with metric name mappings to load automatically. + + Use in the ``METRICS_MAP`` class variable of ``OpenMetricsBaseCheckV2`` + subclasses. The YAML file should contain a flat mapping of Prometheus + metric names to Datadog metric names:: + + METRICS_MAP = ( + MetricsMapping(Path("metrics/default.yaml")), + MetricsMapping(Path("metrics/go.yaml"), predicate=ConfigOptionTruthy("go_metrics")), + ) + """ + + path: Path + predicate: MetricsPredicate | None = None + + def should_load(self, config: InstanceType) -> bool: + """Return whether this mapping should be loaded for the given config.""" + return self.predicate is None or self.predicate.should_load(config) diff --git a/datadog_checks_base/tests/base/checks/openmetrics/test_v2/test_metrics_mapping.py b/datadog_checks_base/tests/base/checks/openmetrics/test_v2/test_metrics_mapping.py new file mode 100644 index 0000000000000..12bbe22440b91 --- /dev/null +++ b/datadog_checks_base/tests/base/checks/openmetrics/test_v2/test_metrics_mapping.py @@ -0,0 +1,422 @@ +# (C) Datadog, Inc. 2025-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from pathlib import Path +from typing import Protocol +from unittest.mock import patch + +import pytest +import yaml + +from datadog_checks.base import OpenMetricsBaseCheckV2 +from datadog_checks.base.checks.openmetrics.v2.metrics_mapping import ( + AllOf, + AnyOf, + ConfigOptionEquals, + ConfigOptionTruthy, + MetricsMapping, +) +from datadog_checks.base.errors import ConfigurationError + +_DEFAULT_INSTANCE: dict[str, object] = {'openmetrics_endpoint': 'http://test:9090/metrics'} + + +class CheckFactory(Protocol): + def __call__( + self, + cls: type[OpenMetricsBaseCheckV2] | None = None, + instance: dict[str, object] | None = None, + ) -> OpenMetricsBaseCheckV2: ... + + +@pytest.fixture +def make_check() -> CheckFactory: + def factory(cls=None, instance=None): + cls = cls or OpenMetricsBaseCheckV2 + inst = _DEFAULT_INSTANCE | (instance or {}) + c = cls('test', {}, [inst]) + c.__NAMESPACE__ = 'test' + return c + + return factory + + +def write_yaml(tmp_path, filename, data): + path = tmp_path / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.dump(data)) + return path + + +# --------------------------------------------------------------------------- +# ConfigOptionTruthy +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "config, default, expected", + [ + ({"opt": True}, True, True), + ({"opt": False}, True, False), + ({}, True, True), + ({}, False, False), + ({"opt": "yes"}, True, True), + ({"opt": "no"}, True, False), + ], + ids=["true", "false", "missing_default_true", "missing_default_false", "string_yes", "string_no"], +) +def test_config_option_truthy(config: dict[str, object], default: bool, expected: bool): + pred = ConfigOptionTruthy("opt", default=default) + assert pred.should_load(config) is expected + + +# --------------------------------------------------------------------------- +# ConfigOptionEquals +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "config, value, expected", + [ + ({"mode": "advanced"}, "advanced", True), + ({"mode": "basic"}, "advanced", False), + ({}, "advanced", False), + ({}, None, True), + ], + ids=["equal", "not_equal", "missing", "none_matches_missing"], +) +def test_config_option_equals(config: dict[str, str], value: str | None, expected: bool): + pred = ConfigOptionEquals("mode", value) + assert pred.should_load(config) is expected + + +# --------------------------------------------------------------------------- +# AllOf / AnyOf +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "cls, config, expected", + [ + (AllOf, {"a": True, "b": True}, True), + (AllOf, {"a": True, "b": False}, False), + (AllOf, {"a": False, "b": False}, False), + (AnyOf, {"a": True, "b": True}, True), + (AnyOf, {"a": True, "b": False}, True), + (AnyOf, {"a": False, "b": False}, False), + ], + ids=["all_both", "all_one", "all_none", "any_both", "any_one", "any_none"], +) +def test_composite_predicates(cls: type[AllOf] | type[AnyOf], config: dict[str, bool], expected: bool): + pred = cls(ConfigOptionTruthy("a"), ConfigOptionTruthy("b")) + assert pred.should_load(config) is expected + + +@pytest.mark.parametrize( + "cls, expected", + [ + (AllOf, True), + (AnyOf, False), + ], + ids=["all_of_vacuous_truth", "any_of_vacuous_falsity"], +) +def test_composite_predicates_empty(cls: type[AllOf] | type[AnyOf], expected: bool): + assert cls().should_load({}) is expected + + +# --------------------------------------------------------------------------- +# MetricsMapping +# --------------------------------------------------------------------------- + + +def test_metrics_mapping_is_frozen(): + mm = MetricsMapping(Path("m.yaml")) + with pytest.raises(AttributeError): + mm.path = Path("other.yaml") + + +def test_metrics_mapping_should_load_no_predicate(): + mm = MetricsMapping(Path("m.yaml")) + assert mm.should_load({}) is True + + +def test_metrics_mapping_should_load_with_predicate(): + mm = MetricsMapping(Path("m.yaml"), predicate=ConfigOptionTruthy("opt", default=False)) + assert mm.should_load({}) is False + assert mm.should_load({"opt": True}) is True + + +# --------------------------------------------------------------------------- +# _load_metrics_file +# --------------------------------------------------------------------------- + + +def test_load_metrics_file(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "go.yaml", {"go_goroutines": "go.goroutines"}) + check = make_check() + assert check._load_metrics_file(tmp_path / "go.yaml") == {"go_goroutines": "go.goroutines"} + + +@pytest.mark.parametrize( + "filename, content, match", + [ + ("broken.yaml", "foo: [bar", "Failed to parse"), + ("empty.yaml", "", "must contain a YAML mapping, got NoneType"), + ("list.yaml", "[1, 2, 3]", "must contain a YAML mapping, got list"), + ], + ids=["malformed", "empty", "non_dict"], +) +def test_load_metrics_file_errors(make_check: CheckFactory, tmp_path: Path, filename: str, content: str, match: str): + (tmp_path / filename).write_text(content) + check = make_check() + with pytest.raises(ConfigurationError, match=match): + check._load_metrics_file(tmp_path / filename) + + +def test_load_metrics_file_missing(make_check: CheckFactory, tmp_path: Path): + check = make_check() + with pytest.raises(ConfigurationError, match="Failed to read metrics file"): + check._load_metrics_file(tmp_path / "nonexistent.yaml") + + +# --------------------------------------------------------------------------- +# _load_file_based_metrics +# --------------------------------------------------------------------------- + + +def test_load_file_based_metrics_no_files(make_check: CheckFactory, tmp_path: Path): + check = make_check() + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + assert check._load_file_based_metrics({}) == [] + + +@pytest.mark.parametrize("filename", ["metrics.yaml", "metrics.yml"]) +def test_load_file_based_metrics_convention_discovery(make_check: CheckFactory, tmp_path: Path, filename: str): + write_yaml(tmp_path, filename, {"raw": "dd.raw"}) + check = make_check() + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + result = check._load_file_based_metrics({}) + assert result == [{"raw": "dd.raw"}] + + +def test_load_file_based_metrics_convention_yaml_takes_precedence(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "metrics.yaml", {"from_yaml": "dd.yaml"}) + write_yaml(tmp_path, "metrics.yml", {"from_yml": "dd.yml"}) + check = make_check() + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + result = check._load_file_based_metrics({}) + assert result == [{"from_yaml": "dd.yaml"}] + + +def test_load_file_based_metrics_explicit(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "metrics/a.yaml", {"m1": "d1"}) + write_yaml(tmp_path, "metrics/b.yaml", {"m2": "d2"}) + + class Check(OpenMetricsBaseCheckV2): + METRICS_MAP = (MetricsMapping(Path("metrics/a.yaml")), MetricsMapping(Path("metrics/b.yaml"))) + + check = make_check(cls=Check) + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + result = check._load_file_based_metrics({}) + assert result == [{"m1": "d1"}, {"m2": "d2"}] + + +def test_load_file_based_metrics_predicate_filters(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "metrics/always.yaml", {"m1": "d1"}) + write_yaml(tmp_path, "metrics/conditional.yaml", {"m2": "d2"}) + + class Check(OpenMetricsBaseCheckV2): + METRICS_MAP = ( + MetricsMapping(Path("metrics/always.yaml")), + MetricsMapping(Path("metrics/conditional.yaml"), predicate=ConfigOptionTruthy("extra", default=False)), + ) + + check_base = make_check(cls=Check) + check_extra = make_check(cls=Check, instance={'extra': True}) + with patch.object(Check, '_get_package_dir', return_value=tmp_path): + assert len(check_base._load_file_based_metrics(check_base.instance)) == 1 + assert len(check_extra._load_file_based_metrics(check_extra.instance)) == 2 + + +def test_load_file_based_metrics_explicit_skips_convention(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "metrics.yml", {"convention": "metric"}) + write_yaml(tmp_path, "explicit.yaml", {"explicit": "metric"}) + + class Check(OpenMetricsBaseCheckV2): + METRICS_MAP = (MetricsMapping(Path("explicit.yaml")),) + + check = make_check(cls=Check) + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + result = check._load_file_based_metrics({}) + assert result == [{"explicit": "metric"}] + + +# --------------------------------------------------------------------------- +# _load_file_based_metrics caching +# --------------------------------------------------------------------------- + + +def test_load_file_based_metrics_cached_across_calls(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "metrics.yml", {"raw": "dd.raw"}) + check = make_check() + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + first = check._load_file_based_metrics({}) + second = check._load_file_based_metrics({}) + assert first is second + + +def test_load_file_based_metrics_cache_ignores_config_changes(make_check: CheckFactory, tmp_path: Path): + """Predicate re-evaluation is suppressed once the cache is populated: a second call with a + different config returns the first-call result without consulting the predicates again.""" + write_yaml(tmp_path, "metrics/always.yaml", {"m1": "d1"}) + write_yaml(tmp_path, "metrics/extra.yaml", {"m2": "d2"}) + + class Check(OpenMetricsBaseCheckV2): + METRICS_MAP = ( + MetricsMapping(Path("metrics/always.yaml")), + MetricsMapping(Path("metrics/extra.yaml"), predicate=ConfigOptionTruthy("extra", default=False)), + ) + + check = make_check(cls=Check) + with patch.object(Check, '_get_package_dir', return_value=tmp_path): + first = check._load_file_based_metrics({'extra': False}) + second = check._load_file_based_metrics({'extra': True}) + assert first is second + assert len(first) == 1 + + +def test_load_file_based_metrics_permanent_failure_fails_once(make_check: CheckFactory, tmp_path: Path): + """A malformed YAML file raises on the first call; subsequent calls return the empty cache.""" + (tmp_path / "metrics.yml").write_text("foo: [bar") + check = make_check() + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + with pytest.raises(ConfigurationError, match="Failed to parse"): + check._load_file_based_metrics({}) + assert check._load_file_based_metrics({}) == [] + + +def test_load_file_based_metrics_multi_file_failure_seals_empty(make_check: CheckFactory, tmp_path: Path): + """A mid-comprehension load failure discards earlier successes; the cache lands as [].""" + write_yaml(tmp_path, "metrics/a.yaml", {"a": "dd.a"}) + (tmp_path / "metrics" / "b.yaml").write_text("foo: [bar") + write_yaml(tmp_path, "metrics/c.yaml", {"c": "dd.c"}) + + class Check(OpenMetricsBaseCheckV2): + METRICS_MAP = ( + MetricsMapping(Path("metrics/a.yaml")), + MetricsMapping(Path("metrics/b.yaml")), + MetricsMapping(Path("metrics/c.yaml")), + ) + + check = make_check(cls=Check) + with patch.object(Check, '_get_package_dir', return_value=tmp_path): + with pytest.raises(ConfigurationError, match="Failed to parse"): + check._load_file_based_metrics({}) + assert check._load_file_based_metrics({}) == [] + + +def test_load_file_based_metrics_does_not_accumulate_on_repeated_scraper_creation( + make_check: CheckFactory, tmp_path: Path +): + """Repeated create_scraper calls (e.g. from refresh_scrapers) must not grow the metrics list.""" + write_yaml(tmp_path, "metrics.yml", {"raw": "dd.raw"}) + check = make_check() + instance = {'openmetrics_endpoint': 'http://test:9090/metrics'} + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + config_first = check.get_config_with_defaults(instance) + config_second = check.get_config_with_defaults(instance) + assert config_first['metrics'] == config_second['metrics'] + + +def test_load_file_based_metrics_does_not_mutate_get_default_config(make_check: CheckFactory, tmp_path: Path): + """File metrics must not mutate the list returned by get_default_config.""" + write_yaml(tmp_path, "metrics.yml", {"raw": "dd.raw"}) + SHARED_METRICS = [{"existing": "metric"}] + + class Check(OpenMetricsBaseCheckV2): + def get_default_config(self): + return {"metrics": SHARED_METRICS} + + check = make_check(cls=Check) + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + check.get_config_with_defaults({'openmetrics_endpoint': 'http://test:9090/metrics'}) + assert SHARED_METRICS == [{"existing": "metric"}] + + +def test_load_file_based_metrics_does_not_mutate_cached_default_dict(make_check: CheckFactory, tmp_path: Path): + """A subclass that caches its defaults dict at module level must not see file metrics accumulate.""" + write_yaml(tmp_path, "metrics.yml", {"raw": "dd.raw"}) + CACHED_DEFAULTS = {"metrics": [{"existing": "metric"}]} + + class Check(OpenMetricsBaseCheckV2): + def get_default_config(self): + return CACHED_DEFAULTS + + instance = {'openmetrics_endpoint': 'http://test:9090/metrics'} + first = make_check(cls=Check) + second = make_check(cls=Check) + with patch.object(Check, '_get_package_dir', return_value=tmp_path): + config_first = first.get_config_with_defaults(instance) + config_second = second.get_config_with_defaults(instance) + assert CACHED_DEFAULTS == {"metrics": [{"existing": "metric"}]} + assert config_first['metrics'] == config_second['metrics'] + assert len(config_first['metrics']) == 2 + + +# --------------------------------------------------------------------------- +# get_config_with_defaults +# --------------------------------------------------------------------------- + + +def test_get_config_with_defaults_merges_file_metrics(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "metrics.yml", {"raw": "dd_name"}) + check = make_check() + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + config = check.get_config_with_defaults({"openmetrics_endpoint": "http://test:9090/metrics"}) + assert {"raw": "dd_name"} in config["metrics"] + + +def test_get_config_with_defaults_combines_with_existing(make_check: CheckFactory, tmp_path: Path): + write_yaml(tmp_path, "metrics.yml", {"file_metric": "dd.file_metric"}) + + class Check(OpenMetricsBaseCheckV2): + def get_default_config(self): + return {"metrics": [{"existing": "metric"}], "extra_option": True} + + check = make_check(cls=Check) + with patch.object(type(check), '_get_package_dir', return_value=tmp_path): + config = check.get_config_with_defaults({"openmetrics_endpoint": "http://test:9090/metrics"}) + assert {"existing": "metric"} in config["metrics"] + assert {"file_metric": "dd.file_metric"} in config["metrics"] + assert config["extra_option"] is True + + +# --------------------------------------------------------------------------- +# Public re-exports (lazy_loader stub) +# --------------------------------------------------------------------------- + + +def test_public_reexports_resolve(): + """The lazy_loader stub at v2/__init__.pyi must expose the documented public surface.""" + from datadog_checks.base.checks.openmetrics.v2 import ( + AllOf, + AnyOf, + ConfigOptionEquals, + ConfigOptionTruthy, + MetricsMapping, + MetricsPredicate, + OpenMetricsBaseCheckV2, + ) + + assert all( + symbol is not None + for symbol in ( + AllOf, + AnyOf, + ConfigOptionEquals, + ConfigOptionTruthy, + MetricsMapping, + MetricsPredicate, + OpenMetricsBaseCheckV2, + ) + ) diff --git a/datadog_checks_dev/changelog.d/23813.added b/datadog_checks_dev/changelog.d/23813.added new file mode 100644 index 0000000000000..d7bc9bd0f1aaf --- /dev/null +++ b/datadog_checks_dev/changelog.d/23813.added @@ -0,0 +1 @@ +Fail `ddev validate agent-reqs` when `requirements-agent-release.txt` pins a `datadog-*` package whose integration folder is no longer present in the repo. diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/agent_reqs.py b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/agent_reqs.py index e9905670273d3..0d5d1bdcf4093 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/agent_reqs.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/agent_reqs.py @@ -13,9 +13,14 @@ echo_warning, ) from datadog_checks.dev.tooling.constants import AGENT_V5_ONLY, NOT_CHECKS, get_agent_release_requirements -from datadog_checks.dev.tooling.release import get_package_name +from datadog_checks.dev.tooling.release import get_folder_name, get_package_name from datadog_checks.dev.tooling.testing import process_checks_option -from datadog_checks.dev.tooling.utils import complete_valid_checks, get_version_string, parse_agent_req_file +from datadog_checks.dev.tooling.utils import ( + complete_valid_checks, + get_valid_checks, + get_version_string, + parse_agent_req_file, +) from datadog_checks.dev.utils import read_file @@ -63,6 +68,33 @@ def agent_reqs(check): if unreleased_checks: joined_checks = ', '.join(unreleased_checks) echo_warning(f"{len(unreleased_checks)} unreleased checks: {joined_checks}") + if check is None or check.lower() == 'all': + stale_released_checks = find_stale_released_checks(agent_reqs_content) + if stale_released_checks: + failed_checks += len(stale_released_checks) + for package_name in stale_released_checks: + folder_name = get_folder_name(package_name) + message = ( + f"{package_name} is pinned in requirements-agent-release.txt " + f"but `{folder_name}` is not present in the repo" + ) + echo_failure(message) + annotate_error(release_requirements_file, message) if failed_checks: echo_failure(f"{failed_checks} checks out of sync") abort() + + +def find_stale_released_checks(agent_reqs_content: dict[str, str]) -> list[str]: + """Return pinned Agent packages that no longer match a repo check.""" + expected_packages = { + get_package_name(check_name) + for check_name in get_valid_checks() + if check_name not in AGENT_V5_ONLY | NOT_CHECKS + } + + return sorted( + package_name + for package_name in agent_reqs_content + if package_name.startswith('datadog-') and package_name not in expected_packages + ) diff --git a/datadog_checks_dev/tests/tooling/commands/validate/test_agent_reqs.py b/datadog_checks_dev/tests/tooling/commands/validate/test_agent_reqs.py new file mode 100644 index 0000000000000..96ebed600a267 --- /dev/null +++ b/datadog_checks_dev/tests/tooling/commands/validate/test_agent_reqs.py @@ -0,0 +1,66 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os + +import pytest +from click.testing import CliRunner + +from datadog_checks.dev.tooling.commands.validate.agent_reqs import agent_reqs +from datadog_checks.dev.tooling.constants import get_root, set_root + + +@pytest.fixture +def isolated_root(): + runner = CliRunner() + previous_root = get_root() + with runner.isolated_filesystem(): + set_root(os.getcwd()) + try: + yield runner + finally: + set_root(previous_root) + + +def test_validate_agent_reqs_fails_on_stale_release_entry(isolated_root): + write_check('foo', '1.0.0') + with open('requirements-agent-release.txt', 'w', encoding='utf-8') as f: + f.write('# DO NOT PASS THIS TO PIP DIRECTLY\ndatadog-foo==1.0.0\ndatadog-snowflake==7.13.0\n') + + result = isolated_root.invoke(agent_reqs) + + assert result.exit_code == 1 + assert ( + 'datadog-snowflake is pinned in requirements-agent-release.txt but `snowflake` is not present in the repo' + ) in result.output + assert 'datadog-foo is pinned' not in result.output + + +def test_validate_agent_reqs_passes_when_every_entry_has_a_folder(isolated_root): + write_check('foo', '1.0.0') + with open('requirements-agent-release.txt', 'w', encoding='utf-8') as f: + f.write('# DO NOT PASS THIS TO PIP DIRECTLY\ndatadog-foo==1.0.0\n') + + result = isolated_root.invoke(agent_reqs) + + assert result.exit_code == 0 + assert 'pinned in requirements-agent-release.txt' not in result.output + + +def test_validate_agent_reqs_does_not_report_stale_entries_when_scoped_to_a_check(isolated_root): + write_check('foo', '1.0.0') + with open('requirements-agent-release.txt', 'w', encoding='utf-8') as f: + f.write('# DO NOT PASS THIS TO PIP DIRECTLY\ndatadog-foo==1.0.0\ndatadog-snowflake==7.13.0\n') + + result = isolated_root.invoke(agent_reqs, ['foo']) + + assert result.exit_code == 0 + assert 'datadog-snowflake is pinned' not in result.output + + +def write_check(name: str, version: str) -> None: + """Create the minimum check structure needed by agent-reqs.""" + check_package = os.path.join(name, 'datadog_checks', name) + os.makedirs(check_package) + with open(os.path.join(check_package, '__about__.py'), 'w', encoding='utf-8') as f: + f.write(f'__version__ = "{version}"\n') diff --git a/datadog_checks_downloader/changelog.d/23144.added b/datadog_checks_downloader/changelog.d/23144.added new file mode 100644 index 0000000000000..2b3e21333eff5 --- /dev/null +++ b/datadog_checks_downloader/changelog.d/23144.added @@ -0,0 +1 @@ +Add v2 TUF pointer downloader support. diff --git a/datadog_checks_downloader/datadog_checks/downloader/cli.py b/datadog_checks_downloader/datadog_checks/downloader/cli.py index be3776c3d3682..8cdd29f44af20 100644 --- a/datadog_checks_downloader/datadog_checks/downloader/cli.py +++ b/datadog_checks_downloader/datadog_checks/downloader/cli.py @@ -2,16 +2,30 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations # 1st party. import argparse +import logging import os import re import sys +import urllib.error # 2nd party. +from tuf.api.exceptions import DownloadError + from .download import DEFAULT_ROOT_LAYOUT_TYPE, REPOSITORY_URL_PREFIX, ROOT_LAYOUTS, TUFDownloader -from .exceptions import NonCanonicalVersion, NonDatadogPackage +from .download_v2 import V2_REPOSITORY_URL, TUFPointerDownloader +from .exceptions import CLIError, MissingVersion, NonCanonicalVersion, NonDatadogPackage, TargetNotFoundError + +V2_FALLBACK_ERRORS: tuple[type[BaseException], ...] = ( + MissingVersion, + TargetNotFoundError, + DownloadError, + TimeoutError, + urllib.error.URLError, +) # Private module functions. @@ -25,6 +39,14 @@ def __is_canonical(version): return re.match(P, version) is not None +def _v2_failure_category(exc: Exception) -> str: + if isinstance(exc, TargetNotFoundError): + return 'target version not found' + if isinstance(exc, (DownloadError, TimeoutError, urllib.error.URLError)): + return 'network error' + return 'other' + + def __find_shipped_integrations(): # Recurse up from site-packages until we find the Agent root directory. # The relative path differs between operating systems. @@ -142,6 +164,88 @@ def run_downloader(tuf_downloader, standard_distribution_name, version, ignore_p # Public module functions. -def download(): - tuf_downloader, standard_distribution_name, version, ignore_python_version = instantiate_downloader() - run_downloader(tuf_downloader, standard_distribution_name, version, ignore_python_version) +def download() -> None: + downloader, name, version, args = instantiate_v2_downloader() + + if args.v2: + warn_v2_ignored_args(args) + run_v2_downloader(downloader, name, version) + return + + try: + run_v2_downloader(downloader, name, version) + except V2_FALLBACK_ERRORS as exc: + # Integrity failures (DigestMismatch / LengthMismatch / MalformedPointerError) are + # intentionally not in V2_FALLBACK_ERRORS β€” they must propagate, not be masked by v1. + logging.getLogger(__name__).info( + 'v2 download failed (%s, %s: %s), falling back to v1', + _v2_failure_category(exc), + type(exc).__name__, + exc, + ) + run_downloader(*instantiate_downloader()) + except CLIError: + # NonDatadogPackage and NonCanonicalVersion: v1 would raise the same. + raise + + +def _v2_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + + parser.add_argument( + 'standard_distribution_name', + type=str, + help='Standard distribution name of the desired Datadog check, e.g. datadog-postgres.', + ) + parser.add_argument( + '--repository', type=str, default=V2_REPOSITORY_URL, help='HTTPS base URL of the v2 TUF repository.' + ) + parser.add_argument('--version', type=str, default=None, help='Version to download (default: latest stable).') + parser.add_argument( + '--unsafe-disable-verification', + action='store_true', + help='Disable TUF verification and wheel digest checks; requires --version and downloads /wheels directly.', + ) + parser.add_argument('-v', '--verbose', action='count', default=0) + parser.add_argument('--v2', action='store_true', default=False) + + # v1 compat flags accepted as no-ops so callers upgrading from v1 get a warning, not an error. + parser.add_argument('--type', type=str, default=None, dest='ignored_type') + parser.add_argument('--ignore-python-version', action='store_true', dest='ignored_ignore_python_version') + parser.add_argument('--force', action='store_true', dest='ignored_force') + + return parser + + +def warn_v2_ignored_args(args: argparse.Namespace) -> None: + if args.ignored_type is not None: + sys.stderr.write('WARNING: --type is not applicable with --v2 and will be ignored.\n') + if args.ignored_ignore_python_version: + sys.stderr.write( + 'NOTE: --ignore-python-version is not applicable with --v2 (wheel selection happens at publish time).\n' + ) + + +def instantiate_v2_downloader() -> tuple[TUFPointerDownloader, str, str | None, argparse.Namespace]: + args = _v2_parser().parse_args() + + if not args.standard_distribution_name.startswith('datadog-'): + raise NonDatadogPackage(args.standard_distribution_name) + + if args.version and not __is_canonical(args.version): + raise NonCanonicalVersion(args.version) + + remainder = min(args.verbose, 5) % 6 + level = (6 - remainder) * 10 + logging.basicConfig(format='%(levelname)-8s: %(message)s', level=level) + + downloader = TUFPointerDownloader( + repository_url=args.repository, + disable_verification=args.unsafe_disable_verification, + ) + return downloader, args.standard_distribution_name, args.version, args + + +def run_v2_downloader(downloader: TUFPointerDownloader, name: str, version: str | None) -> None: + wheel_path = downloader.download(name, version=version) + print(wheel_path) # pylint: disable=print-statement diff --git a/datadog_checks_downloader/datadog_checks/downloader/data/v2/metadata/root.json b/datadog_checks_downloader/datadog_checks/downloader/data/v2/metadata/root.json new file mode 100644 index 0000000000000..e053044657c99 --- /dev/null +++ b/datadog_checks_downloader/datadog_checks/downloader/data/v2/metadata/root.json @@ -0,0 +1,191 @@ +{ + "signatures": [ + { + "keyid": "ac5d650bc9aa17fdad54753fbf64e083f6f613286d0feef991ff61ec26874f2b", + "sig": "3066023100beb16cde4c9e17c725713c6020cb5b11a65dd60a7b46f6842068815593e8f39cfa547ea89b6169474a8ee6a98c22f934023100f215434d25181f6f0d6a75a1bae4f09814678d3409cc0aaf0f8e1cb2b0a7bcb29acd5394b4df1751f7e73c641a17256b" + }, + { + "keyid": "6f0f52eb4cb14d590aafd5f7eb8d9a79477ac89794be2d3caade9fc39b3735e6", + "sig": "306502306130792e890c5257cc8fc951e7c9a4009a5552affbd6bdf5cef3bac647de18f82918adbd4edfaa6d1624e2c378ee0ca4023100db6cff59e145b0113560116eac4466e52fc24e3ec3f02ee39fe7213718c2184d58e5473a5f29429b1a4c63e7ff87b83b" + }, + { + "keyid": "2d019dcc7a3e8da4d22bf364f0e0cb87937b2ced68339f3c53d305d1a9aadcce", + "sig": "3066023100ed93223b4ab9784c00b73937cd431fe9d6af8906548124a10215ad93432523476a3265e712fe99b7555ea30ce5aeff30023100e55ae85f68da6ac322f912cae2a28facb6b3f014770d348a7c9daee100e5eb8ae78cc86321b20711f3b357d900a075a4" + }, + { + "keyid": "e942404daa3e8cb1143ab5f275df2f8c741ae002194147806bd6f05b8e2e816f", + "sig": "3066023100a1a75a85dbe43db459e3d8c1bd935f2717bae0b1cba79ea5b9a5e785b7eb08cc30e08f96ba5fc0ccb9b9c97b9af456450231008dcc197a5de9a93649dfbd27e3f112321441913138c3377487ae85353a982cbffdc88029d681e432e86cfc14c51196ab" + }, + { + "keyid": "1286a08794005a5f1d679e56322f45fd3b55aa198f87bdc699f8213048602000", + "sig": "3064023075188913725a1c2e9af59f8663b6a178156b64d87da126a5970a3b6a3399bdfe7f5c357099f2f1a4e83d52294551c41e0230080ef2ff77b7d558879cbe0eda409c3ba2fe080860506d4f2ede314374e39dc0b2bc466af51fdd258eb76171344d42af" + }, + { + "keyid": "65ccb05ff16285a3b65ea2db2581ed083bb19acfcbd130d5484c151baf28541f", + "sig": "306602310086b9d6f39f795ad188223318f02e1d78b5798d34e333c1933e55891cc1b11cd6771b2d9ab1f5c1fc707d4815e3cc200d023100eb63f35f7cc2d0166357f2c209ecca63b82fc6bc9c310b9a0fa345957b1a0df102036ea6a6d3825d787eb7d3b3131e70" + }, + { + "keyid": "b59ade3245077bd622dc7bf41163a877e05272590cb4830632dc0d034717d735", + "sig": "3066023100fa26ef91f1bb3cdf779cb6bbc43d70bab67a7c66103e61b8998698f469fad0d44002d4a9399ceac304b8ee1a8823fd99023100ecd415e58696ab4778f4bdf5187be3743ac372b29cd139111b3461a0da42f8f44e5bb3ec83c2bd0ce4b6281e585b8889" + }, + { + "keyid": "a07e905cad57b71374ef5e408d61936c31957b35026de0b8db3938878ccad637", + "sig": "3066023100f4802957c21a0916677154494c4360260f5994c35c435d2bf2df39bc7cccca7fb437563d21ae128bcaa7909ead7d6e7802310097513f90e5e7dbe4bcb3f9b20308e966ec38960e8cad4869a4b32be8bd98726ded4a68c671d5f22858dd10ba3b56b04a" + }, + { + "keyid": "a442c20904f96e3a367e16037665bfb2e002bb2e9586cec4c96d83697a49fa2a", + "sig": "30660231008d58d822ee1accd6bff07e79f171d61d122d35c1d51c86b2f2ada76cff695090fdf859127889f9a8d90e539277b5ab5a023100f0ad8d7ba6a25e27316a91bbc61c9b4d31f42c5c93662ea53d660af6b8ab9ee111b135b6844901ccd281fd9246d5f786" + }, + { + "keyid": "8969905969a712d54c9b327939aead62784587b54d1d03cbaa835f79205069bf", + "sig": "3066023100894ecac9291e64ea6b84d168b886ef5829f4ad5b57c83b0ec745b644e4d19983e29c4681b5f744070a679e71fb4325af023100b6365d50a44e59932ea24d17a9fbc975cc7e7b44539d36dc7208470b1ddc9572b06266e149d1793a12a98cd62b803a95" + } + ], + "signed": { + "_type": "root", + "consistent_snapshot": true, + "expires": "2027-04-21T15:31:05Z", + "keys": { + "1286a08794005a5f1d679e56322f45fd3b55aa198f87bdc699f8213048602000": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEruzzCikai9w8LqTLE4cxf0qRIFU6AQve\nnMmudDdNo22MCiOwbuYjJJ1dvRlMiSVrAGyv1+37h8aXGa5Qbx5nb4TEIRfaDth8\nhMbKJcQ7OOK/6SaltjNZh3VaZ396/WIC\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@nouemankhal" + }, + "2d019dcc7a3e8da4d22bf364f0e0cb87937b2ced68339f3c53d305d1a9aadcce": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAErfozI3wqaB8k6o6Mc7SPFiw8s1dLTaxk\nMmhMsdkk7QIl3t+gFzWNdXANEjN027g4S6Ty2CvdzovU37yD24td9pQBh8LGmfPa\nmU5cxtzRaXkCibibJrrvLxyyZTWZXW6C\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@alexeypilyugin" + }, + "4542ee95093cb434e0d80a4bb9dd9d96e6b67cda12759fa2648a7786f822e97d": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEUV4g/gyxCdXKHK07QWO5z6S9lRhL88DO\nOb22g0dCOtxBB2sKojAUw3wXXz+SaUZRFgqfVvezbtsC4LSkkIlwA5MrJDA83kP2\nJRo4BQPtW8wZmtSvkkRQPSfAdXv975pg\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-online-uri": "awskms:arn:aws:kms:us-east-1:510233252802:key/9efe9e34-88f3-4ad3-8828-5340561e7c42" + }, + "65ccb05ff16285a3b65ea2db2581ed083bb19acfcbd130d5484c151baf28541f": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEi44mg+tnJn41Cy4Lr42lQNRuZaHDY4d+\nB/oYkBRTiHl6n6hc6alGLS/1rWijAfSL7x7wgVeOrA5fp1ornW27vPOkRVWJO5Lv\nZcZXwJYi7svVFBkFjBAtAOF6DGuAEWc9\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@lucia-sb" + }, + "6f0f52eb4cb14d590aafd5f7eb8d9a79477ac89794be2d3caade9fc39b3735e6": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEAbWHH4rfNiJFz9gXLPV/QJK0tky4/nW1\nyMPnUe1GRac6UfGcjZvGA7mpmns4FYG1KuHbPhWlEDOQnLjiIiJkY2+Z96tywq6y\n+/e+0Gc2KSsVr0IAWALkTzQE+Q6ru+lj\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@nubtron" + }, + "8969905969a712d54c9b327939aead62784587b54d1d03cbaa835f79205069bf": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEkztjp5ixZrKt94qnSn4bisyEdgs0Wre/\nheazr1zx7MJUCLiHim0lEDWCB64m/YLru+W3/PLwTiQSavO62lB6y3ggjcq/ygwA\n5yxi0bP/MAJBZ0Hl+y+Q8BfKTZSrTb6j\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@hadhemidd" + }, + "a07e905cad57b71374ef5e408d61936c31957b35026de0b8db3938878ccad637": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEEilQwnno5GxJpoyxulKzkkHa0x0/ERDa\nf3m1ZCpF9SoT2B98T+BwT6noD+qlOwX7VKLFSQwl4/od53tu6Wt3s3P70zFviq+Y\n+chUOSCbA5y/TCvfwx4mLBruXI1QbVOh\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@aarakke" + }, + "a442c20904f96e3a367e16037665bfb2e002bb2e9586cec4c96d83697a49fa2a": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEUJ7k4tiIZrWNLhrNrcBBMh4we3GiMlpo\ntwVy72lNw7aMxisK6ttP0mV30Yh1rX37DO6UUdeiWImrYBVfXFkP7z2QD9qKetny\nCeVHycA7uNby7yb7pljv2l2SpTgXACZk\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@iliakur" + }, + "ac5d650bc9aa17fdad54753fbf64e083f6f613286d0feef991ff61ec26874f2b": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEAWOvhm6nk7iY+EYK8ZnrxS49yqLf/ZTR\nJ74WY9Kz3ikjyXASkD4IgqJyyrmbMoqS9k6/RM/Zk6CAfPeZneDh1puVAlxy9nJD\nZp/OW78dVOqrlw1uQ0d+gfe7b4TcUNG4\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@dkirov-dd" + }, + "b59ade3245077bd622dc7bf41163a877e05272590cb4830632dc0d034717d735": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEd/9wooA4OKbC7hUO1OTZN3pnFbc85PDs\n+izKkDDSqj3yk8Pa39OJstT2BHvrn/B0BKMHhE6T/PN/rhorKVIVZ3UZErn1QCgG\nkkcFfA5MQm92SjIr9zAJea9bVUJhZ+PA\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@sarah-witt" + }, + "e942404daa3e8cb1143ab5f275df2f8c741ae002194147806bd6f05b8e2e816f": { + "keytype": "ecdsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE3VG/DJn/wmXh3bQ/LLjGMyKubQ1f5/1P\nJTVDYgTh5AC5zWxDSD26PoNpS29MecItPoM+pMy5YC99mwkEkxjNdwIke1Aons92\n8SVtL3BYH311oC6jLtFt+oqEunL5EdgJ\n-----END PUBLIC KEY-----\n" + }, + "scheme": "ecdsa-sha2-nistp384", + "x-tuf-on-ci-keyowner": "@kyle-neale" + } + }, + "roles": { + "root": { + "keyids": [ + "ac5d650bc9aa17fdad54753fbf64e083f6f613286d0feef991ff61ec26874f2b", + "6f0f52eb4cb14d590aafd5f7eb8d9a79477ac89794be2d3caade9fc39b3735e6", + "2d019dcc7a3e8da4d22bf364f0e0cb87937b2ced68339f3c53d305d1a9aadcce", + "e942404daa3e8cb1143ab5f275df2f8c741ae002194147806bd6f05b8e2e816f", + "1286a08794005a5f1d679e56322f45fd3b55aa198f87bdc699f8213048602000", + "65ccb05ff16285a3b65ea2db2581ed083bb19acfcbd130d5484c151baf28541f", + "b59ade3245077bd622dc7bf41163a877e05272590cb4830632dc0d034717d735", + "a07e905cad57b71374ef5e408d61936c31957b35026de0b8db3938878ccad637", + "a442c20904f96e3a367e16037665bfb2e002bb2e9586cec4c96d83697a49fa2a", + "8969905969a712d54c9b327939aead62784587b54d1d03cbaa835f79205069bf" + ], + "threshold": 2 + }, + "snapshot": { + "keyids": [ + "4542ee95093cb434e0d80a4bb9dd9d96e6b67cda12759fa2648a7786f822e97d" + ], + "threshold": 1, + "x-tuf-on-ci-expiry-period": 365, + "x-tuf-on-ci-signing-period": 60 + }, + "targets": { + "keyids": [ + "ac5d650bc9aa17fdad54753fbf64e083f6f613286d0feef991ff61ec26874f2b", + "6f0f52eb4cb14d590aafd5f7eb8d9a79477ac89794be2d3caade9fc39b3735e6", + "2d019dcc7a3e8da4d22bf364f0e0cb87937b2ced68339f3c53d305d1a9aadcce", + "e942404daa3e8cb1143ab5f275df2f8c741ae002194147806bd6f05b8e2e816f", + "1286a08794005a5f1d679e56322f45fd3b55aa198f87bdc699f8213048602000", + "65ccb05ff16285a3b65ea2db2581ed083bb19acfcbd130d5484c151baf28541f", + "b59ade3245077bd622dc7bf41163a877e05272590cb4830632dc0d034717d735", + "a07e905cad57b71374ef5e408d61936c31957b35026de0b8db3938878ccad637", + "a442c20904f96e3a367e16037665bfb2e002bb2e9586cec4c96d83697a49fa2a", + "8969905969a712d54c9b327939aead62784587b54d1d03cbaa835f79205069bf" + ], + "threshold": 1 + }, + "timestamp": { + "keyids": [ + "4542ee95093cb434e0d80a4bb9dd9d96e6b67cda12759fa2648a7786f822e97d" + ], + "threshold": 1, + "x-tuf-on-ci-expiry-period-hours": 48, + "x-tuf-on-ci-signing-period-hours": 24 + } + }, + "spec_version": "1.0.31", + "version": 1, + "x-tuf-on-ci-expiry-period": 365, + "x-tuf-on-ci-signing-period": 60 + } +} \ No newline at end of file diff --git a/datadog_checks_downloader/datadog_checks/downloader/download_v2.py b/datadog_checks_downloader/datadog_checks/downloader/download_v2.py new file mode 100644 index 0000000000000..82557de31f4c9 --- /dev/null +++ b/datadog_checks_downloader/datadog_checks/downloader/download_v2.py @@ -0,0 +1,140 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +"""TUF pointer-file downloader for the v2 repository format.""" + +from __future__ import annotations + +import hashlib +import importlib.resources +import json +import logging +import tempfile +import urllib.request +from pathlib import Path + +from tuf.ngclient import Updater +from tuf.ngclient.config import UpdaterConfig + +from .exceptions import ( + DigestMismatch, + LengthMismatch, + MalformedPointerError, + MissingVersion, + TargetNotFoundError, +) + +logger = logging.getLogger(__name__) + +V2_REPOSITORY_URL = "https://agent-integration-wheels-prod.s3.amazonaws.com" + +# tuf.ngclient sets its own fetcher timeout; this applies only to the raw wheel urlopen(). +WHEEL_FETCH_TIMEOUT_SECONDS = 60 + +REQUIRED_POINTER_KEYS = ('digest', 'length', 'wheel_path') + + +class TUFPointerDownloader: + """Downloads Datadog integration wheels from a v2 TUF repository.""" + + def __init__(self, repository_url: str, disable_verification: bool = False): + self._repository_url = repository_url.rstrip('/') + self._disable_verification = disable_verification + + if disable_verification: + logger.warning('Running with TUF verification disabled. Integrity is protected only by TLS (HTTPS).') + + def _bootstrap_metadata_dir(self, metadata_dir: Path) -> None: + dest = metadata_dir / 'root.json' + metadata = importlib.resources.files('datadog_checks.downloader') / 'data' / 'v2' / 'metadata' + dest.write_bytes((metadata / 'root.json').read_bytes()) + + def _make_updater(self, metadata_dir: Path, target_dir: Path) -> Updater: + return Updater( + metadata_dir=str(metadata_dir), + metadata_base_url=f'{self._repository_url}/metadata/', + target_base_url=f'{self._repository_url}/targets/', + target_dir=str(target_dir), + config=UpdaterConfig(prefix_targets_with_hash=True), + ) + + @staticmethod + def _target_path(project: str, version: str | None) -> str: + name = version if version is not None else 'latest' + return f'{project}/{name}.json' + + @staticmethod + def _wheel_filename(project: str, version: str) -> str: + distribution = project.replace('-', '_') + return f'{distribution}-{version}-py3-none-any.whl' + + def _direct_wheel_url(self, project: str, version: str) -> str: + return f'{self._repository_url}/wheels/{project}/{self._wheel_filename(project, version)}' + + @staticmethod + def _validate_pointer(project: str, pointer: dict) -> None: + for key in REQUIRED_POINTER_KEYS: + if key not in pointer: + raise MalformedPointerError(project, key) + if not pointer['wheel_path'].startswith('/'): + raise MalformedPointerError(project, 'wheel_path') + + @staticmethod + def _verify_content(project: str, content: bytes, pointer: dict) -> None: + if len(content) != pointer['length']: + raise LengthMismatch(project, pointer['length'], len(content)) + actual_digest = hashlib.sha256(content).hexdigest() + if actual_digest != pointer['digest']: + raise DigestMismatch(project, pointer['digest'], actual_digest) + + def get_pointer(self, project: str, version: str | None = None) -> dict: + """Return the pointer JSON for *project* at *version* (or 'latest' when None).""" + with tempfile.TemporaryDirectory() as tmp: + metadata_dir = Path(tmp) / 'metadata' + target_dir = Path(tmp) / 'targets' + metadata_dir.mkdir() + target_dir.mkdir() + + target_path = self._target_path(project, version) + self._bootstrap_metadata_dir(metadata_dir) + updater = self._make_updater(metadata_dir, target_dir) + updater.refresh() + + target_info = updater.get_targetinfo(target_path) + if target_info is None: + label = version if version is not None else 'latest stable' + raise TargetNotFoundError(f'No TUF target for {project!r} version {label!r}') + + pointer_path = target_dir / target_path + pointer_path.parent.mkdir(parents=True, exist_ok=True) + updater.download_target(target_info, pointer_path) + + return json.loads(pointer_path.read_text(encoding='utf-8')) + + def download(self, project: str, version: str | None = None, dest_dir: Path | None = None) -> Path: + """Download and verify the wheel for *project* at *version*; return its path.""" + if self._disable_verification: + if version is None: + raise MissingVersion('unsafe-disable-verification requires an explicit --version') + wheel_url = self._direct_wheel_url(project, version) + wheel_filename = self._wheel_filename(project, version) + pointer: dict | None = None + else: + pointer = self.get_pointer(project, version) + self._validate_pointer(project, pointer) + wheel_url = self._repository_url + pointer['wheel_path'] + wheel_filename = Path(pointer['wheel_path']).name + + dest = (dest_dir or Path(tempfile.mkdtemp())) / wheel_filename + + logger.info('Downloading wheel from %s', wheel_url) + with urllib.request.urlopen(wheel_url, timeout=WHEEL_FETCH_TIMEOUT_SECONDS) as resp: + content = resp.read() + + if pointer is not None: + self._verify_content(project, content, pointer) + + dest.write_bytes(content) + logger.info('Wrote %s to %s', wheel_filename, dest) + return dest diff --git a/datadog_checks_downloader/datadog_checks/downloader/exceptions.py b/datadog_checks_downloader/datadog_checks/downloader/exceptions.py index bb6b75e05a156..db8764040a700 100644 --- a/datadog_checks_downloader/datadog_checks/downloader/exceptions.py +++ b/datadog_checks_downloader/datadog_checks/downloader/exceptions.py @@ -30,6 +30,10 @@ def __str__(self): return '{}'.format(self.standard_distribution_name) +class MissingVersion(CLIError): + """Raised when --version is required but absent (e.g. with --unsafe-disable-verification).""" + + # Exceptions for the download module. @@ -37,6 +41,41 @@ class TargetNotFoundError(ChecksDownloaderException): """An exception raised when a target is not found.""" +class MalformedPointerError(ChecksDownloaderException): + """Raised when a TUF-signed pointer JSON is invalid or missing fields.""" + + def __init__(self, project: str, field: str): + self.project = project + self.field = field + + def __str__(self) -> str: + return f'{self.project}: pointer field {self.field!r} is missing or malformed' + + +class DigestMismatch(ChecksDownloaderException): + """Raised when the downloaded wheel's sha256 does not match the pointer.""" + + def __init__(self, project: str, expected: str, actual: str): + self.project = project + self.expected = expected + self.actual = actual + + def __str__(self) -> str: + return f'{self.project}: expected digest {self.expected}, got {self.actual}' + + +class LengthMismatch(ChecksDownloaderException): + """Raised when the downloaded wheel's byte length does not match the pointer.""" + + def __init__(self, project: str, expected: int, actual: int): + self.project = project + self.expected = expected + self.actual = actual + + def __str__(self) -> str: + return f'{self.project}: expected length {self.expected}, got {self.actual}' + + class IncorrectRootLayoutType(ChecksDownloaderException): def __init__(self, found, expected): self.found = found diff --git a/datadog_checks_downloader/pyproject.toml b/datadog_checks_downloader/pyproject.toml index 56ecc4d80baee..b40dcb4e75d39 100644 --- a/datadog_checks_downloader/pyproject.toml +++ b/datadog_checks_downloader/pyproject.toml @@ -55,6 +55,9 @@ include = [ include = [ "/datadog_checks/downloader", ] +artifacts = [ + "/datadog_checks/downloader/data/v2/metadata/root.json", +] dev-mode-dirs = [ ".", ] diff --git a/datadog_checks_downloader/tests/test_unit.py b/datadog_checks_downloader/tests/test_unit.py index 9170abf3ee09a..160b7c584aeef 100644 --- a/datadog_checks_downloader/tests/test_unit.py +++ b/datadog_checks_downloader/tests/test_unit.py @@ -1,7 +1,28 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import urllib.error + +import pytest +from tuf.api.exceptions import DownloadError + +from datadog_checks.downloader.cli import _v2_failure_category from datadog_checks.downloader.download import TUFDownloader +from datadog_checks.downloader.exceptions import TargetNotFoundError + + +@pytest.mark.parametrize( + 'exc,expected', + [ + pytest.param(TargetNotFoundError('missing'), 'target version not found', id='target-not-found'), + pytest.param(urllib.error.URLError('timeout'), 'network error', id='network-urlerror'), + pytest.param(DownloadError('boom'), 'network error', id='network-downloaderror'), + pytest.param(TimeoutError('slow'), 'network error', id='network-timeout'), + pytest.param(ValueError('bad pointer'), 'other', id='other'), + ], +) +def test_v2_failure_category(exc, expected): + assert _v2_failure_category(exc) == expected def test_non_official_wheel_filter(mocker): diff --git a/datadog_checks_downloader/tests/test_v2_downloader.py b/datadog_checks_downloader/tests/test_v2_downloader.py new file mode 100644 index 0000000000000..db1977db2be0d --- /dev/null +++ b/datadog_checks_downloader/tests/test_v2_downloader.py @@ -0,0 +1,319 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +"""Unit tests for TUFPointerDownloader (v2 repository format) and the v2 CLI surface.""" + +import hashlib +import json +import urllib.error +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from tuf.api.exceptions import DownloadError + +from datadog_checks.downloader import cli +from datadog_checks.downloader.download_v2 import TUFPointerDownloader +from datadog_checks.downloader.exceptions import ( + DigestMismatch, + LengthMismatch, + MalformedPointerError, + MissingVersion, + NonCanonicalVersion, + NonDatadogPackage, + TargetNotFoundError, +) + +pytestmark = pytest.mark.offline + +PROJECT = 'datadog-postgres' +VERSION = '14.0.0' +WHEEL_NAME = f'datadog_postgres-{VERSION}-py3-none-any.whl' +WHEEL_CONTENT = b'fake wheel bytes for testing' +WHEEL_DIGEST = hashlib.sha256(WHEEL_CONTENT).hexdigest() +WHEEL_LENGTH = len(WHEEL_CONTENT) +REPO_URL = 'https://agent-integration-wheels-staging.s3.amazonaws.com' + +POINTER = { + 'digest': WHEEL_DIGEST, + 'length': WHEEL_LENGTH, + 'version': VERSION, + 'repository': REPO_URL, + 'wheel_path': f'/wheels/{PROJECT}/{WHEEL_NAME}', + 'attestation_path': f'/attestations/{PROJECT}/{VERSION}.sigstore.json', +} + + +def _mock_tuf_updater(pointer: dict) -> MagicMock: + pointer_bytes = json.dumps(pointer).encode() + mock_updater = MagicMock() + mock_updater.get_targetinfo.return_value = MagicMock() + + def fake_download_target(_target_info, dest_path): + Path(dest_path).parent.mkdir(parents=True, exist_ok=True) + Path(dest_path).write_bytes(pointer_bytes) + + mock_updater.download_target.side_effect = fake_download_target + return mock_updater + + +def _mock_response(content: bytes) -> MagicMock: + response = MagicMock() + response.__enter__ = lambda s: s + response.__exit__ = MagicMock(return_value=False) + response.read.return_value = content + return response + + +@pytest.fixture +def mock_urlopen(): + with patch('datadog_checks.downloader.download_v2.urllib.request.urlopen') as mock: + mock.return_value = _mock_response(WHEEL_CONTENT) + yield mock + + +@pytest.fixture +def mock_updater_cls(): + with patch('datadog_checks.downloader.download_v2.Updater') as mock: + mock.return_value = _mock_tuf_updater(POINTER) + yield mock + + +class TestTargetResolution: + @pytest.mark.parametrize( + 'version,expected_target', + [ + pytest.param(VERSION, f'{PROJECT}/{VERSION}.json', id='explicit-version'), + pytest.param(None, f'{PROJECT}/latest.json', id='missing-version'), + ], + ) + def test_get_pointer_requests_expected_target(self, mock_urlopen, mock_updater_cls, version, expected_target): + downloader = TUFPointerDownloader(repository_url=REPO_URL) + downloader.get_pointer(PROJECT, version=version) + + mock_updater = mock_updater_cls.return_value + assert mock_updater.get_targetinfo.call_args[0][0] == expected_target + + +class TestHappyPath: + def test_download_returns_wheel_path(self, mock_urlopen, mock_updater_cls, tmp_path): + downloader = TUFPointerDownloader(repository_url=REPO_URL) + wheel_path = downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + + assert wheel_path.exists() + assert wheel_path.read_bytes() == WHEEL_CONTENT + assert wheel_path.name == WHEEL_NAME + + def test_repository_flag_overrides_pointer_repository(self, mock_urlopen, mock_updater_cls, tmp_path): + prod_pointer = {**POINTER, 'repository': 'https://agent-integration-wheels-prod.s3.amazonaws.com'} + mock_updater_cls.return_value = _mock_tuf_updater(prod_pointer) + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + + mock_urlopen.assert_called_once_with( + f'{REPO_URL}/wheels/{PROJECT}/{WHEEL_NAME}', + timeout=60, + ) + + +class TestTargetNotFound: + def test_raises_when_tuf_target_absent(self, mock_urlopen, mock_updater_cls): + mock_updater = MagicMock() + mock_updater.get_targetinfo.return_value = None + mock_updater_cls.return_value = mock_updater + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + with pytest.raises(TargetNotFoundError, match=PROJECT): + downloader.get_pointer(PROJECT, version='99.99.99') + + +class TestDigestMismatch: + def test_raises_on_corrupted_wheel(self, mock_urlopen, mock_updater_cls, tmp_path): + tampered = b'tampered bytes that match the pointer length'[:WHEEL_LENGTH] + mock_urlopen.return_value = _mock_response(tampered) + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + with pytest.raises(DigestMismatch, match=PROJECT): + downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + assert not (tmp_path / WHEEL_NAME).exists() + + +class TestLengthMismatch: + def test_raises_when_pointer_length_does_not_match_wheel(self, mock_urlopen, mock_updater_cls, tmp_path): + bad_pointer = {**POINTER, 'length': WHEEL_LENGTH + 1} + mock_updater_cls.return_value = _mock_tuf_updater(bad_pointer) + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + with pytest.raises(LengthMismatch) as exc_info: + downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + assert exc_info.value.expected == WHEEL_LENGTH + 1 + assert exc_info.value.actual == WHEEL_LENGTH + assert not (tmp_path / WHEEL_NAME).exists() + + +class TestMalformedPointer: + @pytest.mark.parametrize('missing_key', ['digest', 'length', 'wheel_path']) + def test_raises_when_required_key_missing(self, mock_urlopen, mock_updater_cls, tmp_path, missing_key): + broken_pointer = {k: v for k, v in POINTER.items() if k != missing_key} + mock_updater_cls.return_value = _mock_tuf_updater(broken_pointer) + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + with pytest.raises(MalformedPointerError, match=missing_key): + downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + + def test_raises_when_wheel_path_missing_leading_slash(self, mock_urlopen, mock_updater_cls, tmp_path): + no_slash_pointer = {**POINTER, 'wheel_path': f'wheels/{PROJECT}/{WHEEL_NAME}'} + mock_updater_cls.return_value = _mock_tuf_updater(no_slash_pointer) + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + with pytest.raises(MalformedPointerError, match='wheel_path'): + downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + mock_urlopen.assert_not_called() + + +class TestNetworkErrorMidDownload: + def test_http_error_propagates(self, mock_urlopen, mock_updater_cls, tmp_path): + mock_urlopen.side_effect = urllib.error.HTTPError( + url='http://example/x.whl', code=500, msg='boom', hdrs=None, fp=None + ) + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + with pytest.raises(urllib.error.HTTPError): + downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + + def test_url_error_propagates(self, mock_urlopen, mock_updater_cls, tmp_path): + mock_urlopen.side_effect = urllib.error.URLError('unreachable') + + downloader = TUFPointerDownloader(repository_url=REPO_URL) + with pytest.raises(urllib.error.URLError): + downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + + +class TestDisableVerification: + def test_directly_downloads_wheel_without_tuf_or_digest_checks(self, mock_urlopen, mock_updater_cls, tmp_path): + content = b'bytes not matching any signed pointer' + mock_urlopen.return_value = _mock_response(content) + + downloader = TUFPointerDownloader(repository_url=REPO_URL, disable_verification=True) + wheel_path = downloader.download(PROJECT, version=VERSION, dest_dir=tmp_path) + + mock_urlopen.assert_called_once_with( + f'{REPO_URL}/wheels/{PROJECT}/{WHEEL_NAME}', + timeout=60, + ) + assert wheel_path.name == WHEEL_NAME + assert wheel_path.read_bytes() == content + mock_updater_cls.assert_not_called() + + def test_direct_download_requires_explicit_version(self, tmp_path): + downloader = TUFPointerDownloader(repository_url=REPO_URL, disable_verification=True) + with pytest.raises(MissingVersion, match='requires an explicit --version'): + downloader.download(PROJECT, dest_dir=tmp_path) + + +class TestInstantiateV2Downloader: + def test_rejects_non_datadog_package(self, monkeypatch): + monkeypatch.setattr('sys.argv', ['downloader', 'requests']) + with pytest.raises(NonDatadogPackage, match='requests'): + cli.instantiate_v2_downloader() + + def test_rejects_non_canonical_version(self, monkeypatch): + monkeypatch.setattr('sys.argv', ['downloader', 'datadog-postgres', '--version', 'banana']) + with pytest.raises(NonCanonicalVersion, match='banana'): + cli.instantiate_v2_downloader() + + def test_does_not_warn_when_v1_compat_flags_are_parsed(self, monkeypatch, capsys): + monkeypatch.setattr('sys.argv', ['downloader', 'datadog-postgres', '--type', 'core', '--ignore-python-version']) + cli.instantiate_v2_downloader() + assert capsys.readouterr().err == '' + + def test_warns_for_v1_compat_flags_in_strict_v2_mode(self, monkeypatch, capsys): + monkeypatch.setattr( + 'sys.argv', ['downloader', 'datadog-postgres', '--v2', '--type', 'core', '--ignore-python-version'] + ) + _, _, _, args = cli.instantiate_v2_downloader() + cli.warn_v2_ignored_args(args) + stderr = capsys.readouterr().err + assert 'WARNING: --type' in stderr + assert 'NOTE: --ignore-python-version' in stderr + + def test_force_flag_is_silently_ignored(self, monkeypatch, capsys): + monkeypatch.setattr('sys.argv', ['downloader', 'datadog-postgres', '--force']) + cli.instantiate_v2_downloader() + assert capsys.readouterr().err == '' + + +class TestCliDownloadFallback: + """Covers the cli.download() v2-attempt-then-v1-fallback orchestration.""" + + def test_strict_v2_raises_on_v2_failure(self, monkeypatch): + monkeypatch.setattr('sys.argv', ['downloader', 'datadog-postgres', '--v2']) + monkeypatch.setattr(cli, 'run_v2_downloader', MagicMock(side_effect=TargetNotFoundError('missing'))) + v1 = MagicMock() + monkeypatch.setattr(cli, 'run_downloader', v1) + monkeypatch.setattr(cli, 'instantiate_downloader', MagicMock(return_value=(None, None, None, None))) + + with pytest.raises(TargetNotFoundError): + cli.download() + v1.assert_not_called() + + @pytest.mark.parametrize( + 'fallback_exc', + [ + pytest.param(MissingVersion('missing'), id='missing-version'), + pytest.param(TargetNotFoundError('missing'), id='target-not-found'), + pytest.param(DownloadError('unreachable'), id='download-error'), + pytest.param(TimeoutError('slow'), id='timeout-error'), + pytest.param(urllib.error.URLError('unreachable'), id='url-error'), + ], + ) + def test_default_falls_back_to_v1_on_expected_v2_failures(self, monkeypatch, fallback_exc): + monkeypatch.setattr('sys.argv', ['downloader', 'datadog-postgres']) + monkeypatch.setattr(cli, 'run_v2_downloader', MagicMock(side_effect=fallback_exc)) + v1 = MagicMock() + monkeypatch.setattr(cli, 'run_downloader', v1) + monkeypatch.setattr(cli, 'instantiate_downloader', MagicMock(return_value=('d', 'n', 'v', False))) + + cli.download() + v1.assert_called_once_with('d', 'n', 'v', False) + + def test_default_unsafe_disable_verification_without_version_falls_back_to_v1(self, monkeypatch): + monkeypatch.setattr('sys.argv', ['downloader', 'datadog-postgres', '--unsafe-disable-verification']) + monkeypatch.setattr(cli, 'run_v2_downloader', MagicMock(side_effect=MissingVersion('missing'))) + v1 = MagicMock() + monkeypatch.setattr(cli, 'run_downloader', v1) + monkeypatch.setattr(cli, 'instantiate_downloader', MagicMock(return_value=('d', 'n', None, False))) + + cli.download() + v1.assert_called_once_with('d', 'n', None, False) + + def test_non_datadog_package_does_not_fall_back_to_v1(self, monkeypatch): + monkeypatch.setattr('sys.argv', ['downloader', 'requests']) + v1 = MagicMock() + monkeypatch.setattr(cli, 'run_downloader', v1) + monkeypatch.setattr(cli, 'instantiate_downloader', MagicMock()) + + with pytest.raises(NonDatadogPackage): + cli.download() + v1.assert_not_called() + + @pytest.mark.parametrize( + 'integrity_exc', + [ + pytest.param(DigestMismatch(PROJECT, 'a', 'b'), id='digest-mismatch'), + pytest.param(LengthMismatch(PROJECT, 1, 2), id='length-mismatch'), + pytest.param(MalformedPointerError(PROJECT, 'digest'), id='malformed-pointer'), + ], + ) + def test_integrity_errors_do_not_fall_back_to_v1(self, monkeypatch, integrity_exc): + monkeypatch.setattr('sys.argv', ['downloader', 'datadog-postgres']) + monkeypatch.setattr(cli, 'run_v2_downloader', MagicMock(side_effect=integrity_exc)) + v1 = MagicMock() + monkeypatch.setattr(cli, 'run_downloader', v1) + monkeypatch.setattr(cli, 'instantiate_downloader', MagicMock()) + + with pytest.raises(type(integrity_exc)): + cli.download() + v1.assert_not_called() diff --git a/ddev/changelog.d/23360.changed b/ddev/changelog.d/23360.changed new file mode 100644 index 0000000000000..2769d0165dec4 --- /dev/null +++ b/ddev/changelog.d/23360.changed @@ -0,0 +1 @@ +Migrated ``ddev validate ci`` from Codecov to Datadog Code Coverage. diff --git a/ddev/changelog.d/23813.added b/ddev/changelog.d/23813.added new file mode 100644 index 0000000000000..d1c722f785ec2 --- /dev/null +++ b/ddev/changelog.d/23813.added @@ -0,0 +1 @@ +Skip integrations pinned in Agent release requirements but not actually shipped in a given Agent release, configurable under `[overrides.release.agent.unreleased-integrations]` in `.ddev/config.toml`. diff --git a/ddev/changelog.d/23828.added b/ddev/changelog.d/23828.added new file mode 100644 index 0000000000000..1a808e1d73e89 --- /dev/null +++ b/ddev/changelog.d/23828.added @@ -0,0 +1 @@ +Print the exact workflow run URL when dispatching `ddev dep promote`, via a new `return_run_details` option on `GitHubManager.dispatch_workflow`. diff --git a/ddev/hatch.toml b/ddev/hatch.toml index 16d9b3980fa02..9550fb4a939c3 100644 --- a/ddev/hatch.toml +++ b/ddev/hatch.toml @@ -9,6 +9,7 @@ python = "3.13" e2e-env = false dependencies = [ "pyyaml", + "pytest-asyncio", "vcrpy", ] # TODO: remove this when the old CLI is gone diff --git a/ddev/pyproject.toml b/ddev/pyproject.toml index b82b8db3bc028..73928e956e9d4 100644 --- a/ddev/pyproject.toml +++ b/ddev/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ "Programming Language :: Python :: 3.13", ] dependencies = [ - "anthropic>=0.18.0", + "anthropic>=0.86.0", "click~=8.1.6", "coverage", "datadog-api-client==2.20.0", diff --git a/ddev/src/ddev/ai/__init__.py b/ddev/src/ddev/ai/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/agent/__init__.py b/ddev/src/ddev/ai/agent/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/agent/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/agent/anthropic_client.py b/ddev/src/ddev/ai/agent/anthropic_client.py new file mode 100644 index 0000000000000..ae5de3a8e6d76 --- /dev/null +++ b/ddev/src/ddev/ai/agent/anthropic_client.py @@ -0,0 +1,232 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from __future__ import annotations + +from typing import TYPE_CHECKING, Final, overload + +import anthropic +from anthropic.types import MessageParam + +from ddev.ai.agent.base import BaseAgent +from ddev.ai.agent.exceptions import AgentAPIError, AgentConnectionError, AgentError, AgentRateLimitError +from ddev.ai.agent.types import AgentResponse, ContextUsage, StopReason, TokenUsage, ToolCall, ToolResultMessage +from ddev.ai.tools.registry import ToolRegistry + +if TYPE_CHECKING: + from anthropic.types import ( + CacheControlEphemeralParam, + TextBlockParam, + ToolParam, + ToolResultBlockParam, + ) + +DEFAULT_MODEL: Final[str] = "claude-sonnet-4-6" +DEFAULT_MAX_TOKENS: Final[int] = 8192 # max tokens per response + +# 1h TTL for the static prefix (system + tools): paid once, read for the whole session. +STATIC_CACHE_CONTROL: Final[CacheControlEphemeralParam] = {"type": "ephemeral", "ttl": "1h"} +# Default TTL (currently 5 min, but Anthropic may change it) for the sliding breakpoint +# on the last user message: re-written each turn, so a longer TTL would be wasted. +SLIDING_CACHE_CONTROL: Final[CacheControlEphemeralParam] = {"type": "ephemeral"} + + +class AnthropicAgent(BaseAgent[MessageParam]): + """A wrapper around the Anthropic API that provides a simple interface for interacting with agents.""" + + def __init__( + self, + client: anthropic.AsyncAnthropic, + tools: ToolRegistry, + system_prompt: str, + name: str, + model: str = DEFAULT_MODEL, + max_tokens: int = DEFAULT_MAX_TOKENS, + ) -> None: + """Initialize an AnthropicAgent. + Args: + client: The Anthropic client to use. + tools: The ToolRegistry to use (might not be used in every call if allowed_tools in send() is provided) + system_prompt: The system prompt to use. + name: The name of the agent. + model: The model to use. + max_tokens: The max tokens per response. + """ + + super().__init__(name=name, system_prompt=system_prompt, tools=tools) + self._client = client + self._model = model + self._max_tokens = max_tokens + self._context_window: int | None = None + + async def _get_context_window(self) -> int: + if self._context_window is None: + try: + info = await self._client.models.retrieve(self._model) + except anthropic.APIConnectionError as e: + raise AgentConnectionError(f"Connection failed: {e}") from e + except anthropic.RateLimitError as e: + raise AgentRateLimitError(f"Rate limit exceeded: {e}") from e + except anthropic.APIStatusError as e: + raise AgentAPIError(e.status_code, e.message) from e + except anthropic.APIResponseValidationError as e: + raise AgentError(f"Response validation failed: {e}") from e + + self._context_window = info.max_input_tokens + return self._context_window + + def _get_tool_definitions(self, allowed_tools: list[str] | None) -> list[ToolParam]: + """Filter tool definitions by allowlist. None means all tools.""" + definitions = self._tools.definitions + if allowed_tools is not None: + allowed = set(allowed_tools) + definitions = [d for d in definitions if d["name"] in allowed] + return definitions + + def _map_stop_reason(self, raw: str) -> StopReason: + """Map a raw Anthropic stop_reason string to the generic StopReason enum.""" + # pause_turn gets an explicit check to provide a more informative message than "Unknown stop_reason" + if raw == "pause_turn": + raise AgentError("pause_turn is not supported in batch mode") from None + mapping = { + "end_turn": StopReason.END_TURN, + "tool_use": StopReason.TOOL_USE, + "max_tokens": StopReason.MAX_TOKENS, + "stop_sequence": StopReason.OTHER, + "refusal": StopReason.OTHER, + } + if raw not in mapping: + raise AgentError(f"Unknown stop_reason: {raw!r}") from None + return mapping[raw] + + def _to_tool_result_params(self, messages: list[ToolResultMessage]) -> list[ToolResultBlockParam]: + """Convert model-agnostic ToolResultMessages to Anthropic SDK ToolResultBlockParams.""" + return [ + { + "type": "tool_result", + "tool_use_id": msg.tool_call_id, + "is_error": not msg.result.success, + **( + {"content": msg.result.data} + if msg.result.data is not None + else {"content": msg.result.error or "(unknown error)"} + if not msg.result.success + else {} + ), + } + for msg in messages + ] + + @overload + @staticmethod + def _with_user_cache_breakpoint(content: str) -> list[TextBlockParam]: ... + + @overload + @staticmethod + def _with_user_cache_breakpoint(content: list[ToolResultBlockParam]) -> list[ToolResultBlockParam]: ... + + @staticmethod + def _with_user_cache_breakpoint( + content: str | list[ToolResultBlockParam], + ) -> list[TextBlockParam] | list[ToolResultBlockParam]: + """Return a block list with a sliding cache breakpoint on the last block.""" + if isinstance(content, str): + return [{"type": "text", "text": content, "cache_control": SLIDING_CACHE_CONTROL}] + if not content: + return [] + return [*content[:-1], {**content[-1], "cache_control": SLIDING_CACHE_CONTROL}] + + @staticmethod + def _with_tools_cache_breakpoint(tool_defs: list[ToolParam]) -> list[ToolParam]: + """Return a tool list with a static cache breakpoint on the last tool.""" + if not tool_defs: + return tool_defs + return [*tool_defs[:-1], {**tool_defs[-1], "cache_control": STATIC_CACHE_CONTROL}] + + async def send( + self, + content: str | list[ToolResultMessage], + allowed_tools: list[str] | None = None, + ) -> AgentResponse: + """Send a message to the agent and return the response. + Args: + content: The content to send to the agent. + allowed_tools: The tools in the ToolRegistry to allow the agent to use. + Returns: + An AgentResponse object containing the response from the agent. + """ + tool_defs = self._with_tools_cache_breakpoint(self._get_tool_definitions(allowed_tools)) + + api_content: str | list[ToolResultBlockParam] = ( + self._to_tool_result_params(content) if isinstance(content, list) else content + ) + user_msg_for_history: MessageParam = {"role": "user", "content": api_content} + user_msg_for_request: MessageParam = { + "role": "user", + "content": self._with_user_cache_breakpoint(api_content), + } + messages = [*self._history, user_msg_for_request] + + system_param: list[TextBlockParam] = [ + {"type": "text", "text": self._system_prompt, "cache_control": STATIC_CACHE_CONTROL} + ] + + try: + response = await self._client.messages.create( + model=self._model, + max_tokens=self._max_tokens, + system=system_param, + messages=messages, + tools=tool_defs if tool_defs else anthropic.NOT_GIVEN, + ) + except anthropic.APIConnectionError as e: + raise AgentConnectionError(f"Connection failed: {e}") from e + except anthropic.RateLimitError as e: + raise AgentRateLimitError(f"Rate limit exceeded: {e}") from e + except anthropic.APIStatusError as e: + raise AgentAPIError(e.status_code, e.message) from e + except anthropic.APIResponseValidationError as e: + raise AgentError(f"Response validation failed: {e}") from e + + # stop_reason is None only in streaming responses; we use non-streaming, so None is unexpected + if response.stop_reason is None: + raise AgentError("Received null stop_reason from API") + + stop_reason = self._map_stop_reason(response.stop_reason) + + text_parts: list[str] = [] + tool_calls: list[ToolCall] = [] + + for block in response.content: + if isinstance(block, anthropic.types.TextBlock): + text_parts.append(block.text) + elif isinstance(block, anthropic.types.ToolUseBlock): + tool_calls.append(ToolCall(id=block.id, name=block.name, input=dict(block.input))) + # ThinkingBlock and RedactedThinkingBlock are intentionally ignored. + # Extended thinking support can add a `thinking: str` field to AgentResponse later. + + cache_read = response.usage.cache_read_input_tokens or 0 + cache_creation = response.usage.cache_creation_input_tokens or 0 + used_tokens = response.usage.input_tokens + cache_read + cache_creation + usage = TokenUsage( + input_tokens=response.usage.input_tokens, + output_tokens=response.usage.output_tokens, + cache_read_input_tokens=cache_read, + cache_creation_input_tokens=cache_creation, + context_usage=ContextUsage(window_size=await self._get_context_window(), used_tokens=used_tokens), + ) + + agent_response = AgentResponse( + stop_reason=stop_reason, + text="\n".join(text_parts), + tool_calls=tool_calls, + usage=usage, + ) + + # Save to history only after a successful response. Use the unmarked form so the + # cache_control breakpoint is only ever on the latest user message β€” this keeps the + # request below the 4-marker limit regardless of conversation length. + self._history.extend([user_msg_for_history, {"role": "assistant", "content": response.content}]) + + return agent_response diff --git a/ddev/src/ddev/ai/agent/base.py b/ddev/src/ddev/ai/agent/base.py new file mode 100644 index 0000000000000..ad7b57eeca84c --- /dev/null +++ b/ddev/src/ddev/ai/agent/base.py @@ -0,0 +1,101 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from abc import ABC, abstractmethod +from copy import deepcopy +from typing import Final + +from ddev.ai.agent.types import AgentResponse, ToolResultMessage +from ddev.ai.tools.registry import ToolRegistry + +_COMPACT_SYSTEM_PROMPT: Final[str] = """\ +You are summarizing an agentic conversation to free up context space. +Produce a dense, structured summary that covers ALL of the following: + 1. The original task given to the agent + 2. Every tool call made and the key finding or result from each + 3. Any decisions, conclusions, or hypotheses the agent reached + 4. What has been completed and what work remains + +Rules: +- Be exhaustive on facts and findings; omit raw data already consumed +- Use bullet points, not prose +- The agent will read ONLY this summary to continue β€” it must be self-sufficient +""" + +_COMPACT_REQUEST: Final[str] = "Summarize the conversation so far following your instructions." + + +class BaseAgent[TMessage](ABC): + """Abstract base class for all agent implementations. + + Provides shared, provider-agnostic history management and compaction. + The message type TMessage is supplied by each concrete provider + (e.g. MessageParam for Anthropic). Subclasses must implement send(). + """ + + def __init__(self, name: str, system_prompt: str, tools: ToolRegistry) -> None: + self._history: list[TMessage] = [] + self.name = name + self._system_prompt = system_prompt + self._tools = tools + + @property + def history(self) -> list[TMessage]: + """Read-only snapshot of the conversation history.""" + return deepcopy(self._history) + + def reset(self) -> None: + """Clear conversation history to start a new conversation.""" + self._history = [] + + async def compact(self) -> AgentResponse | None: + """Collapse history to 2 messages: original task + LLM summary. + + Returns the AgentResponse from the compaction call so callers can + account for its token usage. Returns None if history is already ≀ 2. + """ + if len(self._history) <= 2: + return None + + original_prompt = self._history[0] + original_system = self._system_prompt + + self._system_prompt = _COMPACT_SYSTEM_PROMPT + try: + response = await self.send(_COMPACT_REQUEST, allowed_tools=[]) + finally: + self._system_prompt = original_system # restore even if send() raises + + compact_response = self._history[-1] # summary message added by send() + + self.reset() + self._history = [original_prompt, compact_response] + return response + + async def compact_preserving_last_turn(self) -> AgentResponse | None: + """Compact history while keeping the last user+assistant pair intact. + + Used mid-ReAct loop where the last assistant message contains unresolved + tool calls that still need a tool-result response. After compaction the + preserved pair re-anchors the pending turn so the next send(tool_results) + produces a valid alternating message sequence. No-op if history is ≀ 3 + messages (too short to compact without corrupting the sequence). + + Returns the AgentResponse from the compaction call, or None if no + compaction occurred. + """ + if len(self._history) <= 3: + return None + + last_turn = self._history[-2:] # [user(tool_results_N), assistant(tool_use_N+1)] + response = await self.compact() + self._history.extend(last_turn) + return response + + @abstractmethod + async def send( + self, + content: str | list[ToolResultMessage], + allowed_tools: list[str] | None = None, + ) -> AgentResponse: ... diff --git a/ddev/src/ddev/ai/agent/build.py b/ddev/src/ddev/ai/agent/build.py new file mode 100644 index 0000000000000..28a56e620bbbc --- /dev/null +++ b/ddev/src/ddev/ai/agent/build.py @@ -0,0 +1,161 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from ddev.ai.agent.anthropic_client import AnthropicAgent +from ddev.ai.agent.base import BaseAgent +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.registry import ToolRegistry + +if TYPE_CHECKING: + from ddev.ai.phases.config import AgentConfig + +SubagentBuilder = Callable[ + [str, str, list[str]], # (system_prompt, owner_id, tool_names) + tuple[BaseAgent[Any], ToolRegistry], +] +AgentBuilder = Callable[ + [str, str, SubagentBuilder | None, Path | None], # system_prompt, owner_id, subagent_builder, log_dir + tuple[BaseAgent[Any], ToolRegistry], +] + + +def _resolve_client(agent_clients: dict[str, Any], provider: str) -> Any: + client = agent_clients.get(provider) + if client is None: + raise ValueError(f"No client provided for agent provider {provider!r}") + return client + + +def _build_agent_and_registry( + agent_config: AgentConfig, + agent_clients: dict[str, Any], + system_prompt: str, + owner_id: str, + tool_names: list[str], + file_registry: FileRegistry, + subagent_builder: SubagentBuilder | None = None, + log_dir: Path | None = None, +) -> tuple[BaseAgent[Any], ToolRegistry]: + tool_registry = ToolRegistry.from_names( + tool_names, + owner_id=owner_id, + file_registry=file_registry, + subagent_builder=subagent_builder, + log_dir=log_dir, + ) + + if agent_config.provider == "anthropic": + kwargs: dict[str, Any] = {} + if agent_config.model is not None: + kwargs["model"] = agent_config.model + if agent_config.max_tokens is not None: + kwargs["max_tokens"] = agent_config.max_tokens + agent: BaseAgent[Any] = AnthropicAgent( + client=_resolve_client(agent_clients, "anthropic"), + tools=tool_registry, + system_prompt=system_prompt, + name=owner_id, + **kwargs, + ) + return agent, tool_registry + + raise ValueError(f"Unknown agent provider: {agent_config.provider!r}") + + +def build_agent( + agent_config: AgentConfig, + agent_clients: dict[str, Any], + system_prompt: str, + owner_id: str, + file_registry: FileRegistry, + subagent_builder: SubagentBuilder | None = None, + log_dir: Path | None = None, +) -> tuple[BaseAgent[Any], ToolRegistry]: + """Construct a provider-specific BaseAgent and its ToolRegistry from an AgentConfig.""" + return _build_agent_and_registry( + agent_config=agent_config, + agent_clients=agent_clients, + system_prompt=system_prompt, + owner_id=owner_id, + tool_names=agent_config.tools, + file_registry=file_registry, + subagent_builder=subagent_builder, + log_dir=log_dir, + ) + + +def build_subagent( + parent_agent_config: AgentConfig, + agent_clients: dict[str, Any], + file_registry: FileRegistry, + system_prompt: str, + owner_id: str, + tool_names: list[str], +) -> tuple[BaseAgent[Any], ToolRegistry]: + """Build a subagent + ToolRegistry using the shared FileRegistry. + + Reuses the parent's provider/model/max_tokens. No subagent_builder or + log_dir is forwarded, so the subagent cannot recursively spawn subagents β€” + ToolRegistry.from_names will raise if spawn_subagent is in tool_names. + """ + return _build_agent_and_registry( + agent_config=parent_agent_config, + agent_clients=agent_clients, + system_prompt=system_prompt, + owner_id=owner_id, + tool_names=tool_names, + file_registry=file_registry, + ) + + +def make_agent_builder( + agent_config: AgentConfig, + agent_clients: dict[str, Any], + file_registry: FileRegistry, +) -> AgentBuilder: + """Return a closure that builds an agent+registry given system_prompt, owner_id, subagent_builder, log_dir.""" + + def builder( + system_prompt: str, + owner_id: str, + subagent_builder: SubagentBuilder | None, + log_dir: Path | None, + ) -> tuple[BaseAgent[Any], ToolRegistry]: + return build_agent( + agent_config=agent_config, + agent_clients=agent_clients, + system_prompt=system_prompt, + owner_id=owner_id, + file_registry=file_registry, + subagent_builder=subagent_builder, + log_dir=log_dir, + ) + + return builder + + +def make_subagent_builder( + parent_agent_config: AgentConfig, + agent_clients: dict[str, Any], + file_registry: FileRegistry, +) -> SubagentBuilder: + """Return a closure that builds a subagent+registry given (system_prompt, owner_id, tool_names).""" + + def builder(system_prompt: str, owner_id: str, tool_names: list[str]) -> tuple[BaseAgent[Any], ToolRegistry]: + return build_subagent( + parent_agent_config=parent_agent_config, + agent_clients=agent_clients, + file_registry=file_registry, + system_prompt=system_prompt, + owner_id=owner_id, + tool_names=tool_names, + ) + + return builder diff --git a/ddev/src/ddev/ai/agent/exceptions.py b/ddev/src/ddev/ai/agent/exceptions.py new file mode 100644 index 0000000000000..9f19519fa85eb --- /dev/null +++ b/ddev/src/ddev/ai/agent/exceptions.py @@ -0,0 +1,23 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + + +class AgentError(Exception): + """Base class for all errors raised by an agent.""" + + +class AgentConnectionError(AgentError): + """Network failure β€” the API was unreachable.""" + + +class AgentRateLimitError(AgentError): + """Rate limit hit β€” the request may be retried after a delay.""" + + +class AgentAPIError(AgentError): + """The API returned an error status code.""" + + def __init__(self, status_code: int, message: str) -> None: + super().__init__(message) + self.status_code = status_code diff --git a/ddev/src/ddev/ai/agent/types.py b/ddev/src/ddev/ai/agent/types.py new file mode 100644 index 0000000000000..84e380aea77d1 --- /dev/null +++ b/ddev/src/ddev/ai/agent/types.py @@ -0,0 +1,75 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +"""Wire types for the agent layer: enums, dataclasses, and response shapes.""" + +from dataclasses import dataclass +from enum import StrEnum +from typing import Any + +from ddev.ai.tools.core.types import ToolResult + + +class StopReason(StrEnum): + """Generic stop reasons for agent responses, independent of any provider.""" + + END_TURN = "end_turn" + TOOL_USE = "tool_use" + MAX_TOKENS = "max_tokens" + OTHER = "other" + + +@dataclass(frozen=True) +class ToolCall: + """A single tool invocation requested by the model.""" + + id: str + name: str + input: dict[str, Any] + + +@dataclass(frozen=True) +class ToolResultMessage: + """Wraps a tool result to be sent back to the agent, keyed by the originating tool call ID.""" + + tool_call_id: str # matches ToolCall.id + result: ToolResult + + +@dataclass(frozen=True) +class ContextUsage: + """Context window accounting for a single API call.""" + + window_size: int + used_tokens: int + + @property + def context_pct(self) -> float: + return self.used_tokens / self.window_size * 100 + + @property + def remaining_tokens(self) -> int: + return self.window_size - self.used_tokens + + +@dataclass(frozen=True) +class TokenUsage: + """Token accounting from a single API call.""" + + input_tokens: int # tokens sent to the model (system_prompt + history) + output_tokens: int # tokens the model generated + cache_read_input_tokens: int # tokens read from prompt cache + cache_creation_input_tokens: int # tokens written to prompt cache + context_usage: ContextUsage | None = None # None only for agents that don't provide context tracking + + +@dataclass(frozen=True) +class AgentResponse: + """The complete response from a single agent.send() call. + Adds useful metadata to the response of the agent.""" + + stop_reason: StopReason + text: str + tool_calls: list[ToolCall] + usage: TokenUsage diff --git a/ddev/src/ddev/ai/callbacks/__init__.py b/ddev/src/ddev/ai/callbacks/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/callbacks/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/callbacks/callbacks.py b/ddev/src/ddev/ai/callbacks/callbacks.py new file mode 100644 index 0000000000000..d0be2229a27c7 --- /dev/null +++ b/ddev/src/ddev/ai/callbacks/callbacks.py @@ -0,0 +1,218 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from typing import Any, Protocol + +from ddev.ai.agent.types import AgentResponse, ToolCall +from ddev.ai.react.types import ReActResult +from ddev.ai.tools.core.types import ToolResult + +# --------------------------------------------------------------------------- +# ReAct-layer protocols +# --------------------------------------------------------------------------- + + +class OnAgentResponseCallback(Protocol): + """Called after every agent.send() returns, including the first.""" + + async def __call__(self, response: AgentResponse, iteration: int) -> None: ... + + +class OnToolCallCallback(Protocol): + """Called once per (tool_call, result) pair after all tools in a batch execute.""" + + async def __call__(self, tool_call: ToolCall, result: ToolResult, iteration: int) -> None: ... + + +class OnCompleteCallback(Protocol): + """Called when the ReAct loop exits cleanly.""" + + async def __call__(self, result: ReActResult) -> None: ... + + +class OnErrorCallback(Protocol): + """Called when the ReAct loop aborts. The exception is always re-raised after this returns.""" + + async def __call__(self, error: BaseException) -> None: ... + + +class BeforeCompactCallback(Protocol): + """Called immediately before the agent's history is compacted.""" + + async def __call__(self) -> None: ... + + +class AfterCompactCallback(Protocol): + """Called immediately after the agent's history has been compacted.""" + + async def __call__(self) -> None: ... + + +class OnBeforeAgentSendCallback(Protocol): + """Called immediately before each agent.send() request is issued.""" + + async def __call__(self, iteration: int) -> None: ... + + +# --------------------------------------------------------------------------- +# Phase-layer protocols +# --------------------------------------------------------------------------- + + +class OnPhaseStartCallback(Protocol): + """Called once when a phase begins executing, before any agent interaction.""" + + async def __call__(self, phase_id: str) -> None: ... + + +class OnPhaseFinishCallback(Protocol): + """Called once when a phase completes successfully.""" + + async def __call__(self, phase_id: str) -> None: ... + + +# --------------------------------------------------------------------------- +# CallbackSet and Callbacks +# --------------------------------------------------------------------------- + + +class CallbackSet: + """Decorator-based registry for framework lifecycle event handlers. + + Group related handlers in a single instance for semantic cohesion, then + compose multiple instances via Callbacks(): + + Usage:: + logger = CallbackSet() + + @logger.on_complete + async def log_done(result: ReActResult) -> None: + print(f"Done in {result.iterations} iterations") + + callbacks = Callbacks([logger]) + """ + + def __init__(self) -> None: + self._on_agent_response: list[OnAgentResponseCallback] = [] + self._on_tool_call: list[OnToolCallCallback] = [] + self._on_complete: list[OnCompleteCallback] = [] + self._on_error: list[OnErrorCallback] = [] + self._before_compact: list[BeforeCompactCallback] = [] + self._after_compact: list[AfterCompactCallback] = [] + self._on_before_agent_send: list[OnBeforeAgentSendCallback] = [] + self._on_phase_start: list[OnPhaseStartCallback] = [] + self._on_phase_finish: list[OnPhaseFinishCallback] = [] + + async def _fire(self, handlers: list[Any], *args: Any) -> None: + for handler in handlers: + try: + await handler(*args) + except Exception: + pass + + def on_agent_response(self, func: OnAgentResponseCallback) -> OnAgentResponseCallback: + self._on_agent_response.append(func) + return func + + async def fire_agent_response(self, response: AgentResponse, iteration: int) -> None: + await self._fire(self._on_agent_response, response, iteration) + + def on_tool_call(self, func: OnToolCallCallback) -> OnToolCallCallback: + self._on_tool_call.append(func) + return func + + async def fire_tool_call(self, tool_call: ToolCall, result: ToolResult, iteration: int) -> None: + await self._fire(self._on_tool_call, tool_call, result, iteration) + + def on_complete(self, func: OnCompleteCallback) -> OnCompleteCallback: + self._on_complete.append(func) + return func + + async def fire_complete(self, result: ReActResult) -> None: + await self._fire(self._on_complete, result) + + def on_error(self, func: OnErrorCallback) -> OnErrorCallback: + self._on_error.append(func) + return func + + async def fire_error(self, error: BaseException) -> None: + await self._fire(self._on_error, error) + + def on_before_compact(self, func: BeforeCompactCallback) -> BeforeCompactCallback: + self._before_compact.append(func) + return func + + async def fire_before_compact(self) -> None: + await self._fire(self._before_compact) + + def on_after_compact(self, func: AfterCompactCallback) -> AfterCompactCallback: + self._after_compact.append(func) + return func + + async def fire_after_compact(self) -> None: + await self._fire(self._after_compact) + + def on_before_agent_send(self, func: OnBeforeAgentSendCallback) -> OnBeforeAgentSendCallback: + self._on_before_agent_send.append(func) + return func + + async def fire_before_agent_send(self, iteration: int) -> None: + await self._fire(self._on_before_agent_send, iteration) + + def on_phase_start(self, func: OnPhaseStartCallback) -> OnPhaseStartCallback: + self._on_phase_start.append(func) + return func + + async def fire_phase_start(self, phase_id: str) -> None: + await self._fire(self._on_phase_start, phase_id) + + def on_phase_finish(self, func: OnPhaseFinishCallback) -> OnPhaseFinishCallback: + self._on_phase_finish.append(func) + return func + + async def fire_phase_finish(self, phase_id: str) -> None: + await self._fire(self._on_phase_finish, phase_id) + + +class Callbacks: + """Container of CallbackSet instances. Dispatches each fire_* to all contained sets.""" + + def __init__(self, sets: list[CallbackSet] | None = None) -> None: + self._sets: list[CallbackSet] = sets or [] + + async def fire_agent_response(self, response: AgentResponse, iteration: int) -> None: + for s in self._sets: + await s.fire_agent_response(response, iteration) + + async def fire_tool_call(self, tool_call: ToolCall, result: ToolResult, iteration: int) -> None: + for s in self._sets: + await s.fire_tool_call(tool_call, result, iteration) + + async def fire_complete(self, result: ReActResult) -> None: + for s in self._sets: + await s.fire_complete(result) + + async def fire_error(self, error: BaseException) -> None: + for s in self._sets: + await s.fire_error(error) + + async def fire_before_compact(self) -> None: + for s in self._sets: + await s.fire_before_compact() + + async def fire_after_compact(self) -> None: + for s in self._sets: + await s.fire_after_compact() + + async def fire_before_agent_send(self, iteration: int) -> None: + for s in self._sets: + await s.fire_before_agent_send(iteration) + + async def fire_phase_start(self, phase_id: str) -> None: + for s in self._sets: + await s.fire_phase_start(phase_id) + + async def fire_phase_finish(self, phase_id: str) -> None: + for s in self._sets: + await s.fire_phase_finish(phase_id) diff --git a/ddev/src/ddev/ai/phases/__init__.py b/ddev/src/ddev/ai/phases/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/phases/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/phases/agentic_phase.py b/ddev/src/ddev/ai/phases/agentic_phase.py new file mode 100644 index 0000000000000..948bcfb3e7728 --- /dev/null +++ b/ddev/src/ddev/ai/phases/agentic_phase.py @@ -0,0 +1,214 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import logging +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from ddev.ai.agent.base import BaseAgent +from ddev.ai.agent.build import AgentBuilder, SubagentBuilder, make_agent_builder, make_subagent_builder +from ddev.ai.callbacks.callbacks import Callbacks +from ddev.ai.phases.base import Phase, PhaseOutcome +from ddev.ai.phases.checkpoint import CheckpointManager +from ddev.ai.phases.config import AgentConfig, CheckpointConfig, FlowConfigError, PhaseConfig, TaskConfig +from ddev.ai.phases.template import render_inline, render_prompt +from ddev.ai.react.process import ReActProcess +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.registry import TOOL_MANIFEST + + +def render_task_prompt( + task: TaskConfig, + config_dir: Path, + context: dict[str, Any], + resolver: Callable[[str], str] | None = None, +) -> str: + """Render a task prompt β€” from file if prompt_path is set, inline otherwise.""" + if task.prompt_path is not None: + return render_prompt(config_dir / task.prompt_path, context, resolver) + if task.prompt is None: + raise FlowConfigError("TaskConfig must set either 'prompt' or 'prompt_path'") + return render_inline(task.prompt, context, resolver) + + +def render_memory_prompt( + checkpoint: CheckpointConfig, + config_dir: Path, + context: dict[str, Any], +) -> str: + """Render a checkpoint memory prompt β€” from file if memory_prompt_path is set, inline otherwise.""" + if checkpoint.memory_prompt_path is not None: + return render_prompt(config_dir / checkpoint.memory_prompt_path, context) + if checkpoint.memory_prompt is None: + raise FlowConfigError("CheckpointConfig must set either 'memory_prompt' or 'memory_prompt_path'") + return render_inline(checkpoint.memory_prompt, context) + + +class AgenticPhase(Phase): + """Phase that owns an LLM agent and drives one or more ReAct loops.""" + + def __init__( + self, + phase_id: str, + dependencies: list[str], + config: PhaseConfig, + agent_builder: AgentBuilder, + checkpoint_manager: CheckpointManager, + runtime_variables: dict[str, str], + flow_variables: dict[str, str], + config_dir: Path, + file_registry: FileRegistry, + subagent_builder: SubagentBuilder | None = None, + callbacks: Callbacks | None = None, + logger: logging.Logger | None = None, + ) -> None: + super().__init__( + phase_id=phase_id, + dependencies=dependencies, + config=config, + checkpoint_manager=checkpoint_manager, + runtime_variables=runtime_variables, + flow_variables=flow_variables, + config_dir=config_dir, + file_registry=file_registry, + callbacks=callbacks, + logger=logger, + ) + self._agent_builder = agent_builder + self._subagent_builder = subagent_builder + self._subagent_log_dir = ( + checkpoint_manager.root / "subagents" / phase_id if subagent_builder is not None else None + ) + + @classmethod + def validate_config( + cls, + phase_id: str, + config: PhaseConfig, + agents: dict[str, AgentConfig], + ) -> None: + if config.agent is None: + raise FlowConfigError(f"Phase {phase_id!r} (AgenticPhase) requires 'agent'") + if config.agent not in agents: + raise FlowConfigError(f"Phase {phase_id!r} references unknown agent: {config.agent!r}") + if not config.tasks: + raise FlowConfigError(f"Phase {phase_id!r} (AgenticPhase) must have at least one task") + + @classmethod + def extra_init_kwargs( # type: ignore[override] + cls, + *, + phase_id: str, + phase_config: PhaseConfig, + agents: dict[str, AgentConfig], + agent_clients: dict[str, Any], + file_registry: FileRegistry, + **_: Any, + ) -> dict[str, Any]: + if phase_config.agent is None: + raise FlowConfigError(f"Phase {phase_id!r} (AgenticPhase) requires 'agent'") + agent_config = agents[phase_config.agent] + + subagent_builder = None + requires_subagent_builder = any( + spec.requires_subagent_builder + for name in agent_config.tools + if (spec := TOOL_MANIFEST.get(name)) is not None + ) + if requires_subagent_builder: + subagent_builder = make_subagent_builder( + parent_agent_config=agent_config, + agent_clients=agent_clients, + file_registry=file_registry, + ) + + return { + "agent_builder": make_agent_builder( + agent_config=agent_config, + agent_clients=agent_clients, + file_registry=file_registry, + ), + "subagent_builder": subagent_builder, + } + + def before_react(self) -> None: + """Called once before agent/tools are created. Override for phase-specific setup.""" + + def after_react(self) -> None: + """Called once after all tasks complete. Override for phase-specific teardown.""" + + async def run_tasks( + self, + process: ReActProcess, + context: dict[str, Any], + ) -> tuple[int, int]: + """Run the task loop. Returns (total_input_tokens, total_output_tokens). + + Override to customize task execution β€” e.g. add retries, change ordering, etc. + Default implementation iterates through config.tasks sequentially. + """ + total_input = total_output = 0 + last_result = None + for task in self._config.tasks: + if last_result is not None and last_result.context_usage is not None: + if last_result.context_usage.context_pct >= self._config.context_compact_threshold_pct: + compact_in, compact_out = await process.compact() + total_input += compact_in + total_output += compact_out + prompt = render_task_prompt(task, self._config_dir, context, self._resolver) + last_result = await process.start(prompt) + total_input += last_result.total_input_tokens + total_output += last_result.total_output_tokens + return total_input, total_output + + def _build_agent_and_process(self, context: dict[str, Any]) -> tuple[BaseAgent[Any], ReActProcess]: + """Build the agent and ReAct process used to drive task execution.""" + system_prompt = render_prompt( + self._config_dir / "prompts" / f"{self._config.agent}.md", + context, + self._resolver, + ) + agent, tool_registry = self._agent_builder( + system_prompt, + self._phase_id, + self._subagent_builder, + self._subagent_log_dir, + ) + process = ReActProcess( + agent=agent, + tool_registry=tool_registry, + callbacks=self._callbacks, + ) + return agent, process + + async def _run_memory_step( + self, + agent: BaseAgent[Any], + context: dict[str, Any], + ) -> tuple[str, int, int]: + """Run the final summary turn. Returns (memory_text, input_tokens, output_tokens).""" + user_additions = None + if self._config.checkpoint is not None: + user_additions = render_memory_prompt(self._config.checkpoint, self._config_dir, context) + memory_prompt = self._checkpoint_manager.build_memory_prompt(user_additions) + + await self._callbacks.fire_before_agent_send(1) + response = await agent.send(memory_prompt, allowed_tools=[]) + await self._callbacks.fire_agent_response(response, 1) + return response.text, response.usage.input_tokens, response.usage.output_tokens + + async def execute(self, context: dict[str, Any]) -> PhaseOutcome: + self.before_react() + agent, process = self._build_agent_and_process(context) + total_input, total_output = await self.run_tasks(process, context) + self.after_react() + + memory_text, mem_in, mem_out = await self._run_memory_step(agent, context) + + return PhaseOutcome( + memory_text=memory_text, + total_input_tokens=total_input + mem_in, + total_output_tokens=total_output + mem_out, + ) diff --git a/ddev/src/ddev/ai/phases/base.py b/ddev/src/ddev/ai/phases/base.py new file mode 100644 index 0000000000000..227b92560a60f --- /dev/null +++ b/ddev/src/ddev/ai/phases/base.py @@ -0,0 +1,191 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import logging +from abc import abstractmethod +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from ddev.ai.callbacks.callbacks import Callbacks +from ddev.ai.phases.checkpoint import CheckpointManager +from ddev.ai.phases.config import AgentConfig, PhaseConfig +from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.event_bus.exceptions import MessageProcessingError, ProcessorHookError +from ddev.event_bus.orchestrator import AsyncProcessor, BaseMessage + + +@dataclass +class PhaseOutcome: + memory_text: str + total_input_tokens: int = 0 + total_output_tokens: int = 0 + extra_checkpoint: dict[str, Any] = field(default_factory=dict) + + +class PhaseRegistry: + def __init__(self) -> None: + self._registry: dict[str, type["Phase"]] = {} + + def register(self, name: str, phase_cls: type["Phase"]) -> None: + self._registry[name] = phase_cls + + def known_names(self) -> list[str]: + return sorted(self._registry) + + def get(self, name: str) -> type["Phase"]: + if name not in self._registry: + raise ValueError(f"Unknown phase type: {name!r}. Known: {self.known_names()}") + return self._registry[name] + + +class Phase(AsyncProcessor[PhaseTrigger]): + """Lifecycle base for all phases. + + process_message() implements the immutable pipeline skeleton. + Subclasses implement execute() to provide phase-specific logic. + Registered in PhaseRegistry by _discover_and_register_phases() at startup. + """ + + def __init__( + self, + phase_id: str, + dependencies: list[str], + config: PhaseConfig, + checkpoint_manager: CheckpointManager, + runtime_variables: dict[str, str], + flow_variables: dict[str, str], + config_dir: Path, + file_registry: FileRegistry, + callbacks: Callbacks | None = None, + logger: logging.Logger | None = None, + ) -> None: + super().__init__(name=phase_id) + self._phase_id = phase_id + self._dependencies = set(dependencies) + self._remaining_dependencies = set(dependencies) + self._config = config + self._checkpoint_manager = checkpoint_manager + self._runtime_variables = runtime_variables + self._flow_variables = flow_variables + self._config_dir = config_dir + self._callbacks: Callbacks = callbacks or Callbacks() + self._file_registry = file_registry + self._logger = logger or logging.getLogger(__name__) + self._started_at: datetime | None = None + self._resolver: Callable[[str], str] | None = None + self._executed = False + + def should_process_message(self, message: BaseMessage) -> bool: + if isinstance(message, PhaseTrigger): + if message.phase_id is None: + # Initial trigger β€” only root phases (no declared dependencies) respond + if self._dependencies: + return False + else: + # Phase-completion trigger β€” check dependency tracking + if message.phase_id not in self._dependencies: + return False + self._remaining_dependencies.discard(message.phase_id) + if self._remaining_dependencies: + return False + if self._executed: + return False + self._executed = True + return True + + @classmethod + def validate_config( + cls, + phase_id: str, + config: PhaseConfig, + agents: dict[str, AgentConfig], + ) -> None: + """Override to enforce per-subclass config invariants. Raise FlowConfigError on mismatch.""" + return None + + @classmethod + def extra_init_kwargs(cls, **kwargs: Any) -> dict[str, Any]: + """Override to inject subclass-specific kwargs into __init__ at construction time. + + The orchestrator passes every framework-level dep (phase_id, phase_config, agents, + agent_clients, file_registry, checkpoint_manager, ...) as keyword arguments. + Subclasses pick the ones they need by declaring them explicitly and accept the + rest via **kwargs. + """ + return {} + + @abstractmethod + async def execute(self, context: dict[str, Any]) -> PhaseOutcome: ... + + async def process_message(self, message: PhaseTrigger) -> None: + """Immutable pipeline skeleton. Not intended to be overridden β€” implement execute() instead.""" + self._started_at = datetime.now(UTC) + await self._callbacks.fire_phase_start(self._phase_id) + + context: dict[str, Any] = { + **self._flow_variables, + **self._runtime_variables, + "phase_name": self._phase_id, + "checkpoints": self._checkpoint_manager.read(), + } + self._resolver = self._checkpoint_manager.resolve_template_variable + + outcome = await self.execute(context) + + checkpoint_payload: dict[str, Any] = { + "status": "success", + "started_at": self._started_at.isoformat(), + "finished_at": datetime.now(UTC).isoformat(), + "tokens": { + "total_input": outcome.total_input_tokens, + "total_output": outcome.total_output_tokens, + }, + "memory_path": str(self._checkpoint_manager.memory_path(self._phase_id)), + } + reserved = set(checkpoint_payload) & set(outcome.extra_checkpoint) + if reserved: + raise ValueError( + f"Phase {self._phase_id!r}: extra_checkpoint cannot override reserved keys: {sorted(reserved)}" + ) + checkpoint_payload.update(outcome.extra_checkpoint) + + self._checkpoint_manager.write_memory(self._phase_id, outcome.memory_text) + self._checkpoint_manager.write_phase_checkpoint(self._phase_id, checkpoint_payload) + await self._callbacks.fire_phase_finish(self._phase_id) + + async def on_success(self, message: PhaseTrigger) -> None: + """Emit PhaseTrigger to unblock dependent phases.""" + self.submit_message( + PhaseTrigger( + id=f"{self._phase_id}_finished", + phase_id=self._phase_id, + ) + ) + + async def on_error(self, error: MessageProcessingError | ProcessorHookError) -> None: + """Write failed checkpoint and emit PhaseFailedMessage.""" + try: + self._checkpoint_manager.write_phase_checkpoint( + self._phase_id, + { + "status": "failed", + "started_at": self._started_at.isoformat() if self._started_at else None, + "finished_at": datetime.now(UTC).isoformat(), + "error": str(error.original_exception), + }, + ) + except Exception: + self._logger.exception("Failed to write failure checkpoint for phase %s", self._phase_id) + finally: + self.submit_message( + PhaseFailedMessage( + id=f"{self._phase_id}_failed", + phase_id=self._phase_id, + error=str(error.original_exception), + ) + ) diff --git a/ddev/src/ddev/ai/phases/checkpoint.py b/ddev/src/ddev/ai/phases/checkpoint.py new file mode 100644 index 0000000000000..3f32edb158a5b --- /dev/null +++ b/ddev/src/ddev/ai/phases/checkpoint.py @@ -0,0 +1,68 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from pathlib import Path +from typing import Any + +import yaml + + +class CheckpointReadError(Exception): + """Raised when checkpoints.yaml exists but cannot be read or parsed.""" + + +class CheckpointManager: + """Manages checkpoints.yaml and per-phase memory files for the full pipeline.""" + + def __init__(self, path: Path) -> None: + self._path = path + + @property + def root(self) -> Path: + """Directory that holds checkpoints.yaml, per-phase memory files, and any side artifacts.""" + return self._path.parent + + def _ensure_dir(self) -> None: + self.root.mkdir(parents=True, exist_ok=True) + + def read(self) -> dict[str, Any]: + """Return full checkpoint data, keyed by phase_id. Empty dict if file absent.""" + if not self._path.exists(): + return {} + try: + return yaml.safe_load(self._path.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError) as e: + raise CheckpointReadError(f"Failed to load checkpoints from {self._path}: {e}") from e + + def write_phase_checkpoint(self, phase_id: str, data: dict[str, Any]) -> None: + """Write or overwrite one phase's section in checkpoints.yaml.""" + checkpoints = self.read() + checkpoints[phase_id] = data + self._ensure_dir() + self._path.write_text(yaml.dump(checkpoints, default_flow_style=False), encoding="utf-8") + + def build_memory_prompt(self, user_additions: str | None) -> str: + """Build the memory prompt to send to the agent at the end of a phase.""" + base_prompt = "Write a brief summary of what you accomplished in this phase." + return f"{user_additions}\n\n{base_prompt}" if user_additions else base_prompt + + def memory_path(self, phase_id: str) -> Path: + """Return the resolved path to a phase's memory file.""" + return (self.root / f"{phase_id}_memory.md").resolve() + + def write_memory(self, phase_id: str, text: str) -> None: + """Write agent-authored text to this phase's memory file.""" + self._ensure_dir() + self.memory_path(phase_id).write_text(text, encoding="utf-8") + + def memory_content(self, phase_id: str) -> str: + """Return the contents of a phase's memory file, or a NOT FOUND placeholder.""" + path = self.memory_path(phase_id) + return path.read_text(encoding="utf-8") if path.exists() else f"" + + def resolve_template_variable(self, key: str) -> str: + """Resolve a template variable. ``_memory`` keys read the matching memory file.""" + if key.endswith("_memory"): + return self.memory_content(key.removesuffix("_memory")) + return f"" diff --git a/ddev/src/ddev/ai/phases/config.py b/ddev/src/ddev/ai/phases/config.py new file mode 100644 index 0000000000000..5c2564e19066b --- /dev/null +++ b/ddev/src/ddev/ai/phases/config.py @@ -0,0 +1,184 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from __future__ import annotations + +from pathlib import Path + +import yaml +from pydantic import BaseModel, ConfigDict, ValidationError, field_validator, model_validator + +from ddev.ai.tools.registry import ToolRegistry + + +class FlowConfigError(Exception): + """Wraps Pydantic ValidationError or YAML errors with a user-friendly message.""" + + +def _detect_cycles( + dependency_map: dict[str, list[str]], + limit: int = 50, +) -> tuple[list[list[str]], bool]: + """Return every simple cycle in the dependency graph, each as an ordered list of phase IDs.""" + # Enumerate every simple cycle exactly once: from each node, DFS only through + # higher-ranked nodes, so each cycle is reported only when started from its + # lowest-ranked member. (Tiernan-style enumeration with rank canonicalization.) + rank = {n: i for i, n in enumerate(dependency_map)} + cycles: list[list[str]] = [] + + class _LimitReached(Exception): + """Raised when the cycle limit is reached.""" + + pass + + def dfs(start: str, current: str, path: list[str], on_path: set[str]): + for dep in dependency_map.get(current, []): + if dep == start: + cycles.append(path + [start]) + if len(cycles) >= limit: + raise _LimitReached + elif dep in rank and rank[dep] > rank[start] and dep not in on_path: + on_path.add(dep) + dfs(start, dep, path + [dep], on_path) + on_path.discard(dep) + + try: + for start in dependency_map: + dfs(start, start, [start], {start}) + except _LimitReached: + return cycles, True + return cycles, False + + +class TaskConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + name: str + prompt_path: Path | None = None + prompt: str | None = None + + @model_validator(mode="after") + def exactly_one_source(self) -> TaskConfig: + if (self.prompt_path is None) == (self.prompt is None): + raise ValueError("Exactly one of 'prompt_path' or 'prompt' must be set") + return self + + +class CheckpointConfig(BaseModel): + """Optional extra instructions for the memory step. If omitted, only a summary is written.""" + + model_config = ConfigDict(extra="forbid") + memory_prompt: str | None = None + memory_prompt_path: Path | None = None + + @model_validator(mode="after") + def exactly_one_source(self) -> CheckpointConfig: + if (self.memory_prompt is None) == (self.memory_prompt_path is None): + raise ValueError("Exactly one of 'memory_prompt' or 'memory_prompt_path' must be set") + return self + + +class AgentConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + provider: str = "anthropic" + model: str | None = None + max_tokens: int | None = None + tools: list[str] = [] + + @field_validator("tools", mode="after") + @classmethod + def tools_must_be_known(cls, tools: list[str]) -> list[str]: + unknown = set(tools) - set(ToolRegistry.available_tool_names()) + if unknown: + raise ValueError(f"Unknown tool names: {sorted(unknown)}") + return tools + + +class PhaseConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + type: str = "AgenticPhase" + agent: str | None = None + tasks: list[TaskConfig] = [] + context_compact_threshold_pct: int = 80 + checkpoint: CheckpointConfig | None = None + + +class FlowEntry(BaseModel): + model_config = ConfigDict(extra="forbid") + phase: str + dependencies: list[str] = [] + + +class FlowConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + variables: dict[str, str] = {} + agents: dict[str, AgentConfig] + phases: dict[str, PhaseConfig] + flow: list[FlowEntry] + + @model_validator(mode="after") + def cross_references(self) -> FlowConfig: + """Validate all cross-references between agents, phases, and dependencies.""" + scheduled = {entry.phase for entry in self.flow} + seen: set[str] = set() + for entry in self.flow: + if entry.phase in seen: + raise ValueError(f"Duplicate phase in flow: {entry.phase!r}") + seen.add(entry.phase) + if entry.phase not in self.phases: + raise ValueError(f"Flow references unknown phase: {entry.phase!r}") + for dep in entry.dependencies: + if dep not in self.phases: + raise ValueError(f"Phase {entry.phase!r} depends on unknown phase: {dep!r}") + if dep not in scheduled: + raise ValueError(f"Phase {entry.phase!r} depends on {dep!r} which is not scheduled in flow") + + for phase_id, phase in self.phases.items(): + if phase.agent is not None and phase.agent not in self.agents: + raise ValueError(f"Phase {phase_id!r} references unknown agent: {phase.agent!r}") + + dependency_map = {entry.phase: entry.dependencies for entry in self.flow} + cycles, truncated = _detect_cycles(dependency_map) + if cycles: + formatted = "\n ".join(" β†’ ".join(c) for c in cycles) + suffix = f"\n (showing first {len(cycles)}; more cycles exist)" if truncated else "" + raise ValueError(f"Cycle(s) detected in flow:\n {formatted}{suffix}") + + return self + + @classmethod + def from_yaml(cls, path: Path, config_dir: Path) -> FlowConfig: + """Load, parse, and validate flow.yaml. Raises FlowConfigError on any problem.""" + try: + raw = yaml.safe_load(path.read_text()) + except (OSError, yaml.YAMLError) as e: + raise FlowConfigError(f"Failed to load {path}: {e}") from e + + try: + config = cls.model_validate(raw) + except ValidationError as e: + raise FlowConfigError(f"Invalid flow config:\n{e}") from e + + config._validate_files(config_dir) + return config + + def _validate_files(self, config_dir: Path) -> None: + """Check all referenced files exist.""" + for agent_name in self.agents: + system_prompt = config_dir / "prompts" / f"{agent_name}.md" + if not system_prompt.exists(): + raise FlowConfigError(f"System prompt not found for agent {agent_name!r}: {system_prompt}") + + for phase_id, phase in self.phases.items(): + for i, task in enumerate(phase.tasks): + if task.prompt_path is not None: + resolved = config_dir / task.prompt_path + if not resolved.exists(): + raise FlowConfigError( + f"Phase {phase_id!r} task {i} ({task.name!r}): prompt_path not found: {resolved}" + ) + + if phase.checkpoint is not None and phase.checkpoint.memory_prompt_path is not None: + resolved = config_dir / phase.checkpoint.memory_prompt_path + if not resolved.exists(): + raise FlowConfigError(f"Phase {phase_id!r} checkpoint memory_prompt_path not found: {resolved}") diff --git a/ddev/src/ddev/ai/phases/messages.py b/ddev/src/ddev/ai/phases/messages.py new file mode 100644 index 0000000000000..1b67193a9a9d3 --- /dev/null +++ b/ddev/src/ddev/ai/phases/messages.py @@ -0,0 +1,18 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from dataclasses import dataclass + +from ddev.event_bus.orchestrator import BaseMessage + + +@dataclass +class PhaseTrigger(BaseMessage): + phase_id: str | None # None = initial pipeline start; str = the phase that just finished + + +@dataclass +class PhaseFailedMessage(BaseMessage): + phase_id: str + error: str diff --git a/ddev/src/ddev/ai/phases/orchestrator.py b/ddev/src/ddev/ai/phases/orchestrator.py new file mode 100644 index 0000000000000..914d6aadf557e --- /dev/null +++ b/ddev/src/ddev/ai/phases/orchestrator.py @@ -0,0 +1,144 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import importlib +import inspect +import logging +from pathlib import Path +from typing import Any + +from ddev.ai.callbacks.callbacks import Callbacks +from ddev.ai.phases.base import Phase, PhaseRegistry +from ddev.ai.phases.checkpoint import CheckpointManager +from ddev.ai.phases.config import FlowConfig, FlowConfigError +from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.event_bus.exceptions import FatalProcessingError +from ddev.event_bus.orchestrator import BaseMessage, EventBusOrchestrator + + +def _discover_and_register_phases( + registry: PhaseRegistry, + phases_dir: Path | None = None, + import_prefix: str = "ddev.ai.phases", +) -> None: + """Import all non-private modules in phases_dir and register Phase subclasses.""" + if phases_dir is None: + phases_dir = Path(__file__).parent + for py_file in phases_dir.glob("*.py"): + if py_file.stem.startswith("_"): + continue + try: + module = importlib.import_module(f"{import_prefix}.{py_file.stem}") + except Exception as e: + raise FlowConfigError(f"Failed to import phase module '{py_file.stem}': {e}") from e + for _, obj in inspect.getmembers(module, inspect.isclass): + if issubclass(obj, Phase) and not inspect.isabstract(obj) and obj.__module__ == module.__name__: + registry.register(obj.__name__, obj) + + +class PhaseOrchestrator(EventBusOrchestrator): + def __init__( + self, + flow_yaml_path: Path, + checkpoint_path: Path, + runtime_variables: dict[str, str], + agent_clients: dict[str, Any], + file_access_policy: FileAccessPolicy, + callbacks: Callbacks | None = None, + grace_period: float = 10, + logger: logging.Logger | None = None, + ) -> None: + """Initialize the orchestrator. + + ``agent_clients`` maps provider name (e.g. ``"anthropic"``) to a constructed + provider client. ``build_agent`` looks up the right one based on each + ``AgentConfig.provider``. + + ``file_access_policy`` must have ``write_root`` set to the integration + output directory so that agent writes are confined to that path. + """ + super().__init__(logger=logger or logging.getLogger(__name__), grace_period=grace_period) + self._flow_yaml_path = flow_yaml_path + self._checkpoint_path = checkpoint_path + self._runtime_variables = runtime_variables + self._agent_clients = agent_clients + self._callbacks: Callbacks = callbacks or Callbacks() + self._phase_registry = PhaseRegistry() + self._failed_phase: str | None = None + self._failed_error: str | None = None + self._file_registry = FileRegistry(policy=file_access_policy) + + async def on_initialize(self) -> None: + """Discover custom phases, parse flow.yaml, construct phases, submit PhaseTrigger.""" + config_dir = self._flow_yaml_path.parent + + _discover_and_register_phases(self._phase_registry) + + config = FlowConfig.from_yaml(self._flow_yaml_path, config_dir) + + flow_phase_ids = {entry.phase for entry in config.flow} + for phase_id, phase_config in config.phases.items(): + if phase_id not in flow_phase_ids: + self._logger.warning("Phase %r is defined but not referenced in flow β€” it will not run", phase_id) + continue + try: + phase_cls = self._phase_registry.get(phase_config.type) + except ValueError as e: + raise FlowConfigError(str(e)) from e + phase_cls.validate_config(phase_id, phase_config, config.agents) + + checkpoint_manager = CheckpointManager(self._checkpoint_path) + + dependency_map: dict[str, list[str]] = {entry.phase: entry.dependencies for entry in config.flow} + + for entry in config.flow: + phase_id = entry.phase + phase_config = config.phases[phase_id] + dependencies = dependency_map[phase_id] + + phase_cls = self._phase_registry.get(phase_config.type) + phase_kwargs: dict[str, Any] = { + "phase_id": phase_id, + "dependencies": dependencies, + "config": phase_config, + "checkpoint_manager": checkpoint_manager, + "runtime_variables": self._runtime_variables, + "flow_variables": config.variables, + "config_dir": config_dir, + "file_registry": self._file_registry, + "callbacks": self._callbacks, + "logger": self._logger, + } + phase_kwargs.update( + phase_cls.extra_init_kwargs( + phase_id=phase_id, + phase_config=phase_config, + agents=config.agents, + agent_clients=self._agent_clients, + file_registry=self._file_registry, + ) + ) + + phase = phase_cls(**phase_kwargs) + + self.register_processor(phase, [PhaseTrigger]) + + self.submit_message(PhaseTrigger(id="start", phase_id=None)) + + async def on_message_received(self, message: BaseMessage) -> None: + """Stop the entire pipeline immediately when any phase fails.""" + if isinstance(message, PhaseFailedMessage): + self._failed_phase = message.phase_id + self._failed_error = message.error + raise FatalProcessingError(f"Phase '{message.phase_id}' failed: {message.error}") + + async def on_finalize(self, exception: Exception | None) -> None: + if exception is not None and self._failed_phase is not None: + self._logger.error( + "Pipeline aborted: phase '%s' failed: %s", + self._failed_phase, + self._failed_error or "", + ) diff --git a/ddev/src/ddev/ai/phases/template.py b/ddev/src/ddev/ai/phases/template.py new file mode 100644 index 0000000000000..b7143d8afca4f --- /dev/null +++ b/ddev/src/ddev/ai/phases/template.py @@ -0,0 +1,39 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from collections.abc import Callable, Iterator, Mapping +from pathlib import Path +from string import Template +from typing import Any + + +class _SafeMapping(Mapping[str, str]): + """Returns the resolver result or an UNDEFINED placeholder for missing keys instead of raising.""" + + def __init__(self, context: dict[str, Any], resolver: Callable[[str], str] | None = None) -> None: + self._context = context + self._resolver = resolver + + def __getitem__(self, key: str) -> str: + if key in self._context: + return str(self._context[key]) + if self._resolver is not None: + return self._resolver(key) + return f"" + + def __iter__(self) -> Iterator[str]: + return iter(self._context) + + def __len__(self) -> int: + return len(self._context) + + +def render_prompt(template_path: Path, context: dict[str, Any], resolver: Callable[[str], str] | None = None) -> str: + """Render a template file with the given context.""" + return Template(template_path.read_text()).substitute(_SafeMapping(context, resolver)) + + +def render_inline(prompt: str, context: dict[str, Any], resolver: Callable[[str], str] | None = None) -> str: + """Render an inline prompt string with the given context.""" + return Template(prompt).substitute(_SafeMapping(context, resolver)) diff --git a/ddev/src/ddev/ai/react/__init__.py b/ddev/src/ddev/ai/react/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/react/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/react/process.py b/ddev/src/ddev/ai/react/process.py new file mode 100644 index 0000000000000..c925af8e31e6a --- /dev/null +++ b/ddev/src/ddev/ai/react/process.py @@ -0,0 +1,154 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import asyncio +from typing import Any + +from ddev.ai.agent.base import BaseAgent +from ddev.ai.agent.exceptions import AgentError +from ddev.ai.agent.types import AgentResponse, StopReason, ToolResultMessage +from ddev.ai.callbacks.callbacks import Callbacks +from ddev.ai.react.types import ReActResult +from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.registry import ToolRegistry + + +class ReActProcess: + """ + Manages the ReAct (Reason + Act) loop for a single task. + + Sends a prompt to an agent, executes any tool calls in parallel, + feeds results back, and repeats until the agent stops requesting tools. + """ + + def __init__( + self, + agent: BaseAgent[Any], + tool_registry: ToolRegistry, + callbacks: Callbacks | None = None, + compact_threshold_pct: float | None = 75.0, + ) -> None: + """ + Args: + agent: A BaseAgent subclass (e.g. AnthropicAgent). + tool_registry: Registry of tools available in this loop. + callbacks: Optional Callbacks instance to observe loop events. + compact_threshold_pct: Context usage percentage at which the loop auto-compacts. + None disables auto-compaction entirely. + """ + self._agent = agent + self._tool_registry = tool_registry + self._callbacks: Callbacks = callbacks or Callbacks() + self._compact_threshold_pct = compact_threshold_pct + + def reset(self) -> None: + """Clear the agent's conversation history.""" + self._agent.reset() + + async def compact(self, response: AgentResponse | None = None) -> tuple[int, int]: + """Compact the agent's conversation history unconditionally. + + Args: + response: The last agent response. If None, compaction is unconditional. + + Returns (input_tokens, output_tokens) from the compaction API call. + Returns (0, 0) if history was already compact and no API call was made. + """ + await self._callbacks.fire_before_compact() + + compact_response = None + if response is None or response.stop_reason != StopReason.TOOL_USE: + compact_response = await self._agent.compact() + else: + compact_response = await self._agent.compact_preserving_last_turn() + + await self._callbacks.fire_after_compact() + if compact_response is None: + return 0, 0 + return compact_response.usage.input_tokens, compact_response.usage.output_tokens + + def _is_compact_needed(self, response: AgentResponse) -> bool: + if self._compact_threshold_pct is None: + return False + ctx = response.usage.context_usage + if ctx is None or ctx.context_pct < self._compact_threshold_pct: + return False + return True + + async def start(self, prompt: str, allowed_tools: list[str] | None = None) -> ReActResult: + """ + Run the ReAct loop for a single task. + + Args: + prompt: The initial user prompt to send to the agent. + allowed_tools: Optional subset of tools the agent may call in this run. None means all. + + Returns: + A ReActResult summarising the final response, token counts, and iteration count. + + Raises: + Every exception is forwarded after notifying callbacks. + """ + try: + await self._callbacks.fire_before_agent_send(1) + + response = await self._agent.send(prompt, allowed_tools) + iterations = 1 + total_input = response.usage.input_tokens + total_output = response.usage.output_tokens + + await self._callbacks.fire_agent_response(response, iterations) + + # No iteration cap β€” this is an interactive CLI tool; the user can Ctrl+C to stop. + while response.stop_reason == StopReason.TOOL_USE: + if not response.tool_calls: + raise AgentError("Agent returned stop_reason=TOOL_USE with no tool calls") + + raw_results = await asyncio.gather( + *[self._tool_registry.run(tc.name, tc.input) for tc in response.tool_calls], + return_exceptions=True, + ) + tool_results: list[ToolResult] = [ + r if isinstance(r, ToolResult) else ToolResult(success=False, error=f"{type(r).__name__}: {r}") + for r in raw_results + ] + total_input += sum(result.total_input_tokens for result in tool_results) + total_output += sum(result.total_output_tokens for result in tool_results) + + tool_call_results = list(zip(response.tool_calls, tool_results, strict=True)) + + for tc, result in tool_call_results: + await self._callbacks.fire_tool_call(tc, result, iterations) + + messages = [ToolResultMessage(tool_call_id=tc.id, result=result) for tc, result in tool_call_results] + + await self._callbacks.fire_before_agent_send(iterations + 1) + + response = await self._agent.send(messages, allowed_tools) + iterations += 1 + total_input += response.usage.input_tokens + total_output += response.usage.output_tokens + + await self._callbacks.fire_agent_response(response, iterations) + + if self._is_compact_needed(response): + compact_in, compact_out = await self.compact(response) + total_input += compact_in + total_output += compact_out + + react_result = ReActResult( + final_response=response, + iterations=iterations, + total_input_tokens=total_input, + total_output_tokens=total_output, + context_usage=response.usage.context_usage, + ) + + await self._callbacks.fire_complete(react_result) + + return react_result + + except BaseException as e: + await self._callbacks.fire_error(e) + raise diff --git a/ddev/src/ddev/ai/react/types.py b/ddev/src/ddev/ai/react/types.py new file mode 100644 index 0000000000000..4a6b2345433dc --- /dev/null +++ b/ddev/src/ddev/ai/react/types.py @@ -0,0 +1,18 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from dataclasses import dataclass + +from ddev.ai.agent.types import AgentResponse, ContextUsage + + +@dataclass(frozen=True) +class ReActResult: + """Immutable summary of a completed ReAct loop run.""" + + final_response: AgentResponse + iterations: int + total_input_tokens: int # sum across all iterations + total_output_tokens: int # sum across all iterations + context_usage: ContextUsage | None # promoted from final_response.usage.context_usage diff --git a/ddev/src/ddev/ai/tools/__init__.py b/ddev/src/ddev/ai/tools/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/tools/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/tools/agents/__init__.py b/ddev/src/ddev/ai/tools/agents/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/tools/agents/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/tools/agents/agent_logger.py b/ddev/src/ddev/ai/tools/agents/agent_logger.py new file mode 100644 index 0000000000000..14a20a43d567a --- /dev/null +++ b/ddev/src/ddev/ai/tools/agents/agent_logger.py @@ -0,0 +1,105 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import json +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from ddev.ai.agent.types import AgentResponse, ToolCall +from ddev.ai.callbacks.callbacks import Callbacks, CallbackSet +from ddev.ai.tools.core.types import ToolResult + + +class AgentLogger: + """Append-only JSONL writer for ReAct events plus subagent start/finish bookkeeping. + + Owns the file handle. Call build_callbacks() to obtain a Callbacks object whose + handlers route ReAct events through _emit. Call close() in a finally to release + the handle. Assumes log_path.parent already exists. + """ + + def __init__(self, log_path: Path) -> None: + self._log_path = log_path + self._fh = log_path.open("a", encoding="utf-8") + self._closed = False + + @property + def log_path(self) -> Path: + return self._log_path + + def _emit(self, event: dict[str, Any]) -> None: + if self._closed: + return + record = {"ts": datetime.now(UTC).isoformat(), **event} + self._fh.write(json.dumps(record, default=str) + "\n") + self._fh.flush() + + def log_start(self, *, system_prompt: str, prompt: str, tools: list[str]) -> None: + self._emit({"event": "start", "system_prompt": system_prompt, "prompt": prompt, "tools": tools}) + + def log_finish(self, *, success: bool, **fields: Any) -> None: + self._emit({"event": "finish", "success": success, **fields}) + + def close(self) -> None: + if not self._closed: + self._fh.close() + self._closed = True + + def build_callbacks(self) -> Callbacks: + cb_set = CallbackSet() + + @cb_set.on_before_agent_send + async def _on_before_send(iteration: int) -> None: + self._emit({"event": "before_agent_send", "iter": iteration}) + + @cb_set.on_agent_response + async def _on_agent_response(response: AgentResponse, iteration: int) -> None: + self._emit( + { + "event": "agent_response", + "iter": iteration, + "text": response.text, + "tool_calls": [{"id": tc.id, "name": tc.name, "input": tc.input} for tc in response.tool_calls], + "stop_reason": str(response.stop_reason), + "tokens": { + "input": response.usage.input_tokens, + "output": response.usage.output_tokens, + "cache_read": response.usage.cache_read_input_tokens, + "cache_creation": response.usage.cache_creation_input_tokens, + }, + } + ) + + @cb_set.on_tool_call + async def _on_tool_call(tool_call: ToolCall, result: ToolResult, iteration: int) -> None: + self._emit( + { + "event": "tool_call", + "iter": iteration, + "tool_call_id": tool_call.id, + "name": tool_call.name, + "input": tool_call.input, + "result": { + "success": result.success, + "data": result.data, + "error": result.error, + "truncated": result.truncated, + }, + } + ) + + @cb_set.on_before_compact + async def _on_before_compact() -> None: + self._emit({"event": "before_compact"}) + + @cb_set.on_after_compact + async def _on_after_compact() -> None: + self._emit({"event": "after_compact"}) + + @cb_set.on_error + async def _on_error(error: BaseException) -> None: + self._emit({"event": "error", "exception": f"{type(error).__name__}: {error}"}) + + return Callbacks([cb_set]) diff --git a/ddev/src/ddev/ai/tools/agents/spawn_subagent.py b/ddev/src/ddev/ai/tools/agents/spawn_subagent.py new file mode 100644 index 0000000000000..6cc5139fea006 --- /dev/null +++ b/ddev/src/ddev/ai/tools/agents/spawn_subagent.py @@ -0,0 +1,169 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from pathlib import Path +from typing import Annotated + +from pydantic import Field + +from ddev.ai.agent.build import SubagentBuilder +from ddev.ai.agent.types import StopReason +from ddev.ai.react.process import ReActProcess +from ddev.ai.tools.agents.agent_logger import AgentLogger +from ddev.ai.tools.core.base import BaseTool, BaseToolInput +from ddev.ai.tools.core.types import ToolResult + + +class SpawnSubagentInput(BaseToolInput): + system_prompt: Annotated[ + str, + Field(description="System prompt that defines the subagent's role and behavior."), + ] + prompt: Annotated[ + str, + Field(description="The task prompt sent to the subagent as its first (and only) user turn."), + ] + tools: Annotated[ + list[str], + Field( + description=( + "Names of tools the subagent may use. Must be a subset of your tool list and " + "may not include 'spawn_subagent'. May be empty if the subagent should answer " + "from the prompt alone." + ), + ), + ] = [] + name: Annotated[ + str | None, + Field( + description=("Optional short human-readable name for the subagent."), + pattern=r"^$|^[A-Za-z0-9._-]{1,64}$", + ), + ] = None + + +class SpawnSubagentTool(BaseTool[SpawnSubagentInput]): + """Delegate a self-contained subtask to a fresh subagent. + + The subagent runs one Reason-Action loop with the provided system prompt, user prompt, and tool subset. + Only the subagent's final assistant message is returned to you. Instruct the subagent in your prompt + to put anything you need in its final message. Include every piece of context the subagent needs + inside the system prompt and the user prompt.""" + + def __init__( + self, + owner_id: str, + subagent_builder: SubagentBuilder, + allowed_tools: list[str], + log_dir: Path, + ) -> None: + self._owner_id = owner_id + self._subagent_builder = subagent_builder + # Parent may itself have spawn_subagent; never offer it to children. + self._allowed_tools = set(allowed_tools) - {self.name} + self._log_dir = log_dir + self._counter = 0 + + @property + def name(self) -> str: + return "spawn_subagent" + + def _label(self, tool_input: SpawnSubagentInput) -> str: + return tool_input.name or "unnamed" + + async def __call__(self, tool_input: SpawnSubagentInput) -> ToolResult: + label = self._label(tool_input) + + # Subset validation β€” return failed ToolResult; no log file is opened. + if self.name in tool_input.tools: + return ToolResult( + success=False, + error=( + f"Subagent {label!r} not spawned: subagents cannot spawn further subagents " + f"('{self.name}' is not allowed in 'tools')." + ), + ) + disallowed = sorted(set(tool_input.tools) - self._allowed_tools) + if disallowed: + return ToolResult( + success=False, + error=( + f"Subagent {label!r} not spawned: disallowed tools requested: {disallowed}. " + f"Allowed subset: {sorted(self._allowed_tools)}." + ), + ) + + try: + self._log_dir.mkdir(parents=True, exist_ok=True) + except OSError as e: + return ToolResult( + success=False, + error=(f"Subagent {label!r} not spawned: cannot create log directory {self._log_dir}: {e}"), + ) + + self._counter += 1 + subagent_id = f"{self._owner_id}.sub.{self._counter:03d}-{label}" + log_path = self._log_dir / f"{self._counter:03d}-{label}.jsonl" + + try: + logger = AgentLogger(log_path) + except OSError as e: + return ToolResult( + success=False, + error=f"Subagent {label!r} not spawned: cannot open log file {log_path}: {e}", + ) + + try: + logger.log_start( + system_prompt=tool_input.system_prompt, + prompt=tool_input.prompt, + tools=tool_input.tools, + ) + + try: + agent, tool_registry = self._subagent_builder( + tool_input.system_prompt, + subagent_id, + tool_input.tools, + ) + except Exception as e: + logger.log_finish(success=False, error=f"build failed: {type(e).__name__}: {e}") + return ToolResult( + success=False, + error=f"Subagent {label!r} failed to build: {type(e).__name__}: {e}", + ) + + process = ReActProcess( + agent=agent, + tool_registry=tool_registry, + callbacks=logger.build_callbacks(), + ) + try: + result = await process.start(tool_input.prompt) + except Exception as e: + logger.log_finish(success=False, error=f"{type(e).__name__}: {e}") + return ToolResult( + success=False, + error=f"Subagent {label!r} failed: {type(e).__name__}: {e}", + ) + + logger.log_finish( + success=True, + iterations=result.iterations, + total_input_tokens=result.total_input_tokens, + total_output_tokens=result.total_output_tokens, + stop_reason=str(result.final_response.stop_reason), + ) + + data = result.final_response.text + if result.final_response.stop_reason == StopReason.MAX_TOKENS: + data = "[SUBAGENT HIT MAX_TOKENS β€” RESPONSE MAY BE TRUNCATED]\n\n" + data + return ToolResult( + success=True, + data=data, + total_input_tokens=result.total_input_tokens, + total_output_tokens=result.total_output_tokens, + ) + finally: + logger.close() diff --git a/ddev/src/ddev/ai/tools/core/__init__.py b/ddev/src/ddev/ai/tools/core/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/tools/core/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/tools/core/base.py b/ddev/src/ddev/ai/tools/core/base.py new file mode 100644 index 0000000000000..3df60077712db --- /dev/null +++ b/ddev/src/ddev/ai/tools/core/base.py @@ -0,0 +1,114 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import inspect +import logging +import typing +from abc import ABC, abstractmethod +from types import get_original_bases +from typing import Any + +from anthropic.types import ToolParam +from pydantic import BaseModel, ConfigDict + +from .types import ToolResult + +logger = logging.getLogger(__name__) + + +class BaseToolInput(BaseModel): + model_config = ConfigDict(extra='forbid') + + @classmethod + def to_input_schema(cls) -> dict[str, object]: + schema = cls.model_json_schema() + schema.pop('title', None) + for prop in schema.get('properties', {}).values(): + prop.pop('title', None) + prop.pop('default', None) + if 'anyOf' in prop: + non_null = [t for t in prop['anyOf'] if t != {'type': 'null'}] + if len(non_null) == 1: + prop.update(non_null[0]) + del prop['anyOf'] + return schema + + +def _get_input_type(cls: type) -> type[BaseToolInput]: + """Extract the TInput type from a BaseTool subclass""" + if resolved := _resolve_base_tool_arg(cls, {}): + return resolved + raise TypeError(f"{cls.__name__} must be parameterized with an input type: class MyTool(BaseTool[MyInput])") + + +def _resolve_base_tool_arg(cls: type, type_map: dict[Any, Any]) -> type[BaseToolInput] | None: + """Resolve the TInput type from a BaseTool subclass, resolving through intermediate generics.""" + for base in get_original_bases(cls): + origin = typing.get_origin(base) or base + args = typing.get_args(base) + + if origin is BaseTool and args: + resolved = type_map.get(args[0], args[0]) + if isinstance(resolved, type): + return resolved + elif isinstance(origin, type) and issubclass(origin, BaseTool) and origin is not BaseTool: + # Call recursively until we find the generic type of the first BaseTool ancestor. + # Example: + # class EchoTool(BaseTool[EchoInput]): + # pass + # class ChildTool[T](EchoTool): + # pass + # class ConcreteChildTool(ChildTool[int]): + # pass + # _get_input_type(ConcreteChildTool) will resolve to EchoInput. + type_params = origin.__type_params__ + new_map = {param: type_map.get(arg, arg) for param, arg in zip(type_params, args, strict=False)} + if resolved := _resolve_base_tool_arg(origin, new_map): + return resolved + + return None + + +class BaseTool[TInput: BaseToolInput](ABC): + @property + @abstractmethod + def name(self) -> str: + """Unique tool name used in API calls.""" + ... + + @property + def description(self) -> str: + return inspect.cleandoc(self.__class__.__doc__) if self.__class__.__doc__ else "" + + @property + def input_schema(self) -> dict[str, object]: + return _get_input_type(type(self)).to_input_schema() + + @property + def definition(self) -> ToolParam: + """Build the Anthropic SDK ToolParam from this tool's metadata.""" + return { + "name": self.name, + "description": self.description, + "input_schema": self.input_schema, + } + + async def run(self, raw: dict[str, object]) -> ToolResult: + """Coerce raw dict to the typed Input class and delegate to __call__.""" + try: + input_cls = _get_input_type(type(self)) + validated: TInput = input_cls.model_validate(raw) # type: ignore[assignment] + except (TypeError, ValueError) as e: + return ToolResult(success=False, error=str(e)) + try: + return await self(validated) + except Exception as e: + msg = str(e) or repr(e) + logger.exception("Unhandled exception in tool %s: %s", type(self).__name__, msg) + return ToolResult(success=False, error=f"{type(e).__name__}: {msg}") + + @abstractmethod + async def __call__(self, tool_input: TInput) -> ToolResult: + """Call the tool with a typed input instance.""" + ... diff --git a/ddev/src/ddev/ai/tools/core/protocol.py b/ddev/src/ddev/ai/tools/core/protocol.py new file mode 100644 index 0000000000000..0ef3bc426ee3c --- /dev/null +++ b/ddev/src/ddev/ai/tools/core/protocol.py @@ -0,0 +1,19 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from typing import Protocol + +from anthropic.types import ToolParam + +from .types import ToolResult + + +class ToolProtocol(Protocol): + @property + def name(self) -> str: ... + @property + def description(self) -> str: ... + @property + def definition(self) -> ToolParam: ... + async def run(self, raw: dict[str, object]) -> ToolResult: ... diff --git a/ddev/src/ddev/ai/tools/core/truncation.py b/ddev/src/ddev/ai/tools/core/truncation.py new file mode 100644 index 0000000000000..6888929a173fa --- /dev/null +++ b/ddev/src/ddev/ai/tools/core/truncation.py @@ -0,0 +1,109 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import re +from dataclasses import dataclass +from typing import Final + +from ddev.ai.tools.core.types import ToolResult + +MAX_CHARS: Final = 50_000 +HEAD_RATIO: Final = 0.6 + +ERROR_PATTERN = re.compile(r"ERROR|FAILED|Exception|Traceback|fatal|panic", re.IGNORECASE) + + +@dataclass +class TruncationMeta: + total_size: int + shown_size: int + truncated_size: int + hint: str + + +@dataclass +class TruncateResult: + output: str + truncated: bool + meta: TruncationMeta | None + + +def extract_error_lines(lines: list[str]) -> list[tuple[int, str]]: + """Return (index, line) pairs from lines matching error patterns.""" + return [(i, line) for i, line in enumerate(lines) if ERROR_PATTERN.search(line)] + + +def truncate( + content: str, + max_chars: int = MAX_CHARS, + head_ratio: float = HEAD_RATIO, +) -> TruncateResult: + """Truncate content using error-aware head+tail strategy.""" + if len(content) <= max_chars: + return TruncateResult(output=content, truncated=False, meta=None) + + total = len(content) + gap_marker_approx = 50 + content_budget = max(0, max_chars - gap_marker_approx) + head_chars = int(content_budget * head_ratio) + tail_chars = content_budget - head_chars + + head = content[:head_chars] + tail = content[-tail_chars:] if tail_chars > 0 else "" + middle = content[head_chars : total - tail_chars] + + error_lines = extract_error_lines(middle.splitlines()) + + errors_dropped = False + if error_lines: + error_snippet = "\n".join(line for _, line in error_lines) + available = max_chars - len(error_snippet) - gap_marker_approx + if available > 0: + head_share = int(available * head_ratio) + tail_share = available - head_share + head = content[:head_share] + tail = content[-tail_share:] if tail_share > 0 else "" + removed = total - len(head) - len(error_snippet) - len(tail) + gap = f"\n\n[... {removed} characters removed (errors preserved above) ...]\n\n" + result = head + gap + error_snippet + "\n" + tail + else: + errors_dropped = True + removed = total - head_chars - tail_chars + gap = f"\n\n[... {removed} characters removed ...]\n\n" + result = head + gap + tail + else: + removed = total - head_chars - tail_chars + gap = f"\n\n[... {removed} characters removed ...]\n\n" + result = head + gap + tail + + shown = len(result) + if errors_dropped: + hint = ( + f"Output truncated: showing {shown} of {total} characters. " + f"Error lines were detected in the truncated region but could not be preserved " + f"(error snippet exceeded the remaining budget)." + ) + else: + hint = f"Output truncated: showing {shown} of {total} characters." + meta = TruncationMeta( + total_size=total, + shown_size=shown, + truncated_size=total - shown, + hint=hint, + ) + return TruncateResult(output=result, truncated=True, meta=meta) + + +def make_tool_result(success: bool, data: str, result: TruncateResult) -> ToolResult: + """Build a ToolResult, forwarding truncation metadata when present.""" + if result.truncated and result.meta is not None: + return ToolResult( + success=success, + data=data, + truncated=True, + total_size=result.meta.total_size, + shown_size=result.meta.shown_size, + hint=result.meta.hint, + ) + return ToolResult(success=success, data=data) diff --git a/ddev/src/ddev/ai/tools/core/types.py b/ddev/src/ddev/ai/tools/core/types.py new file mode 100644 index 0000000000000..b5a0fc413d492 --- /dev/null +++ b/ddev/src/ddev/ai/tools/core/types.py @@ -0,0 +1,19 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from pydantic import BaseModel + + +class ToolResult(BaseModel): + """Validated result of a tool call.""" + + success: bool + data: str | None = None + error: str | None = None + truncated: bool = False + total_size: int | None = None + shown_size: int | None = None + hint: str | None = None + total_input_tokens: int = 0 + total_output_tokens: int = 0 diff --git a/ddev/src/ddev/ai/tools/fs/__init__.py b/ddev/src/ddev/ai/tools/fs/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/tools/fs/append_file.py b/ddev/src/ddev/ai/tools/fs/append_file.py new file mode 100644 index 0000000000000..5b3ba918c938e --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/append_file.py @@ -0,0 +1,48 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.core.types import ToolResult + +from .base import FileRegistryTool +from .file_access_policy import FileAccessError + + +class AppendFileInput(BaseToolInput): + path: Annotated[str, Field(description="Path of the file to append to")] + content: Annotated[str, Field(description="Content to append to the file")] + + +class AppendFileTool(FileRegistryTool[AppendFileInput]): + """Appends content to the end of an existing file. + Fails if the file was modified since the last read.""" + + @property + def name(self) -> str: + return "append_file" + + async def __call__(self, tool_input: AppendFileInput) -> ToolResult: + try: + path = self._assert_writable(tool_input.path) + except FileAccessError as e: + return ToolResult(success=False, error=str(e)) + + async with self._registry.lock_for(str(path)): + current_content, fail = self._read_verified(str(path)) + if fail: + return fail + + content_to_append = tool_input.content.replace("\r\n", "\n") + separator = "" if not current_content or current_content.endswith("\n") else "\n" + new_content = current_content + separator + content_to_append + + try: + path.write_text(new_content, encoding="utf-8") + except OSError as e: + return ToolResult(success=False, error=str(e)) + self._register(str(path), new_content) + return ToolResult(success=True, data=f"Content appended to: {path}") diff --git a/ddev/src/ddev/ai/tools/fs/base.py b/ddev/src/ddev/ai/tools/fs/base.py new file mode 100644 index 0000000000000..54c523703d768 --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/base.py @@ -0,0 +1,42 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from pathlib import Path + +from ddev.ai.tools.core.base import BaseTool, BaseToolInput +from ddev.ai.tools.core.types import ToolResult + +from .file_registry import FileRegistry + + +class FileRegistryTool[TInput: BaseToolInput](BaseTool[TInput]): + """Abstract base for file system tools with hash-based consistency checks. + + Each tool instance is bound to an owner_id; all registry operations run + under that identity so read-before-write is enforced per owner. + """ + + def __init__(self, file_registry: FileRegistry, owner_id: str) -> None: + self._registry = file_registry + self._owner_id = owner_id + + def _register(self, path: str, content: str) -> None: + self._registry.record(self._owner_id, path, content) + + def _assert_writable(self, path: str) -> Path: + return self._registry.policy.assert_writable(path) + + def _assert_readable(self, path: str) -> Path: + return self._registry.policy.assert_readable(path) + + def _read_verified(self, path: str) -> tuple[str, ToolResult | None]: + """Read file content and verify it matches this agent's last recorded hash.""" + if not self._registry.is_known(self._owner_id, path): + return "", ToolResult(success=False, error=f"Not authorized to modify '{path}'.") + try: + content = Path(path).read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as e: + return "", ToolResult(success=False, error=str(e)) + if not self._registry.verify(self._owner_id, path, content): + return "", ToolResult(success=False, error=f"File '{path}' has changed since last read. Re-read and retry.") + return content, None diff --git a/ddev/src/ddev/ai/tools/fs/create_file.py b/ddev/src/ddev/ai/tools/fs/create_file.py new file mode 100644 index 0000000000000..e73f79e383be0 --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/create_file.py @@ -0,0 +1,47 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.core.types import ToolResult + +from .base import FileRegistryTool +from .file_access_policy import FileAccessError + + +class CreateFileInput(BaseToolInput): + path: Annotated[str, Field(description="Path of the file to create")] + content: Annotated[str, Field(description="Content of the file to create")] = "" + + +class CreateFileTool(FileRegistryTool[CreateFileInput]): + """Creates a new file and writes content into it (default: empty content). + Parent directories are created automatically if they do not exist (no need to call mkdir first). + Registers the file in the file registry. + Fails if the file already exists. + Use edit_file to modify existing files.""" + + @property + def name(self) -> str: + return "create_file" + + async def __call__(self, tool_input: CreateFileInput) -> ToolResult: + try: + path = self._assert_writable(tool_input.path) + except FileAccessError as e: + return ToolResult(success=False, error=str(e)) + + async with self._registry.lock_for(str(path)): + try: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "x", encoding="utf-8") as fh: + fh.write(tool_input.content) + except FileExistsError: + return ToolResult(success=False, error=f"File already exists: {path}") + except OSError as e: + return ToolResult(success=False, error=str(e)) + self._register(str(path), tool_input.content) + return ToolResult(success=True, data=f"File created: {path}") diff --git a/ddev/src/ddev/ai/tools/fs/edit_file.py b/ddev/src/ddev/ai/tools/fs/edit_file.py new file mode 100644 index 0000000000000..f5b90a4b1b179 --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/edit_file.py @@ -0,0 +1,70 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.core.types import ToolResult + +from .base import FileRegistryTool +from .file_access_policy import FileAccessError + + +class EditFileInput(BaseToolInput): + path: Annotated[str, Field(description="Path of the file to edit")] + old_string: Annotated[ + str, + Field( + description=( + "Exact non-empty text to replace. Must appear exactly once in the file " + "(hint: include surrounding context if needed)." + ), + min_length=1, + ), + ] + new_string: Annotated[str, Field(description="Text to replace old_string with")] + + +class EditFileTool(FileRegistryTool[EditFileInput]): + """Edits a file by replacing an exact string with a new one. + Fails if the file was modified since the last read. + old_string must appear exactly once in the file β€” if it appears multiple times, the call fails.""" + + @property + def name(self) -> str: + return "edit_file" + + async def __call__(self, tool_input: EditFileInput) -> ToolResult: + try: + path = self._assert_writable(tool_input.path) + except FileAccessError as e: + return ToolResult(success=False, error=str(e)) + + async with self._registry.lock_for(str(path)): + content, fail = self._read_verified(str(path)) + if fail: + return fail + + # Normalize line endings to avoid issues with different OSs + old_string = tool_input.old_string.replace("\r\n", "\n") + new_string = tool_input.new_string.replace("\r\n", "\n") + + count = content.count(old_string) + if count == 0: + return ToolResult(success=False, error="old_string not found in file") + if count > 1: + return ToolResult( + success=False, + error=f"old_string appears {count} times in the file", + hint="Include more surrounding context to make it unique", + ) + + new_content = content.replace(old_string, new_string, 1) + try: + path.write_text(new_content, encoding="utf-8") + except OSError as e: + return ToolResult(success=False, error=str(e)) + self._register(str(path), new_content) + return ToolResult(success=True, data=f"File edited: {path}") diff --git a/ddev/src/ddev/ai/tools/fs/file_access_policy.py b/ddev/src/ddev/ai/tools/fs/file_access_policy.py new file mode 100644 index 0000000000000..d25fcb0f99573 --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/file_access_policy.py @@ -0,0 +1,142 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os +from collections.abc import Iterable +from fnmatch import fnmatch +from pathlib import Path + + +def canonicalize_path(path: str | Path) -> Path: + """Single source of truth for path canonicalization across the fs layer. + + Every component that compares, indexes, or operates on filesystem paths + must run them through this function so the policy, the tools, and the + registry agree on what path each input names. + + ``strict=False`` allows resolving paths for files that don't exist yet + (e.g. pre-creation checks). Idempotent: calling on an already-canonical + path returns the same path. + """ + return Path(path).expanduser().resolve(strict=False) + + +WILDCARD_CHARS = "*?[" + + +def _canonicalize_pattern(pat: str) -> str: + """Resolve a path pattern's static prefix while leaving wildcards intact. + + Splits at the first wildcard character (``*``, ``?`` or ``[``), runs + expanduser + resolve on the leading prefix, then re-attaches the + wildcard suffix. Patterns without wildcards are fully resolved. Used so + symlinked deny roots (e.g. ``~/.ssh -> /secrets/ssh``) are matched + against the same target the path side canonicalizes to. + """ + indices = [pat.find(c) for c in WILDCARD_CHARS if c in pat] + idx = min(indices) if indices else -1 + if idx == -1: + return str(canonicalize_path(pat)) + prefix, suffix = pat[:idx], pat[idx:] + if not prefix: + return suffix + resolved = str(canonicalize_path(prefix)) + # canonicalize_path strips trailing separators; restore one if the prefix + # had it so "/" + "*" stays "/*" instead of "*". + if prefix.endswith(("/", os.sep)) and not resolved.endswith(("/", os.sep)): + resolved += os.sep + return f"{resolved}{suffix}" + + +DEFAULT_DENY_PATTERNS: tuple[str, ...] = ( + # Location-independent: secrets identified by name or extension. + ".env", + ".env.*", + ".envrc", + ".netrc", + "*.pem", + "*.key", + # Location-rooted: entire directories of secrets. + "~/.ssh/*", + "~/.aws/*", + "~/.gnupg/*", + "~/.config/gcloud/*", + "~/.kube/*", + "~/.docker/*", +) + + +class FileAccessError(Exception): + """Raised when a file access violates the configured policy.""" + + +class FileAccessPolicy: + """Global file access policy shared across agents and phases in a run. + + Enforces a two-zone model based on ``write_root``: inside it, all reads + and writes are allowed. Outside, writes are always denied; reads are + allowed only if the path does not match a deny pattern. + + Each entry in ``deny_patterns`` is an fnmatch-style glob. Patterns are + classified at construction time: + + - **Basename patterns** (no ``/``) are matched against the resolved + path's basename β€” they apply globally regardless of location. Use + these for location-independent rules like ``*.pem`` or ``.env``. + - **Path patterns** (contain ``/``) are matched against the resolved + path's full string. Their static prefix is run through + ``expanduser + resolve`` at construction so symlinked roots cannot + bypass the rule. Use these for location-specific rules like + ``~/.ssh/*`` or ``~/.aws/credentials``. + + Paths checked at runtime go through ``canonicalize_path`` before + matching, so symlinks and ``..`` cannot bypass the checks. + """ + + def __init__( + self, + write_root: Path | str, + deny_patterns: Iterable[str] = DEFAULT_DENY_PATTERNS, + ) -> None: + self._write_root = canonicalize_path(write_root) + patterns = tuple(deny_patterns) + self._deny_patterns: tuple[str, ...] = patterns + + basename: list[str] = [] + path: list[str] = [] + for p in patterns: + (path if "/" in p else basename).append(p) + self._basename_patterns: tuple[str, ...] = tuple(basename) + self._path_patterns: tuple[str, ...] = tuple(_canonicalize_pattern(p) for p in path) + + @property + def write_root(self) -> Path: + return self._write_root + + @property + def deny_patterns(self) -> tuple[str, ...]: + return self._deny_patterns + + @property + def basename_patterns(self) -> tuple[str, ...]: + return self._basename_patterns + + def _is_denied(self, resolved: Path) -> bool: + if any(fnmatch(resolved.name, pat) for pat in self._basename_patterns): + return True + full = str(resolved) + return any(fnmatch(full, pat) for pat in self._path_patterns) + + def assert_readable(self, path: str | Path) -> Path: + resolved = canonicalize_path(path) + if resolved.is_relative_to(self._write_root): + return resolved + if self._is_denied(resolved): + raise FileAccessError(f"Read denied by policy: {resolved}") + return resolved + + def assert_writable(self, path: str | Path) -> Path: + resolved = canonicalize_path(path) + if not resolved.is_relative_to(self._write_root): + raise FileAccessError(f"Write denied: {resolved} is outside write root {self._write_root}") + return resolved diff --git a/ddev/src/ddev/ai/tools/fs/file_registry.py b/ddev/src/ddev/ai/tools/fs/file_registry.py new file mode 100644 index 0000000000000..013956d9401aa --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/file_registry.py @@ -0,0 +1,54 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio +import hashlib + +from .file_access_policy import FileAccessPolicy, canonicalize_path + + +class FileRegistry: + """Tracks the files each owner has seen, along with their last-seen content hash. + + One FileRegistry is intended to be shared across all owners in a run. Hashes + are partitioned by owner_id so that each owner must independently read or + create a file before modifying it; reads by owner A never authorize writes + by owner B. Only SHA-256 digests are stored (not file contents). + + Path-level locks are shared across owners so that concurrent writes to the + same file are serialized regardless of which owner initiated them. + + _hashes layout: {owner_id: {normalized_path: sha256_hex}}. + _locks and _hashes grow for the registry's lifetime and are never evicted. + """ + + def __init__(self, policy: FileAccessPolicy) -> None: + self._policy = policy + self._hashes: dict[str, dict[str, str]] = {} + self._locks: dict[str, asyncio.Lock] = {} + + @property + def policy(self) -> FileAccessPolicy: + return self._policy + + def _normalize(self, path: str) -> str: + return canonicalize_path(path).as_posix() + + def _hash(self, content: str) -> str: + return hashlib.sha256(content.encode()).hexdigest() + + def record(self, owner_id: str, path: str, content: str) -> None: + self._hashes.setdefault(owner_id, {})[self._normalize(path)] = self._hash(content) + + def is_known(self, owner_id: str, path: str) -> bool: + return self._normalize(path) in self._hashes.get(owner_id, {}) + + def verify(self, owner_id: str, path: str, content: str) -> bool: + """Check whether content matches what this agent last recorded for path.""" + stored = self._hashes.get(owner_id, {}).get(self._normalize(path)) + return stored is not None and self._hash(content) == stored + + def lock_for(self, path: str) -> asyncio.Lock: + # Safe under single-threaded asyncio; asyncio.Lock is not thread-safe. + # Path-level (not agent-scoped) so concurrent writes from different agents serialize. + return self._locks.setdefault(self._normalize(path), asyncio.Lock()) diff --git a/ddev/src/ddev/ai/tools/fs/mkdir.py b/ddev/src/ddev/ai/tools/fs/mkdir.py new file mode 100644 index 0000000000000..e5c7ea3a83647 --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/mkdir.py @@ -0,0 +1,39 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseTool, BaseToolInput +from ddev.ai.tools.core.types import ToolResult + +from .file_access_policy import FileAccessError, FileAccessPolicy + + +class MkdirInput(BaseToolInput): + path: Annotated[str, Field(description="Path of the directory to create")] + + +class MkdirTool(BaseTool[MkdirInput]): + """Creates a directory at the given path, including any missing parent directories. + Use to create directories for config files, logs, source code. + Writes are restricted to the configured write root.""" + + def __init__(self, policy: FileAccessPolicy) -> None: + self._policy = policy + + @property + def name(self) -> str: + return "mkdir" + + async def __call__(self, tool_input: MkdirInput) -> ToolResult: + try: + path = self._policy.assert_writable(tool_input.path) + except FileAccessError as e: + return ToolResult(success=False, error=str(e)) + try: + path.mkdir(parents=True, exist_ok=True) + except OSError as e: + return ToolResult(success=False, error=str(e)) + return ToolResult(success=True, data=f"Directory created: {path}") diff --git a/ddev/src/ddev/ai/tools/fs/read_file.py b/ddev/src/ddev/ai/tools/fs/read_file.py new file mode 100644 index 0000000000000..c1b6293c3f395 --- /dev/null +++ b/ddev/src/ddev/ai/tools/fs/read_file.py @@ -0,0 +1,57 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.core.truncation import make_tool_result, truncate +from ddev.ai.tools.core.types import ToolResult + +from .base import FileRegistryTool +from .file_access_policy import FileAccessError + + +class ReadFileInput(BaseToolInput): + path: Annotated[str, Field(description="Absolute or relative path to the file to read")] + offset: Annotated[ + int, Field(description="Line number to start reading from (0-indexed, default: 0). Must be >= 0.", ge=0) + ] = 0 + limit: Annotated[ + int | None, Field(description="Number of lines to read (default: all remaining lines). Must be >= 1.", ge=1) + ] = None + + +class ReadFileTool(FileRegistryTool[ReadFileInput]): + """Reads contents of a text file from the host filesystem. + Use to inspect config files, logs, source code. Do not use for binary files. + The output is a numbered list of lines starting from 0. + Supports offset/limit for paging through large files. + File does not need to be registered in the file registry. + Note: data="" is a valid result meaning no lines in range.""" + + @property + def name(self) -> str: + return "read_file" + + async def __call__(self, tool_input: ReadFileInput) -> ToolResult: + try: + path = self._assert_readable(tool_input.path) + except FileAccessError as e: + return ToolResult(success=False, error=str(e)) + try: + content = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as e: + return ToolResult(success=False, error=f"{tool_input.path}: {e}") + + self._register(str(path), content) + + offset = tool_input.offset + limit = tool_input.limit + lines = content.splitlines(keepends=True) + slice_ = lines[offset : offset + limit] if limit is not None else lines[offset:] + width = len(str(offset + len(slice_))) + numbered = "".join(f"{offset + i:{width}}: {line}" for i, line in enumerate(slice_)) + truncate_result = truncate(numbered) + return make_tool_result(success=True, data=truncate_result.output, result=truncate_result) diff --git a/ddev/src/ddev/ai/tools/http/__init__.py b/ddev/src/ddev/ai/tools/http/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/tools/http/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/tools/http/http_get.py b/ddev/src/ddev/ai/tools/http/http_get.py new file mode 100644 index 0000000000000..a257763ad6c59 --- /dev/null +++ b/ddev/src/ddev/ai/tools/http/http_get.py @@ -0,0 +1,49 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +import httpx +from pydantic import Field + +from ddev.ai.tools.core.base import BaseTool, BaseToolInput +from ddev.ai.tools.core.truncation import make_tool_result, truncate +from ddev.ai.tools.core.types import ToolResult + + +class HttpGetInput(BaseToolInput): + url: Annotated[str, Field(description="Full URL to probe (must start with http:// or https://)")] + timeout: Annotated[float, Field(description="Request timeout in seconds (default: 10)", gt=0)] = 10.0 + + +class HttpGetTool(BaseTool[HttpGetInput]): + """Performs an HTTP GET request to check if an endpoint is reachable. + Use to validate that a metrics endpoint is accessible and inspect its response. + Returns the HTTP status code and response body (truncated if large).""" + + @property + def name(self) -> str: + return "http_get" + + async def __call__(self, tool_input: HttpGetInput) -> ToolResult: + url: str = tool_input.url + timeout: float = tool_input.timeout + + if not url.startswith(("http://", "https://")): + return ToolResult(success=False, error="URL must start with http:// or https://") + + try: + async with httpx.AsyncClient(timeout=timeout) as client: + response = await client.get(url) + except httpx.TimeoutException: + return ToolResult(success=False, error=f"Request timed out after {timeout}s") + except httpx.RequestError as e: + return ToolResult(success=False, error=f"Request failed for {url}: {e}") + + body = response.text + result = truncate(body) + + status_line = f"Status: {response.status_code}" + output = f"{status_line}\n\n{result.output}" + + return make_tool_result(success=True, data=output, result=result) diff --git a/ddev/src/ddev/ai/tools/registry.py b/ddev/src/ddev/ai/tools/registry.py new file mode 100644 index 0000000000000..48b99e8f065da --- /dev/null +++ b/ddev/src/ddev/ai/tools/registry.py @@ -0,0 +1,161 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from importlib import import_module +from pathlib import Path +from typing import TYPE_CHECKING + +from anthropic.types import ToolParam + +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry + +from .core.protocol import ToolProtocol +from .core.types import ToolResult + +if TYPE_CHECKING: + from ddev.ai.agent.build import SubagentBuilder + + +@dataclass +class ToolContext: + """Shared resources passed to every tool factory during construction.""" + + file_registry: FileRegistry + owner_id: str + allowed_tool_names: tuple[str, ...] = field(default_factory=tuple) + subagent_builder: SubagentBuilder | None = None + log_dir: Path | None = None + + @property + def policy(self) -> FileAccessPolicy: + return self.file_registry.policy + + +def _plain_factory(tool_cls: type[ToolProtocol], ctx: ToolContext) -> ToolProtocol: + return tool_cls() + + +def _file_registry_factory(tool_cls: type, ctx: ToolContext) -> ToolProtocol: + return tool_cls(ctx.file_registry, ctx.owner_id) + + +def _file_policy_factory(tool_cls: type, ctx: ToolContext) -> ToolProtocol: + return tool_cls(ctx.policy) + + +def _spawn_subagent_factory(tool_cls: type, ctx: ToolContext) -> ToolProtocol: + if ctx.subagent_builder is None or ctx.log_dir is None: + raise ValueError( + "Tool 'spawn_subagent' requires both 'subagent_builder' and 'log_dir' to be " + "passed to ToolRegistry.from_names." + ) + allowed = [name for name in ctx.allowed_tool_names if name != "spawn_subagent"] + return tool_cls( + owner_id=ctx.owner_id, + subagent_builder=ctx.subagent_builder, + allowed_tools=allowed, + log_dir=ctx.log_dir, + ) + + +@dataclass(frozen=True) +class ToolSpec: + """Lazy pointer to a tool class and how to construct it. + + ``module`` is relative to the registry's package (e.g. ``"fs.read_file"``). + ``factory`` receives the already-imported class and the shared ToolContext + and returns a constructed tool instance. + ``requires_subagent_builder`` marks agentic tools that need subagent wiring. + """ + + module: str + cls: str + factory: Callable[[type, ToolContext], ToolProtocol] = _plain_factory + requires_subagent_builder: bool = False + + +TOOL_MANIFEST: dict[str, ToolSpec] = { + "read_file": ToolSpec("fs.read_file", "ReadFileTool", factory=_file_registry_factory), + "create_file": ToolSpec("fs.create_file", "CreateFileTool", factory=_file_registry_factory), + "edit_file": ToolSpec("fs.edit_file", "EditFileTool", factory=_file_registry_factory), + "append_file": ToolSpec("fs.append_file", "AppendFileTool", factory=_file_registry_factory), + "grep": ToolSpec("shell.grep", "GrepTool", factory=_file_policy_factory), + "list_files": ToolSpec("shell.list_files", "ListFilesTool"), + "mkdir": ToolSpec("fs.mkdir", "MkdirTool", factory=_file_policy_factory), + "http_get": ToolSpec("http.http_get", "HttpGetTool"), + "ddev_create": ToolSpec("shell.ddev.create", "DdevCreateTool"), + "ddev_test": ToolSpec("shell.ddev.ddev_test", "DdevTestTool"), + "ddev_env_show": ToolSpec("shell.ddev.env_show", "DdevEnvShowTool"), + "ddev_env_start": ToolSpec("shell.ddev.env_start", "DdevEnvStartTool"), + "ddev_env_stop": ToolSpec("shell.ddev.env_stop", "DdevEnvStopTool"), + "ddev_env_test": ToolSpec("shell.ddev.env_test", "DdevEnvTestTool"), + "ddev_release_changelog": ToolSpec("shell.ddev.release_changelog", "DdevReleaseChangelogTool"), + "ddev_validate": ToolSpec("shell.ddev.validate", "DdevValidateTool"), + "spawn_subagent": ToolSpec( + "agents.spawn_subagent", + "SpawnSubagentTool", + factory=_spawn_subagent_factory, + requires_subagent_builder=True, + ), +} + + +class ToolRegistry: + """Registry holding all available tools.""" + + def __init__(self, tools: list[ToolProtocol]) -> None: + self._tools: dict[str, ToolProtocol] = {tool.name: tool for tool in tools} + + @staticmethod + def available_tool_names() -> list[str]: + """Return all tool names that from_names can resolve.""" + return list(TOOL_MANIFEST) + + @classmethod + def from_names( + cls, + tool_names: list[str], + *, + owner_id: str, + file_registry: FileRegistry, + subagent_builder: SubagentBuilder | None = None, + log_dir: Path | None = None, + ) -> ToolRegistry: + """Build a ToolRegistry from a list of tool name strings. + + The file_registry is shared across all owners in a run so that the access + policy applies globally; hashes inside it are partitioned by owner_id so + each owner must still read-before-write on its own. + """ + ctx = ToolContext( + file_registry=file_registry, + owner_id=owner_id, + allowed_tool_names=tuple(tool_names), + subagent_builder=subagent_builder, + log_dir=log_dir, + ) + tools: list[ToolProtocol] = [] + for name in tool_names: + spec = TOOL_MANIFEST.get(name) + if spec is None: + raise ValueError(f"Unknown tool name: {name!r}") + tool_cls = getattr(import_module(f"{__package__}.{spec.module}"), spec.cls) + tools.append(spec.factory(tool_cls, ctx)) + return cls(tools) + + @property + def definitions(self) -> list[ToolParam]: + """Return Anthropic SDK tool definitions for all registered tools.""" + return [tool.definition for tool in self._tools.values()] + + async def run(self, name: str, raw: dict[str, object]) -> ToolResult: + """Execute a tool by name, returning an error result if not found.""" + tool = self._tools.get(name) + if tool is None: + return ToolResult(success=False, error=f"Unknown tool: {name!r}") + return await tool.run(raw) diff --git a/ddev/src/ddev/ai/tools/shell/__init__.py b/ddev/src/ddev/ai/tools/shell/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/tools/shell/base.py b/ddev/src/ddev/ai/tools/shell/base.py new file mode 100644 index 0000000000000..2ff25110e4161 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/base.py @@ -0,0 +1,64 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio +from abc import abstractmethod +from collections.abc import Callable +from typing import ClassVar + +from ddev.ai.tools.core.base import BaseTool, BaseToolInput +from ddev.ai.tools.core.truncation import make_tool_result, truncate +from ddev.ai.tools.core.types import ToolResult + + +class CmdTool[TInput: BaseToolInput](BaseTool[TInput]): + """Base for tools that execute shell commands.""" + + timeout: ClassVar[int] = 10 + + @abstractmethod + def cmd(self, tool_input: TInput) -> list[str]: + """Builds the shell command from validated tool input.""" + ... + + async def __call__(self, tool_input: TInput) -> ToolResult: + return await run_command(self.cmd(tool_input), timeout=self.timeout) + + +async def run_command( + cmd: list[str], + timeout: int = 10, + stdout_filter: Callable[[str], str] | None = None, +) -> ToolResult: + try: + proc = await asyncio.create_subprocess_exec( + *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + stdout_bytes, stderr_bytes = await asyncio.wait_for(proc.communicate(), timeout=timeout) + except FileNotFoundError: + return ToolResult(success=False, error=f"Command not found: {cmd[0]!r}") + except asyncio.TimeoutError: + proc.kill() + await proc.communicate() + return ToolResult(success=False, error=f"Command timed out after {timeout}s: {cmd}") + except Exception as e: + return ToolResult(success=False, error=f"{type(e).__name__}: {e}") + + # errors="replace" to keep output readable in case of non-UTF-8 characters + stdout = stdout_bytes.decode("utf-8", errors="replace") + stderr = stderr_bytes.decode("utf-8", errors="replace") + + if stdout_filter is not None: + stdout = stdout_filter(stdout) + + output = stdout + if proc.returncode != 0 and stderr: + output = (output + "\n" + stderr) if output else stderr + elif not output and stderr: + output = stderr + + if not output.strip(): + return ToolResult(success=proc.returncode == 0, data="(no output)") + + result = truncate(output) + return make_tool_result(success=proc.returncode == 0, data=result.output, result=result) diff --git a/ddev/src/ddev/ai/tools/shell/ddev/__init__.py b/ddev/src/ddev/ai/tools/shell/ddev/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/ai/tools/shell/ddev/create.py b/ddev/src/ddev/ai/tools/shell/ddev/create.py new file mode 100644 index 0000000000000..7e77e8ec19e0c --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/create.py @@ -0,0 +1,39 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated, Literal + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + +IntegrationType = Literal["check", "check_only", "event", "jmx", "logs", "metrics_crawler", "snmp_tile", "tile"] + + +class CreateInput(BaseToolInput): + integration: Annotated[str, Field(description="Name of the new integration (snake_case)")] + integration_type: Annotated[ + IntegrationType, + Field( + description="Template type: 'check' (standard Agent check), 'check_only' (no hatch env)," + " 'event', 'jmx', 'logs', 'metrics_crawler', 'snmp_tile', 'tile'" + ), + ] + + +class DdevCreateTool(CmdTool[CreateInput]): + """Scaffolds a new Datadog Agent integration with all boilerplate files and + directory structure. Creates a directory named after the integration (snake_case) + in the current working directory. Use before writing any integration code.""" + + timeout = 60 + + @property + def name(self) -> str: + return "ddev_create" + + def cmd(self, tool_input: CreateInput) -> list[str]: + # Capitalize to avoid ddev's islower() interactive prompt; normalize_package_name restores snake_case + name = tool_input.integration.capitalize() + return ["ddev", "--no-interactive", "create", "--type", tool_input.integration_type, "--skip-manifest", name] diff --git a/ddev/src/ddev/ai/tools/shell/ddev/ddev_test.py b/ddev/src/ddev/ai/tools/shell/ddev/ddev_test.py new file mode 100644 index 0000000000000..eccdd343fc284 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/ddev_test.py @@ -0,0 +1,43 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + + +class DdevTestInput(BaseToolInput): + integration: Annotated[str, Field(description="Integration name to test")] + lint: Annotated[bool, Field(description="Run linter / style checks only (-s / --lint)")] = False + fmt: Annotated[bool, Field(description="Fix formatting and linting errors (-fs / --fmt)")] = False + pytest_args: Annotated[ + list[str] | None, + Field(description='Extra pytest arguments passed after `--` (e.g. ["-k", "test_my_func", "-s"])'), + ] = None + + +class DdevTestTool(CmdTool[DdevTestInput]): + """Runs unit and integration tests for the given integration. Set `lint=true` + to run the linter only. Set `fmt=true` to fix formatting and linting errors. + Use `pytest_args` to pass extra pytest arguments (e.g. `["-k", "test_my_func"]`) + to run specific tests instead of the full suite.""" + + timeout = 600 + + @property + def name(self) -> str: + return "ddev_test" + + def cmd(self, tool_input: DdevTestInput) -> list[str]: + cmd = ["ddev", "--no-interactive", "test"] + if tool_input.lint: + cmd.append("-s") + if tool_input.fmt: + cmd.append("-fs") + cmd.append(tool_input.integration) + if tool_input.pytest_args: + cmd += ["--"] + tool_input.pytest_args + return cmd diff --git a/ddev/src/ddev/ai/tools/shell/ddev/env_show.py b/ddev/src/ddev/ai/tools/shell/ddev/env_show.py new file mode 100644 index 0000000000000..4ecb5ff8b0495 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/env_show.py @@ -0,0 +1,27 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + + +class EnvShowInput(BaseToolInput): + integration: Annotated[str, Field(description="Integration name")] + + +class DdevEnvShowTool(CmdTool[EnvShowInput]): + """Lists all available E2E environment names for an integration. Call this + before `ddev_env_test` or `ddev_env_start` to discover valid environment names.""" + + timeout = 30 + + @property + def name(self) -> str: + return "ddev_env_show" + + def cmd(self, tool_input: EnvShowInput) -> list[str]: + return ["ddev", "--no-interactive", "env", "show", tool_input.integration] diff --git a/ddev/src/ddev/ai/tools/shell/ddev/env_start.py b/ddev/src/ddev/ai/tools/shell/ddev/env_start.py new file mode 100644 index 0000000000000..45ba0d90a870f --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/env_start.py @@ -0,0 +1,33 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + + +class EnvStartInput(BaseToolInput): + integration: Annotated[str, Field(description="Integration name")] + environment: Annotated[str, Field(description="Environment name (e.g. py3.11-1.23)")] + dev: Annotated[bool, Field(description="Mount local check code into the container")] = False + + +class DdevEnvStartTool(CmdTool[EnvStartInput]): + """Starts a Docker-based E2E test environment for an integration. Use + `dev=true` to mount local check code into the container.""" + + timeout = 300 + + @property + def name(self) -> str: + return "ddev_env_start" + + def cmd(self, tool_input: EnvStartInput) -> list[str]: + cmd = ["ddev", "--no-interactive", "env", "start"] + if tool_input.dev: + cmd.append("--dev") + cmd += [tool_input.integration, tool_input.environment] + return cmd diff --git a/ddev/src/ddev/ai/tools/shell/ddev/env_stop.py b/ddev/src/ddev/ai/tools/shell/ddev/env_stop.py new file mode 100644 index 0000000000000..3f55a01db76bf --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/env_stop.py @@ -0,0 +1,27 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + + +class EnvStopInput(BaseToolInput): + integration: Annotated[str, Field(description="Integration name")] + environment: Annotated[str, Field(description="Environment name (e.g. py3.11-1.23)")] + + +class DdevEnvStopTool(CmdTool[EnvStopInput]): + """Stops and removes the Docker environment for the given integration and environment name.""" + + timeout = 120 + + @property + def name(self) -> str: + return "ddev_env_stop" + + def cmd(self, tool_input: EnvStopInput) -> list[str]: + return ["ddev", "--no-interactive", "env", "stop", tool_input.integration, tool_input.environment] diff --git a/ddev/src/ddev/ai/tools/shell/ddev/env_test.py b/ddev/src/ddev/ai/tools/shell/ddev/env_test.py new file mode 100644 index 0000000000000..915de5debdd41 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/env_test.py @@ -0,0 +1,34 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + + +class EnvTestInput(BaseToolInput): + integration: Annotated[str, Field(description="Integration name")] + environment: Annotated[str, Field(description="Environment name (e.g. py3.11-1.23)")] + dev: Annotated[bool, Field(description="Pass --dev flag (use if env was started with --dev)")] = False + + +class DdevEnvTestTool(CmdTool[EnvTestInput]): + """Runs E2E tests for the given integration in the specified environment. + `ddev env test` starts the environment, runs the tests, and stops it automatically β€” + no prior `ddev_env_start` is needed. Use `dev=true` to pass the `--dev` flag.""" + + timeout = 600 + + @property + def name(self) -> str: + return "ddev_env_test" + + def cmd(self, tool_input: EnvTestInput) -> list[str]: + cmd = ["ddev", "--no-interactive", "env", "test"] + if tool_input.dev: + cmd.append("--dev") + cmd += [tool_input.integration, tool_input.environment] + return cmd diff --git a/ddev/src/ddev/ai/tools/shell/ddev/release_changelog.py b/ddev/src/ddev/ai/tools/shell/ddev/release_changelog.py new file mode 100644 index 0000000000000..cf3e3006e7d89 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/release_changelog.py @@ -0,0 +1,41 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated, Literal + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + + +class ReleaseChangelogInput(BaseToolInput): + change_type: Annotated[ + Literal["fixed", "added", "changed"], + Field(description="Type of change: 'fixed' (patch), 'added' (minor), 'changed' (major)"), + ] + integration: Annotated[str, Field(description="Integration name")] + message: Annotated[str, Field(description="Human-readable changelog message")] + + +class DdevReleaseChangelogTool(CmdTool[ReleaseChangelogInput]): + """Creates a changelog entry file for the integration. `change_type` must be + `"fixed"` (patch bump), `"added"` (minor bump), or `"changed"` (major bump).""" + + timeout = 30 + + @property + def name(self) -> str: + return "ddev_release_changelog" + + def cmd(self, tool_input: ReleaseChangelogInput) -> list[str]: + return [ + "ddev", + "release", + "changelog", + "new", + tool_input.change_type, + tool_input.integration, + "-m", + tool_input.message, + ] diff --git a/ddev/src/ddev/ai/tools/shell/ddev/validate.py b/ddev/src/ddev/ai/tools/shell/ddev/validate.py new file mode 100644 index 0000000000000..9d6d87a050e84 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/ddev/validate.py @@ -0,0 +1,61 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated, Literal + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.shell.base import CmdTool + +ValidateSubcommand = Literal["config", "models", "metadata", "all"] + + +class DdevValidateInput(BaseToolInput): + subcommand: Annotated[ + ValidateSubcommand, + Field( + description=( + "Which validator to run. Options:" + "- 'config': validates assets/configuration/spec.yaml against data/conf.yaml.example. " + "- 'models': validates spec.yaml against datadog_checks//config_models/. " + "- 'metadata': validates metadata.csv. " + "- 'all': runs all ~20 validators in parallel. Some of these always scan the entire " + "repository, so the output may include failures for files outside of . " + "IGNORE those unrelated failures β€” only act on failures that reference files inside " + "your integration's directory. It might take a long time to run. Use 'all' only as " + "a final sweep to catch issues the targeted validators do not cover." + ) + ), + ] + integration: Annotated[str, Field(description="Integration name to validate")] + sync: Annotated[ + bool, + Field( + description=( + "Regenerate / auto-fix derived files instead of only checking. " + "For 'config', regenerates conf.yaml.example. " + "For 'models', regenerates config_models/. " + "For 'metadata', rewrites metadata.csv into canonical form. " + "For 'all', auto-fixes every validator that supports it." + ) + ), + ] = False + + +class DdevValidateTool(CmdTool[DdevValidateInput]): + """Validates an integration's spec, config example, config models, or metadata.csv. + Set `sync=true` to regenerate the derived files from spec.yaml.""" + + timeout = 660 + + @property + def name(self) -> str: + return "ddev_validate" + + def cmd(self, tool_input: DdevValidateInput) -> list[str]: + cmd = ["ddev", "--no-interactive", "validate", tool_input.subcommand] + if tool_input.sync: + cmd.append("--fix" if tool_input.subcommand == "all" else "--sync") + cmd.append(tool_input.integration) + return cmd diff --git a/ddev/src/ddev/ai/tools/shell/grep.py b/ddev/src/ddev/ai/tools/shell/grep.py new file mode 100644 index 0000000000000..3d8ba491b7b4c --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/grep.py @@ -0,0 +1,102 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from pathlib import Path +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.fs.file_access_policy import FileAccessError, FileAccessPolicy, canonicalize_path + +from .base import CmdTool, run_command + + +class GrepInput(BaseToolInput): + pattern: Annotated[str, Field(description="Regex pattern to search for")] + path: Annotated[str, Field(description="File or directory to search in")] + recursive: Annotated[bool, Field(description="Search recursively in directories (default: true)")] = True + + +class GrepTool(CmdTool[GrepInput]): + """Searches for a regex pattern in files. Returns matching lines with file path and line + numbers. Use to find specific config values, ports, hostnames across files. Supports extended + regex syntax. Output might be truncated for large results. + """ + + timeout = 30 + + def __init__(self, policy: FileAccessPolicy) -> None: + self._policy = policy + + @property + def name(self) -> str: + return "grep" + + async def __call__(self, tool_input: GrepInput) -> ToolResult: + try: + self._policy.assert_readable(tool_input.path) + except FileAccessError as e: + return ToolResult(success=False, error=str(e)) + result = await run_command( + self.cmd(tool_input), + timeout=self.timeout, + stdout_filter=self._filter_stdout if tool_input.recursive else None, + ) + # grep exits 1 when no lines match β€” not a failure + if not result.success and result.error is None: + return result.model_copy(update={"success": True}) + return result + + def cmd(self, tool_input: GrepInput) -> list[str]: + cmd = ["grep", "-n", "-E", "--null", "-I", "--no-messages"] + if tool_input.recursive: + cmd.append("-r") + cmd.extend(self._exclude_flags(canonicalize_path(tool_input.path))) + cmd += ["--", tool_input.pattern, tool_input.path] + return cmd + + def _exclude_flags(self, search_path: Path) -> list[str]: + # Skip --exclude= flags when the search overlaps write_root: either the + # search is inside write_root (all files are visible) or write_root is + # inside the search (mixing zones). In both cases the post-filter handles + # per-line decisions correctly. Only apply flags when the entire search + # is outside write_root, where deny patterns are fully in effect. + write_root = self._policy.write_root + if search_path.is_relative_to(write_root) or write_root.is_relative_to(search_path): + return [] + return [f"--exclude={pat}" for pat in self._policy.basename_patterns] + + def _filter_stdout(self, stdout: str) -> str: + """Filter stdout to only include lines whose filename is allowed by the policy. + If the filename is denied, we return 'Read denied by policy' instead of the line. + + ``grep --null`` output: ``\\0:\\n``. Split on the + first NUL and run the filename through ``assert_readable`` (which + canonicalizes through symlinks). + + Only use when recursive is True. + """ + decision: dict[str, bool] = {} + result: list[str] = [] + emitted_denials: set[str] = set() + for line in stdout.splitlines(): + nul = line.find("\0") + if nul == -1: + continue + filename, rest = line[:nul], line[nul + 1 :] + allowed = decision.get(filename) + if allowed is None: + try: + self._policy.assert_readable(filename) + allowed = True + except FileAccessError: + allowed = False + decision[filename] = allowed + if allowed: + result.append(f"{filename}:{rest}") + elif filename not in emitted_denials: + result.append(f"{filename}: Read denied by policy") + emitted_denials.add(filename) + return "\n".join(result) diff --git a/ddev/src/ddev/ai/tools/shell/list_files.py b/ddev/src/ddev/ai/tools/shell/list_files.py new file mode 100644 index 0000000000000..a54ed6baceb44 --- /dev/null +++ b/ddev/src/ddev/ai/tools/shell/list_files.py @@ -0,0 +1,32 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput + +from .base import CmdTool + + +class ListFilesInput(BaseToolInput): + path: Annotated[str, Field(description="Path to list files from")] + recursive: Annotated[bool, Field(description="Whether to list recursively (default: false)")] = False + + +class ListFilesTool(CmdTool[ListFilesInput]): + """Lists files and directories at the given path. Use to explore directory structure and find + config files. Non-recursive by default - set recursive=true for a deep listing.""" + + timeout = 30 + + @property + def name(self) -> str: + return "list_files" + + def cmd(self, tool_input: ListFilesInput) -> list[str]: + cmd = ["find", tool_input.path, "-mindepth", "1"] + if not tool_input.recursive: + cmd += ["-maxdepth", "1"] + return cmd diff --git a/ddev/src/ddev/cli/dep/promote.py b/ddev/src/ddev/cli/dep/promote.py index d298e7bbe1c23..c7f3210de3671 100644 --- a/ddev/src/ddev/cli/dep/promote.py +++ b/ddev/src/ddev/cli/dep/promote.py @@ -3,6 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from __future__ import annotations +import logging import re from typing import TYPE_CHECKING @@ -39,20 +40,26 @@ def promote(app: Application, pr_url: str): pr_number = int(match.group(1)) - with app.status(f'Fetching PR #{pr_number} head...'): - head_sha, head_ref = app.github.get_pr_head(pr_number) + httpx_logger = logging.getLogger('httpx') + previous_level = httpx_logger.level + httpx_logger.setLevel(logging.WARNING) + try: + with app.status(f'Fetching PR #{pr_number} head...'): + head_sha, head_ref = app.github.get_pr_head(pr_number) - app.display_info(f'PR #{pr_number} β€” branch: {head_ref}, SHA: {head_sha}') + app.display_info(f'PR #{pr_number}: branch {head_ref}, SHA {head_sha}') - with app.status('Dispatching promote workflow...'): - app.github.dispatch_workflow( - workflow_id=PROMOTE_WORKFLOW, - ref=PROMOTE_WORKFLOW_REF, - inputs={'pr_number': str(pr_number), 'head_sha': head_sha}, - ) + with app.status('Dispatching promote workflow...'): + run_details = app.github.dispatch_workflow( + workflow_id=PROMOTE_WORKFLOW, + ref=PROMOTE_WORKFLOW_REF, + inputs={'pr_number': str(pr_number), 'head_sha': head_sha}, + return_run_details=True, + ) - runs_url = ( - f'https://github.com/{app.github.repo_id}/actions/workflows/{PROMOTE_WORKFLOW}?query=event%3Aworkflow_dispatch' - ) - app.display_success(f'Promote workflow dispatched for PR #{pr_number}.') - app.display_info(f'Recent runs: {runs_url}') + if not run_details: + app.abort('Workflow dispatched but no run details were returned.') + app.display_success(f'Promote workflow dispatched for PR #{pr_number}.') + app.display_info(f'Workflow run: {run_details["html_url"]}') + finally: + httpx_logger.setLevel(previous_level) diff --git a/ddev/src/ddev/cli/release/agent/changelog.py b/ddev/src/ddev/cli/release/agent/changelog.py index fba92bc29375d..114a7b295f721 100644 --- a/ddev/src/ddev/cli/release/agent/changelog.py +++ b/ddev/src/ddev/cli/release/agent/changelog.py @@ -58,7 +58,10 @@ def changelog(app: Application, since: str, to: str, write: bool, force: bool): app.repo.git.fetch_tags() - changes_per_agent = get_changes_per_agent(app.repo, since, to) + try: + changes_per_agent = get_changes_per_agent(app.repo, since, to) + except ValueError as exc: + app.abort(str(exc)) # store the changelog in memory changelog_contents = StringIO() diff --git a/ddev/src/ddev/cli/release/agent/common.py b/ddev/src/ddev/cli/release/agent/common.py index a1c8673d812fa..71cbb55c69f50 100644 --- a/ddev/src/ddev/cli/release/agent/common.py +++ b/ddev/src/ddev/cli/release/agent/common.py @@ -11,6 +11,7 @@ AgentChangelog = dict[str, dict[str, tuple[str, bool, bool]]] DATADOG_PACKAGE_PREFIX = 'datadog-' +UNRELEASED_INTEGRATIONS_CONFIG = '/overrides/release/agent/unreleased-integrations' def get_agent_tags(repo: Repository, since: str, to: str) -> list[str]: @@ -73,11 +74,8 @@ def get_changes_per_agent(repo: Repository, since: str, to: str) -> AgentChangel file_contents = repo.git.show_file(req_file_name, agent_tags[i]) catalog_prev = parse_agent_req_file(file_contents) - # at some point in the git history, the requirements file erroneously - # contained the folder name instead of the package name for each check, - # let's be resilient by normalizing all entries to be folder names - catalog_now = normalize_catalog(catalog_now) - catalog_prev = normalize_catalog(catalog_prev) + catalog_now = exclude_unreleased_integrations(repo, normalize_catalog(catalog_now), current_tag) + catalog_prev = exclude_unreleased_integrations(repo, normalize_catalog(catalog_prev), agent_tags[i]) changes_per_agent[current_tag] = {} @@ -94,10 +92,56 @@ def get_changes_per_agent(repo: Repository, since: str, to: str) -> AgentChangel return changes_per_agent +# at some point in the git history, the requirements file erroneously +# contained the folder name instead of the package name for each check, +# let's be resilient by normalizing all entries to be folder names def normalize_catalog(catalog: dict[str, str]) -> dict[str, str]: return {normalize_package_name(k): v for k, v in catalog.items()} +def exclude_unreleased_integrations(repo: Repository, catalog: dict[str, str], agent_version: str) -> dict[str, str]: + """Filter integrations listed as unreleased for ``agent_version``; catalog keys may be raw or folder-normalized.""" + skipped_integrations = get_unreleased_integrations(repo, agent_version) + if not skipped_integrations: + return catalog + return { + name: version for name, version in catalog.items() if normalize_package_name(name) not in skipped_integrations + } + + +def get_unreleased_integrations(repo: Repository, agent_version: str) -> set[str]: + unreleased_integrations = repo.config.get(UNRELEASED_INTEGRATIONS_CONFIG, default={}) + by_integration = unreleased_integrations.get('by-integration', {}) + by_agent_version_range = unreleased_integrations.get('by-agent-version-range', {}) + + skipped_integrations = { + normalize_package_name(name) for name, versions in by_integration.items() if agent_version in versions + } + for version_range, integration_names in by_agent_version_range.items(): + if agent_version_in_range(agent_version, version_range): + skipped_integrations.update(normalize_package_name(name) for name in integration_names) + + return skipped_integrations + + +def agent_version_in_range(agent_version: str, version_range: str) -> bool: + from packaging.version import parse as parse_version + + parts = version_range.split('..', 1) + if len(parts) != 2: + raise ValueError( + f"Invalid version range {version_range!r} in " + f"{UNRELEASED_INTEGRATIONS_CONFIG}/by-agent-version-range; " + "expected format: 'START..END'" + ) + start, end = parts + version = parse_version(agent_version) + start_version = parse_version(start) + end_version = parse_version(end) + + return start_version <= version <= end_version + + def normalize_package_name(name: str) -> str: """ Given a Python package name for a check, return the corresponding folder diff --git a/ddev/src/ddev/cli/release/agent/integrations.py b/ddev/src/ddev/cli/release/agent/integrations.py index 9c8281baebe4c..156a3aa0b60e7 100644 --- a/ddev/src/ddev/cli/release/agent/integrations.py +++ b/ddev/src/ddev/cli/release/agent/integrations.py @@ -29,7 +29,7 @@ def integrations(app: Application, since: str, to: str, write: bool, force: bool tool will generate the list for every Agent since version 6.3.0 (before that point we don't have enough information to build the log). """ - from ddev.cli.release.agent.common import get_agent_tags, parse_agent_req_file + from ddev.cli.release.agent.common import exclude_unreleased_integrations, get_agent_tags, parse_agent_req_file agent_tags = get_agent_tags(app.repo, since, to) # get the list of integrations shipped with the agent from the requirements file @@ -40,7 +40,11 @@ def integrations(app: Application, since: str, to: str, write: bool, force: bool integrations_contents.write(f'## Datadog Agent version {tag}\n\n') # Requirements for current tag file_contents = app.repo.git.show_file(req_file_name, tag) - for name, ver in parse_agent_req_file(file_contents).items(): + try: + catalog = exclude_unreleased_integrations(app.repo, parse_agent_req_file(file_contents), tag) + except ValueError as exc: + app.abort(str(exc)) + for name, ver in catalog.items(): integrations_contents.write(f'* {name}: {ver}\n') integrations_contents.write('\n') diff --git a/ddev/src/ddev/cli/release/agent/integrations_changelog.py b/ddev/src/ddev/cli/release/agent/integrations_changelog.py index b3f70a33e86ec..343fa385964ea 100644 --- a/ddev/src/ddev/cli/release/agent/integrations_changelog.py +++ b/ddev/src/ddev/cli/release/agent/integrations_changelog.py @@ -39,7 +39,10 @@ def integrations_changelog(app: Application, integrations: tuple[str], since: st if not integrations: integrations = [integration.name for integration in app.repo.integrations.iter_all('all')] - changes_per_agent = get_changes_per_agent(app.repo, since, to) + try: + changes_per_agent = get_changes_per_agent(app.repo, since, to) + except ValueError as exc: + app.abort(str(exc)) integrations_versions: dict[str, dict[str, str]] = defaultdict(dict) for agent_version, version_changes in changes_per_agent.items(): diff --git a/ddev/src/ddev/cli/validate/all/orchestrator.py b/ddev/src/ddev/cli/validate/all/orchestrator.py index 3ed7b21439293..2a5fa3b7a3da7 100644 --- a/ddev/src/ddev/cli/validate/all/orchestrator.py +++ b/ddev/src/ddev/cli/validate/all/orchestrator.py @@ -42,7 +42,7 @@ class ValidationConfig: description="Verify check versions match the Agent requirements file", ), "ci": ValidationConfig( - description="Validate CI configuration and Codecov settings", + description="Validate CI configuration and code coverage settings", repo_wide=True, fix_flag="--sync", ), diff --git a/ddev/src/ddev/cli/validate/ci.py b/ddev/src/ddev/cli/validate/ci.py index d968b24d87023..ee50af3cc6800 100644 --- a/ddev/src/ddev/cli/validate/ci.py +++ b/ddev/src/ddev/cli/validate/ci.py @@ -3,6 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from __future__ import annotations +from collections import Counter from typing import TYPE_CHECKING, Any import click @@ -10,33 +11,20 @@ if TYPE_CHECKING: from ddev.cli.application import Application +DEFAULT_COVERAGE_THRESHOLD = 75 -def read_file(file, encoding='utf-8'): - # type: (str, str) -> str - with open(file, 'r', encoding=encoding) as f: - return f.read() - -def write_file(file, contents, encoding='utf-8'): - with open(file, 'w', encoding=encoding) as f: - f.write(contents) - - -def code_coverage_enabled(check_name, app): +def code_coverage_enabled(check_name: str, app: Application) -> bool: if check_name in ('datadog_checks_base', 'datadog_checks_dev', 'datadog_checks_downloader', 'ddev'): return True return app.repo.integrations.get(check_name).is_agent_check -def get_coverage_sources(check_name, app): +def get_coverage_sources(check_name: str, app: Application) -> list[str]: package_path = app.repo.integrations.get(check_name).package_directory package_dir = package_path.relative_to(app.repo.path) - return sorted([str(package_dir.as_posix()), f'{check_name}/tests']) - - -def sort_projects(projects): - return sorted(projects.items(), key=lambda item: (item[0] != 'default', item[0])) + return [f'{package_dir.as_posix()}/'] @click.command() @@ -46,9 +34,7 @@ def ci(app: Application, sync: bool): """Validate CI infrastructure configuration.""" import hashlib import json - import os import re - from collections import defaultdict import yaml @@ -237,184 +223,168 @@ def ci(app: Application, sync: bool): app.abort('CI configuration is not in sync, try again with the `--sync` flag') validation_tracker = app.create_validation_tracker('CI configuration validation') - error_message = '' - warning_message = '' repo_choice = app.repo.name valid_repos = ['core', 'marketplace', 'extras', 'internal'] if repo_choice not in valid_repos: app.abort(f'Unknown repository `{repo_choice}`') - # marketplace does not have a .codecov.yml file - if app.repo.name == 'marketplace': + if is_marketplace: + validation_tracker.success() + validation_tracker.display() return - testable_checks = {integration.name for integration in app.repo.integrations.iter_testable('all')} + _validate_code_coverage(app, sync, validation_tracker, repo_choice) - cached_display_names: defaultdict[str, str] = defaultdict(str) - codecov_config_relative_path = '.codecov.yml' +def _validate_code_coverage( + app: Application, + sync: bool, + validation_tracker: Any, + repo_choice: str, +) -> None: + import yaml + + config_filename = 'code-coverage.datadog.yml' + config_path = app.repo.path / config_filename - path_split = str(codecov_config_relative_path).split('/') - codecov_config_path = os.path.join(app.repo.path, *path_split) - if not os.path.isfile(codecov_config_path): - error_message = 'Unable to find the Codecov config file' - validation_tracker.error((repo_choice,), message=error_message) + if not config_path.is_file(): + validation_tracker.error( + (repo_choice,), message=f'Unable to find the code coverage config file: {config_filename}' + ) validation_tracker.display() app.abort() - codecov_config = yaml.safe_load(read_file(codecov_config_path)) - projects = codecov_config.setdefault('coverage', {}).setdefault('status', {}).setdefault('project', {}) - defined_checks = set() - success = True - fixed = False - - for project, data in list(projects.items()): - if project == 'default': - continue + config = yaml.safe_load(config_path.read_text()) + if config is None: + config = {} - project_flags = data.get('flags', []) - if len(project_flags) != 1: - success = False - error_message += f'Project `{project}` must have exactly one flag\n' - continue + testable_checks = {integration.name for integration in app.repo.integrations.iter_testable('all')} + excluded_jobs = { + name for name, conf in app.repo.config.get('/overrides/ci', {}).items() if conf.get('exclude', False) + } - check_name = project_flags[0] + expected_checks = set() + for check in testable_checks: + if check not in excluded_jobs and code_coverage_enabled(check, app): + expected_checks.add(check) - if check_name in defined_checks: - success = False - error_message += f'Check `{check_name}` is defined as a flag in more than one project\n' - continue + existing_services = config.get('services') or [] + existing_service_id_list = [s['id'] for s in existing_services if 'id' in s] + existing_service_ids = set(existing_service_id_list) - defined_checks.add(check_name) - # Project names cannot contain spaces, see: - # https://github.com/DataDog/integrations-core/pull/6760#issuecomment-634976885 - if check_name in cached_display_names: - display_name = cached_display_names[check_name].replace(' ', '_') - else: - try: - integration = app.repo.integrations.get(check_name) - except OSError as e: - if str(e).startswith('Integration does not exist: '): - continue + success = True + fixed = False + error_message = '' - raise + duplicate_services = sorted( + service_id for service_id, count in Counter(existing_service_id_list).items() if count > 1 + ) + if duplicate_services: + num_duplicate = len(duplicate_services) + service_label = 'service IDs' if num_duplicate > 1 else 'service ID' + duplicate_service_names = ', '.join(duplicate_services) + message = f'Code coverage config has {num_duplicate} duplicate {service_label}: {duplicate_service_names}\n' - display_name = integration.display_name - display_name = display_name.replace(' ', '_') - cached_display_names[check_name] = display_name + if sync: + fixed = True + deduplicated_services = [] + seen_service_ids = set() + for service in existing_services: + service_id = service.get('id', '') + if service_id in seen_service_ids: + app.display_success(f'Removed duplicate service `{service_id}`\n') + continue - if project != display_name: - message = f'Project `{project}` should be called `{display_name}`\n' + seen_service_ids.add(service_id) + deduplicated_services.append(service) - if sync: - fixed = True - warning_message += message - if display_name not in projects: - projects[display_name] = data - del projects[project] - app.display_success(f'Renamed project to `{display_name}`\n') - else: - success = False - error_message += message + existing_services = deduplicated_services + else: + success = False + error_message += message - # This works because we ensure there is a 1 to 1 correspondence between projects and checks (flags) - excluded_jobs = { - name for name, config in app.repo.config.get('/overrides/ci', {}).items() if config.get('exclude', False) - } - missing_projects = testable_checks - set(defined_checks) - excluded_jobs - - not_agent_checks = set() - for check in set(missing_projects): - if not code_coverage_enabled(check, app): - not_agent_checks.add(check) - missing_projects.discard(check) - - if missing_projects: - num_missing_projects = len(missing_projects) - message = ( - f"Codecov config has {num_missing_projects} missing project{'s' if num_missing_projects > 1 else ''}\n" - ) + stale_services = sorted(existing_service_ids - expected_checks) + if stale_services: + num_stale = len(stale_services) + service_label = 'services' if num_stale > 1 else 'service' + stale_service_names = ', '.join(stale_services) + message = f'Code coverage config has {num_stale} stale {service_label}: {stale_service_names}\n' if sync: fixed = True - warning_message += message - - for missing_check in sorted(missing_projects): - display_name = app.repo.integrations.get(missing_check).display_name - display_name = display_name.replace(' ', '_') - projects[display_name] = {'target': 75, 'flags': [missing_check]} - app.display_success(f'Added project `{display_name}`\n') + existing_services = [s for s in existing_services if s.get('id', '') not in stale_services] + for service_id in stale_services: + app.display_success(f'Removed stale service `{service_id}`\n') else: success = False error_message += message - flags = codecov_config.setdefault('flags', {}) - defined_checks = set() - - for flag, data in list(flags.items()): - defined_checks.add(flag) - - expected_coverage_paths = get_coverage_sources(flag, app) - - configured_coverage_paths = data.get('paths', []) - if configured_coverage_paths != expected_coverage_paths: - message = f'Flag `{flag}` has incorrect coverage source paths\n' - - if sync: - fixed = True - warning_message += message - data['paths'] = expected_coverage_paths - app.display_success(f'Configured coverage paths for flag `{flag}`\n') - else: - success = False - error_message += message - - if not data.get('carryforward'): - message = f'Flag `{flag}` must have carryforward set to true\n' + # Validate existing services have correct paths + for service in existing_services: + service_id = service.get('id', '') + if service_id not in expected_checks: + continue + expected_paths = get_coverage_sources(service_id, app) + configured_paths = service.get('paths', []) + if sorted(configured_paths) != sorted(expected_paths): + message = f'Service `{service_id}` has incorrect coverage source paths\n' if sync: fixed = True - warning_message += message - data['carryforward'] = True - app.display_success(f'Enabled the carryforward feature for flag `{flag}`\n') + service['paths'] = expected_paths + app.display_success(f'Configured coverage paths for service `{service_id}`\n') else: success = False error_message += message - missing_flags = testable_checks - set(defined_checks) - excluded_jobs - for check in set(missing_flags): - if check in not_agent_checks or not code_coverage_enabled(check, app): - missing_flags.discard(check) - - if missing_flags: - num_missing_flags = len(missing_flags) - message = f"Codecov config has {num_missing_flags} missing flag{'s' if num_missing_flags > 1 else ''}\n" + missing_services = sorted(expected_checks - existing_service_ids) + if missing_services: + num_missing = len(missing_services) + message = f"Code coverage config has {num_missing} missing service{'s' if num_missing > 1 else ''}\n" if sync: fixed = True - warning_message += message + for check_name in missing_services: + existing_services.append( + { + 'id': check_name, + 'paths': get_coverage_sources(check_name, app), + } + ) + app.display_success(f'Added service `{check_name}`\n') + else: + success = False + error_message += message - for missing_check in sorted(missing_flags): - flags[missing_check] = {'carryforward': True, 'paths': get_coverage_sources(missing_check, app)} - app.display_success(f'Added flag `{missing_check}`\n') + gates = config.get('gates') or [] + if not gates: + message = 'Code coverage config has no coverage gates\n' + if sync: + fixed = True + gates.append( + { + 'type': 'total_coverage_percentage', + 'config': {'threshold': DEFAULT_COVERAGE_THRESHOLD}, + } + ) + config['gates'] = gates + app.display_success(f'Added default coverage gate with {DEFAULT_COVERAGE_THRESHOLD}% threshold\n') else: success = False error_message += message if not success: - message = 'Try running `ddev validate ci --sync`\n' - app.display_info(message) - validation_tracker.error((codecov_config_path,), message=error_message) - + app.display_info('Try running `ddev validate ci --sync`\n') + validation_tracker.error((str(config_path),), message=error_message) validation_tracker.display() app.abort() elif fixed: - codecov_config['coverage']['status']['project'] = dict(sort_projects(projects)) - codecov_config['flags'] = dict(sorted(flags.items())) - output = yaml.safe_dump(codecov_config, default_flow_style=False, sort_keys=False) - write_file(codecov_config_path, output) - app.display_success(f'Successfully fixed {codecov_config_relative_path}') + config['services'] = sorted(existing_services, key=lambda s: s.get('id', '')) + + output = yaml.safe_dump(config, default_flow_style=False, sort_keys=False) + config_path.write_text(output) + app.display_success(f'Successfully fixed {config_filename}') validation_tracker.success() validation_tracker.display() diff --git a/ddev/src/ddev/utils/github.py b/ddev/src/ddev/utils/github.py index ef314fb7d9fe7..bae40dc9ff23c 100644 --- a/ddev/src/ddev/utils/github.py +++ b/ddev/src/ddev/utils/github.py @@ -6,9 +6,11 @@ import json from functools import cached_property from time import time -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, overload if TYPE_CHECKING: + from typing import Any, Literal + from httpx import Client from ddev.cli.terminal import BorrowedStatus @@ -217,12 +219,48 @@ def get_pull_request_labels(self, pr_number: int) -> list[str] | None: return None return [label['name'] for label in response.json().get('labels', [])] - def dispatch_workflow(self, workflow_id: str, ref: str, inputs: dict[str, Any]) -> None: - """Trigger a workflow_dispatch event.""" - self.__api_post( + @overload + def dispatch_workflow( + self, + workflow_id: str, + ref: str, + inputs: dict[str, Any], + return_run_details: Literal[False] = False, + ) -> None: ... + + @overload + def dispatch_workflow( + self, + workflow_id: str, + ref: str, + inputs: dict[str, Any], + return_run_details: Literal[True], + ) -> dict[str, Any]: ... + + def dispatch_workflow( + self, + workflow_id: str, + ref: str, + inputs: dict[str, Any], + return_run_details: bool = False, + ) -> dict[str, Any] | None: + """Trigger a workflow_dispatch event. + + When ``return_run_details`` is true, request the new run's details from + the API and return the parsed JSON response (``workflow_run_id``, + ``run_url``, ``html_url``). The default keeps the prior fire-and-forget + behavior and returns ``None``. + """ + payload: dict[str, Any] = {'ref': ref, 'inputs': inputs} + if return_run_details: + payload['return_run_details'] = True + response = self.__api_post( self.WORKFLOW_DISPATCH_API.format(repo_id=self.repo_id, workflow_id=workflow_id), - content=json.dumps({'ref': ref, 'inputs': inputs}), + content=json.dumps(payload), ) + if not return_run_details: + return None + return response.json() def get_pull_request_comments(self, pr_number: int) -> list[dict]: response = self.__api_get( diff --git a/ddev/tests/ai/__init__.py b/ddev/tests/ai/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/agent/__init__.py b/ddev/tests/ai/agent/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/agent/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/agent/test_anthropic_client.py b/ddev/tests/ai/agent/test_anthropic_client.py new file mode 100644 index 0000000000000..5ec4892923220 --- /dev/null +++ b/ddev/tests/ai/agent/test_anthropic_client.py @@ -0,0 +1,635 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import anthropic +import pytest + +from ddev.ai.agent.anthropic_client import AnthropicAgent +from ddev.ai.agent.exceptions import AgentAPIError, AgentConnectionError, AgentError, AgentRateLimitError +from ddev.ai.agent.types import StopReason, ToolResultMessage +from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.registry import ToolRegistry + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def make_usage( + input_tokens: int = 10, + output_tokens: int = 20, + cache_read: int | None = None, + cache_creation: int | None = None, +) -> SimpleNamespace: + return SimpleNamespace( + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_read_input_tokens=cache_read, + cache_creation_input_tokens=cache_creation, + ) + + +def make_text_block(text: str) -> anthropic.types.TextBlock: + return anthropic.types.TextBlock(type="text", text=text) + + +def make_tool_use_block( + id: str = "toolu_01", + name: str = "read_file", + input: dict | None = None, +) -> anthropic.types.ToolUseBlock: + return anthropic.types.ToolUseBlock( + type="tool_use", + id=id, + name=name, + input=input or {"path": "/tmp/file.txt"}, + ) + + +def make_response( + stop_reason: str | None, + content: list, + usage: SimpleNamespace | None = None, +) -> SimpleNamespace: + return SimpleNamespace( + stop_reason=stop_reason, + content=content, + usage=usage or make_usage(), + ) + + +FAKE_CONTEXT_WINDOW = 200_000 + + +def make_agent( + tools: ToolRegistry | None = None, + mock_response: SimpleNamespace | None = None, +) -> tuple[AnthropicAgent, AsyncMock]: + client = MagicMock(spec=anthropic.AsyncAnthropic) + client.messages = MagicMock() + client.messages.create = AsyncMock(return_value=mock_response or make_response("end_turn", [])) + client.models = MagicMock() + client.models.retrieve = AsyncMock(return_value=SimpleNamespace(max_input_tokens=FAKE_CONTEXT_WINDOW)) + registry = tools or ToolRegistry([]) + agent = AnthropicAgent( + client=client, + tools=registry, + system_prompt="You are helpful.", + name="test-agent", + ) + return agent, client.messages.create + + +# --------------------------------------------------------------------------- +# end_turn with a single TextBlock +# --------------------------------------------------------------------------- + + +async def test_end_turn_single_text_block() -> None: + content = [make_text_block("Hello!")] + resp = make_response("end_turn", content) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Hi") + + assert result.stop_reason is StopReason.END_TURN + assert result.text == "Hello!" + assert result.tool_calls == [] + assert len(agent.history) == 2 + assert agent.history[0] == {"role": "user", "content": "Hi"} + assert agent.history[1] == {"role": "assistant", "content": content} + + +# --------------------------------------------------------------------------- +# tool_use +# --------------------------------------------------------------------------- + + +async def test_tool_use_single_block() -> None: + block = make_tool_use_block(id="toolu_42", name="read_file", input={"path": "/etc/hosts"}) + resp = make_response("tool_use", [block]) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Read hosts") + + assert result.stop_reason is StopReason.TOOL_USE + assert len(result.tool_calls) == 1 + tc = result.tool_calls[0] + assert tc.id == "toolu_42" + assert tc.name == "read_file" + assert tc.input == {"path": "/etc/hosts"} + + +# --------------------------------------------------------------------------- +# mixed TextBlock + ToolUseBlock +# --------------------------------------------------------------------------- + + +async def test_mixed_text_and_tool_use() -> None: + content = [ + make_text_block("I'll read the file for you."), + make_tool_use_block(id="toolu_01", name="read_file"), + ] + resp = make_response("tool_use", content) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Read a file") + + assert result.text == "I'll read the file for you." + assert len(result.tool_calls) == 1 + + +# --------------------------------------------------------------------------- +# Multiple TextBlocks are concatenated +# --------------------------------------------------------------------------- + + +async def test_multiple_text_blocks_are_concatenated() -> None: + content = [make_text_block("Hello, "), make_text_block("world!")] + resp = make_response("end_turn", content) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Hi") + + assert result.text == "Hello, \nworld!" + + +# --------------------------------------------------------------------------- +# max_tokens is a normal response (not an error) +# --------------------------------------------------------------------------- + + +async def test_max_tokens_is_not_an_error() -> None: + resp = make_response("max_tokens", [make_text_block("Truncated...")]) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Tell me everything") + + assert result.stop_reason is StopReason.MAX_TOKENS + assert len(agent.history) == 2 + + +# --------------------------------------------------------------------------- +# allowed_tools filtering +# --------------------------------------------------------------------------- + + +class FakeTool: + def __init__(self, name: str) -> None: + self._name = name + + @property + def name(self) -> str: + return self._name + + @property + def description(self) -> str: + return "" + + @property + def definition(self) -> dict: + return {"name": self._name, "description": "", "input_schema": {}} + + async def run(self, raw: dict) -> ToolResult: + pass + + +async def test_allowed_tools_filters_to_subset() -> None: + registry = ToolRegistry([FakeTool(n) for n in ["read_file", "grep", "mkdir"]]) + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(tools=registry, mock_response=resp) + + await agent.send("Hi", allowed_tools=["read_file"]) + + sent_names = [t["name"] for t in create_mock.call_args.kwargs["tools"]] + assert sent_names == ["read_file"] + + +async def test_allowed_tools_none_passes_all() -> None: + registry = ToolRegistry([FakeTool(n) for n in ["a", "b"]]) + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(tools=registry, mock_response=resp) + + await agent.send("Hi", allowed_tools=None) + + sent_names = [t["name"] for t in create_mock.call_args.kwargs["tools"]] + assert sent_names == ["a", "b"] + + +@pytest.mark.parametrize("allowed_tools", [[], ["nonexistent_tool"]]) +async def test_allowed_tools_passes_not_given(allowed_tools: list[str]) -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(mock_response=resp) + + await agent.send("Hi", allowed_tools=allowed_tools) + + assert create_mock.call_args.kwargs["tools"] is anthropic.NOT_GIVEN + + +# --------------------------------------------------------------------------- +# API errors map to the correct AgentError subclass +# --------------------------------------------------------------------------- + + +def _make_error_agent(side_effect: Exception) -> AnthropicAgent: + client = MagicMock(spec=anthropic.AsyncAnthropic) + client.messages = MagicMock() + client.messages.create = AsyncMock(side_effect=side_effect) + return AnthropicAgent(client=client, tools=ToolRegistry([]), system_prompt="", name="t") + + +async def test_connection_error_maps_to_agent_connection_error() -> None: + agent = _make_error_agent(anthropic.APIConnectionError(request=MagicMock())) + + with pytest.raises(AgentConnectionError) as exc_info: + await agent.send("Hi") + + assert "Connection failed" in str(exc_info.value) + assert agent.history == [] + + +async def test_rate_limit_error_maps_to_agent_rate_limit_error() -> None: + agent = _make_error_agent( + anthropic.RateLimitError( + message="rate limit", + response=MagicMock(status_code=429, headers={}), + body=None, + ) + ) + + with pytest.raises(AgentRateLimitError) as exc_info: + await agent.send("Hi") + + assert "Rate limit exceeded" in str(exc_info.value) + assert agent.history == [] + + +async def test_api_status_error_maps_to_agent_api_error() -> None: + agent = _make_error_agent( + anthropic.APIStatusError( + message="internal server error", + response=MagicMock(status_code=500), + body=None, + ) + ) + + with pytest.raises(AgentAPIError) as exc_info: + await agent.send("Hi") + + assert exc_info.value.status_code == 500 + assert agent.history == [] + + +async def test_response_validation_error_maps_to_agent_error() -> None: + agent = _make_error_agent(anthropic.APIResponseValidationError(response=MagicMock(), body=None)) + + with pytest.raises(AgentError) as exc_info: + await agent.send("Hi") + + assert "Response validation failed" in str(exc_info.value) + assert agent.history == [] + + +# --------------------------------------------------------------------------- +# Unknown stop_reason raises AgentError, history unchanged +# --------------------------------------------------------------------------- + + +async def test_unknown_stop_reason_raises_agent_error() -> None: + resp = make_response("totally_unknown_reason", []) + agent, _ = make_agent(mock_response=resp) + + with pytest.raises(AgentError) as exc_info: + await agent.send("Hi") + + assert agent.history == [] + assert "Unknown stop_reason" in str(exc_info.value) + assert "totally_unknown_reason" in str(exc_info.value) + + +# --------------------------------------------------------------------------- +# cache_read_input_tokens=None defaults to 0 +# --------------------------------------------------------------------------- + + +async def test_cache_tokens_none_defaults_to_zero() -> None: + usage = make_usage(cache_read=None, cache_creation=None) + resp = make_response("end_turn", [make_text_block("ok")], usage=usage) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Hi") + + assert result.usage.cache_read_input_tokens == 0 + assert result.usage.cache_creation_input_tokens == 0 + + +# --------------------------------------------------------------------------- +# ContextUsage fields +# --------------------------------------------------------------------------- + + +async def test_context_usage_fields() -> None: + usage = make_usage(input_tokens=1000, cache_read=500, cache_creation=200) + resp = make_response("end_turn", [make_text_block("ok")], usage=usage) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Hi") + + ctx = result.usage.context_usage + assert ctx.window_size == FAKE_CONTEXT_WINDOW + assert ctx.used_tokens == 1700 # 1000 + 500 + 200 + assert ctx.context_pct == pytest.approx(1700 / FAKE_CONTEXT_WINDOW * 100) + assert ctx.remaining_tokens == FAKE_CONTEXT_WINDOW - 1700 + + +# --------------------------------------------------------------------------- +# context_window is fetched once and cached across multiple sends +# --------------------------------------------------------------------------- + + +async def test_context_window_fetched_once() -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, _ = make_agent(mock_response=resp) + agent._client.messages.create = AsyncMock(return_value=resp) + + await agent.send("First") + await agent.send("Second") + + agent._client.models.retrieve.assert_awaited_once() + + +# --------------------------------------------------------------------------- +# Multi-turn β€” send str then send tool results β†’ history has 4 entries +# --------------------------------------------------------------------------- + + +async def test_multi_turn_history_grows_correctly() -> None: + tool_resp = make_response("tool_use", [make_tool_use_block(id="toolu_01")]) + text_resp = make_response("end_turn", [make_text_block("Done.")]) + + client = MagicMock(spec=anthropic.AsyncAnthropic) + client.messages = MagicMock() + client.messages.create = AsyncMock(side_effect=[tool_resp, text_resp]) + client.models = MagicMock() + client.models.retrieve = AsyncMock(return_value=SimpleNamespace(max_input_tokens=FAKE_CONTEXT_WINDOW)) + agent = AnthropicAgent(client=client, tools=ToolRegistry([]), system_prompt="", name="t") + + first = await agent.send("Do X") + assert first.stop_reason is StopReason.TOOL_USE + assert len(agent.history) == 2 + + tool_results = [ToolResultMessage(tool_call_id="toolu_01", result=ToolResult(success=True, data="result"))] + second = await agent.send(tool_results) + assert second.stop_reason is StopReason.END_TURN + assert len(agent.history) == 4 + assert agent.history[2]["role"] == "user" + assert agent.history[3]["role"] == "assistant" + + +# --------------------------------------------------------------------------- +# history property returns a copy +# --------------------------------------------------------------------------- + + +async def test_history_property_returns_copy() -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, _ = make_agent(mock_response=resp) + await agent.send("Hi") + + snapshot = agent.history + snapshot.clear() + + assert len(agent.history) == 2 + + +# --------------------------------------------------------------------------- +# reset() clears history +# --------------------------------------------------------------------------- + + +async def test_reset_clears_history() -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, _ = make_agent(mock_response=resp) + await agent.send("Hi") + assert len(agent.history) == 2 + + agent.reset() + assert agent.history == [] + + +# --------------------------------------------------------------------------- +# pause_turn raises AgentError +# --------------------------------------------------------------------------- + + +async def test_pause_turn_raises_agent_error() -> None: + resp = make_response("pause_turn", []) + agent, _ = make_agent(mock_response=resp) + + with pytest.raises(AgentError): + await agent.send("Hi") + + assert agent.history == [] + + +# --------------------------------------------------------------------------- +# refusal and stop_sequence map to StopReason.OTHER +# --------------------------------------------------------------------------- + + +async def test_refusal_maps_to_other() -> None: + resp = make_response("refusal", [make_text_block("I can't do that")]) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Do something bad") + + assert result.stop_reason is StopReason.OTHER + + +async def test_stop_sequence_maps_to_other() -> None: + resp = make_response("stop_sequence", [make_text_block("Stopped.")]) + agent, _ = make_agent(mock_response=resp) + + result = await agent.send("Hi") + + assert result.stop_reason is StopReason.OTHER + + +# --------------------------------------------------------------------------- +# Error mid-conversation leaves history unchanged +# --------------------------------------------------------------------------- + + +async def test_error_mid_conversation_leaves_history_unchanged() -> None: + ok_resp = make_response("end_turn", [make_text_block("ok")]) + client = MagicMock(spec=anthropic.AsyncAnthropic) + client.messages = MagicMock() + client.messages.create = AsyncMock( + side_effect=[ + ok_resp, + anthropic.APIConnectionError(request=MagicMock()), + ] + ) + client.models = MagicMock() + client.models.retrieve = AsyncMock(return_value=SimpleNamespace(max_input_tokens=FAKE_CONTEXT_WINDOW)) + agent = AnthropicAgent(client=client, tools=ToolRegistry([]), system_prompt="", name="t") + + await agent.send("First message") + history_after_first = agent.history[:] + + with pytest.raises(AgentConnectionError): + await agent.send("Second message") + + assert agent.history == history_after_first + + +# --------------------------------------------------------------------------- +# Prompt caching: static breakpoints (system + last tool, 1h TTL) +# --------------------------------------------------------------------------- + + +async def test_system_prompt_sent_as_block_with_static_cache_control() -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(mock_response=resp) + + await agent.send("Hi") + + assert create_mock.call_args.kwargs["system"] == [ + { + "type": "text", + "text": "You are helpful.", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ] + + +@pytest.mark.parametrize( + "tool_names", + [["only"], ["a", "b"], ["a", "b", "c", "d"]], + ids=["single_tool", "two_tools", "four_tools"], +) +async def test_only_last_tool_carries_static_cache_control(tool_names: list[str]) -> None: + registry = ToolRegistry([FakeTool(n) for n in tool_names]) + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(tools=registry, mock_response=resp) + + await agent.send("Hi") + + sent_tools = create_mock.call_args.kwargs["tools"] + assert all("cache_control" not in t for t in sent_tools[:-1]) + assert sent_tools[-1]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + + +async def test_allowed_tools_subset_places_cache_control_on_last_of_subset() -> None: + registry = ToolRegistry([FakeTool(n) for n in ["a", "b", "c"]]) + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(tools=registry, mock_response=resp) + + await agent.send("Hi", allowed_tools=["a", "b"]) + + sent_tools = create_mock.call_args.kwargs["tools"] + assert [t["name"] for t in sent_tools] == ["a", "b"] + assert "cache_control" not in sent_tools[0] + assert sent_tools[-1]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + + +# --------------------------------------------------------------------------- +# Prompt caching: sliding breakpoint on the last user message block (default TTL) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "content", + [ + pytest.param("Hi there", id="str"), + pytest.param( + [ToolResultMessage(tool_call_id="t1", result=ToolResult(success=True, data="r1"))], + id="single_tool_result", + ), + pytest.param( + [ToolResultMessage(tool_call_id=f"t{i}", result=ToolResult(success=True, data=f"r{i}")) for i in range(3)], + id="multiple_tool_results", + ), + ], +) +async def test_sliding_cache_control_on_last_user_block_only( + content: str | list[ToolResultMessage], +) -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(mock_response=resp) + + await agent.send(content) + + blocks = create_mock.call_args.kwargs["messages"][-1]["content"] + assert isinstance(blocks, list) and blocks + assert all("cache_control" not in b for b in blocks[:-1]) + assert blocks[-1]["cache_control"] == {"type": "ephemeral"} + assert "ttl" not in blocks[-1]["cache_control"] + + +async def test_empty_tool_results_produces_empty_content_block() -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, create_mock = make_agent(mock_response=resp) + + await agent.send([]) + + blocks = create_mock.call_args.kwargs["messages"][-1]["content"] + assert blocks == [] + + +# --------------------------------------------------------------------------- +# Prompt caching: history must not retain cache_control markers +# (otherwise multi-turn requests would exceed the 4-marker limit) +# --------------------------------------------------------------------------- + + +async def test_history_str_message_keeps_raw_str_form() -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, _ = make_agent(mock_response=resp) + + await agent.send("Hi") + + assert agent.history[0] == {"role": "user", "content": "Hi"} + + +async def test_history_tool_result_blocks_have_no_cache_control() -> None: + resp = make_response("end_turn", [make_text_block("ok")]) + agent, _ = make_agent(mock_response=resp) + + tool_results = [ + ToolResultMessage(tool_call_id=f"t{i}", result=ToolResult(success=True, data=f"r{i}")) for i in range(2) + ] + await agent.send(tool_results) + + blocks = agent.history[0]["content"] + assert all("cache_control" not in b for b in blocks) + + +async def test_multi_turn_only_latest_user_message_in_request_has_cache_control() -> None: + first_resp = make_response("tool_use", [make_tool_use_block(id="t1")]) + second_resp = make_response("end_turn", [make_text_block("done")]) + + client = MagicMock(spec=anthropic.AsyncAnthropic) + client.messages = MagicMock() + client.messages.create = AsyncMock(side_effect=[first_resp, second_resp]) + client.models = MagicMock() + client.models.retrieve = AsyncMock(return_value=SimpleNamespace(max_input_tokens=FAKE_CONTEXT_WINDOW)) + agent = AnthropicAgent(client=client, tools=ToolRegistry([]), system_prompt="sp", name="t") + + await agent.send("First") + await agent.send([ToolResultMessage(tool_call_id="t1", result=ToolResult(success=True, data="r"))]) + + first_call_messages = client.messages.create.call_args_list[0].kwargs["messages"] + assert first_call_messages[-1]["content"] == [ + {"type": "text", "text": "First", "cache_control": {"type": "ephemeral"}} + ] + + second_call_messages = client.messages.create.call_args_list[1].kwargs["messages"] + assert second_call_messages[0] == {"role": "user", "content": "First"} + latest_blocks = second_call_messages[-1]["content"] + assert all("cache_control" not in b for b in latest_blocks[:-1]) + assert latest_blocks[-1]["cache_control"] == {"type": "ephemeral"} diff --git a/ddev/tests/ai/agent/test_base.py b/ddev/tests/ai/agent/test_base.py new file mode 100644 index 0000000000000..c5fd73a10a4b9 --- /dev/null +++ b/ddev/tests/ai/agent/test_base.py @@ -0,0 +1,236 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import pytest + +from ddev.ai.agent.base import _COMPACT_SYSTEM_PROMPT, BaseAgent +from ddev.ai.agent.types import AgentResponse, StopReason, TokenUsage, ToolResultMessage +from ddev.ai.tools.registry import ToolRegistry + +_AGENT_NAME: str = "test" +_AGENT_SYSTEM_PROMPT: str = "original" + +# --------------------------------------------------------------------------- +# Minimal concrete agent for testing BaseAgent +# --------------------------------------------------------------------------- + + +class ConcreteAgent(BaseAgent[dict]): + """Minimal BaseAgent subclass that records send() calls and replays configured responses.""" + + def __init__(self, responses: list[str | Exception] | None = None) -> None: + super().__init__(name=_AGENT_NAME, system_prompt=_AGENT_SYSTEM_PROMPT, tools=ToolRegistry([])) + self._responses = list(responses or []) + self._idx = 0 + self.send_calls: list[dict] = [] + + async def send( + self, + content: str | list[ToolResultMessage], + allowed_tools: list[str] | None = None, + ) -> AgentResponse: + self.send_calls.append( + {"content": content, "allowed_tools": allowed_tools, "system_prompt": self._system_prompt} + ) + if self._idx < len(self._responses): + resp = self._responses[self._idx] + self._idx += 1 + if isinstance(resp, Exception): + raise resp + text = resp + else: + text = "default summary" + self._idx += 1 + + # Simulate what a real provider does: append user + assistant messages. + self._history.extend( + [ + {"role": "user", "content": content}, + {"role": "assistant", "content": text}, + ] + ) + return AgentResponse( + stop_reason=StopReason.END_TURN, + text=text, + tool_calls=[], + usage=TokenUsage(input_tokens=0, output_tokens=0, cache_read_input_tokens=0, cache_creation_input_tokens=0), + ) + + +def make_history(n_messages: int) -> list[dict]: + """Build n_messages alternating user/assistant dicts for seeding _history directly.""" + return [{"role": "user" if i % 2 == 0 else "assistant", "content": f"msg-{i}"} for i in range(n_messages)] + + +# --------------------------------------------------------------------------- +# compact() β€” history length after compact (guard cases + collapse) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "n_messages, responses, expected_len", + [ + (0, None, 0), + (1, None, 1), + (2, None, 2), + (4, ["summary"], 2), + ], +) +async def test_compact_history_length( + n_messages: int, + responses: list[str] | None, + expected_len: int, +) -> None: + agent = ConcreteAgent(responses=responses) + agent._history = make_history(n_messages) + await agent.compact() + assert len(agent.history) == expected_len + + +async def test_compact_first_message_is_original_task() -> None: + agent = ConcreteAgent(responses=["summary"]) + original = make_history(4)[0] + agent._history = make_history(4) + await agent.compact() + assert agent.history[0] == original + + +async def test_compact_second_message_is_summary_response() -> None: + agent = ConcreteAgent(responses=["the summary text"]) + agent._history = make_history(4) + await agent.compact() + summary_msg = agent.history[1] + assert summary_msg["role"] == "assistant" + assert summary_msg["content"] == "the summary text" + + +# --------------------------------------------------------------------------- +# compact() β€” system prompt swap +# --------------------------------------------------------------------------- + + +async def test_compact_uses_compaction_system_prompt() -> None: + agent = ConcreteAgent(responses=["summary"]) + agent._history = make_history(4) + await agent.compact() + assert agent.send_calls[0]["system_prompt"] == _COMPACT_SYSTEM_PROMPT + + +async def test_compact_restores_original_system_prompt() -> None: + agent = ConcreteAgent(responses=["summary"]) + agent._history = make_history(4) + await agent.compact() + assert agent._system_prompt == _AGENT_SYSTEM_PROMPT + + +async def test_compact_restores_system_prompt_on_send_error() -> None: + agent = ConcreteAgent(responses=[RuntimeError("api failure")]) + agent._history = make_history(4) + with pytest.raises(RuntimeError): + await agent.compact() + assert agent._system_prompt == _AGENT_SYSTEM_PROMPT + + +# --------------------------------------------------------------------------- +# compact() β€” send() called with no tools +# --------------------------------------------------------------------------- + + +async def test_compact_send_uses_no_tools() -> None: + agent = ConcreteAgent(responses=["summary"]) + agent._history = make_history(4) + await agent.compact() + assert agent.send_calls[0]["allowed_tools"] == [] + + +# --------------------------------------------------------------------------- +# compact() β€” error leaves history unchanged +# --------------------------------------------------------------------------- + + +async def test_compact_leaves_history_unchanged_on_send_error() -> None: + agent = ConcreteAgent(responses=[RuntimeError("api failure")]) + original_history = make_history(4) + agent._history = list(original_history) + with pytest.raises(RuntimeError): + await agent.compact() + assert agent._history == original_history + + +# --------------------------------------------------------------------------- +# compact() β€” return value +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("n_messages", [0, 1, 2]) +async def test_compact_returns_none_when_history_too_short(n_messages: int) -> None: + agent = ConcreteAgent() + agent._history = make_history(n_messages) + result = await agent.compact() + assert result is None + + +async def test_compact_returns_response_when_compaction_occurs() -> None: + agent = ConcreteAgent(responses=["the summary"]) + agent._history = make_history(4) + result = await agent.compact() + assert result is not None + assert result.text == "the summary" + + +# --------------------------------------------------------------------------- +# compact_preserving_last_turn() β€” guard: history too short +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("n_messages", [0, 1, 2, 3]) +async def test_compact_preserving_last_turn_does_nothing_when_history_is_short(n_messages: int) -> None: + agent = ConcreteAgent() + agent._history = make_history(n_messages) + await agent.compact_preserving_last_turn() + assert len(agent.send_calls) == 0 + assert len(agent._history) == n_messages + + +# --------------------------------------------------------------------------- +# compact_preserving_last_turn() β€” collapses middle, keeps last two +# --------------------------------------------------------------------------- + + +async def test_compact_preserving_last_turn_keeps_last_two_messages() -> None: + agent = ConcreteAgent(responses=["summary"]) + agent._history = make_history(6) + last_two = agent._history[-2:] + await agent.compact_preserving_last_turn() + assert agent.history[-2:] == last_two + + +async def test_compact_preserving_last_turn_first_message_is_original_task() -> None: + agent = ConcreteAgent(responses=["summary"]) + original = make_history(6)[0] + agent._history = make_history(6) + await agent.compact_preserving_last_turn() + assert agent.history[0] == original + + +async def test_compact_preserving_last_turn_produces_four_messages() -> None: + agent = ConcreteAgent(responses=["summary"]) + agent._history = make_history(6) + await agent.compact_preserving_last_turn() + # original + summary + last user + last assistant + assert len(agent.history) == 4 + + +# --------------------------------------------------------------------------- +# compact_preserving_last_turn() β€” error leaves history unchanged +# --------------------------------------------------------------------------- + + +async def test_compact_preserving_last_turn_leaves_history_unchanged_on_error() -> None: + agent = ConcreteAgent(responses=[RuntimeError("api failure")]) + original_history = make_history(6) + agent._history = list(original_history) + with pytest.raises(RuntimeError): + await agent.compact_preserving_last_turn() + assert agent._history == original_history diff --git a/ddev/tests/ai/agent/test_build.py b/ddev/tests/ai/agent/test_build.py new file mode 100644 index 0000000000000..2ee6a42762bf6 --- /dev/null +++ b/ddev/tests/ai/agent/test_build.py @@ -0,0 +1,130 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from unittest.mock import MagicMock + +import pytest + +from ddev.ai.agent.anthropic_client import AnthropicAgent +from ddev.ai.agent.build import build_agent, build_subagent, make_agent_builder, make_subagent_builder +from ddev.ai.phases.config import AgentConfig +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.registry import ToolRegistry + + +@pytest.fixture +def policy(tmp_path) -> FileAccessPolicy: + return FileAccessPolicy(write_root=tmp_path) + + +@pytest.fixture +def file_registry(policy) -> FileRegistry: + return FileRegistry(policy=policy) + + +@pytest.fixture +def clients() -> dict: + return {"anthropic": MagicMock()} + + +# --------------------------------------------------------------------------- +# Core builder behaviour +# --------------------------------------------------------------------------- + + +def test_unknown_provider_raises(file_registry, clients): + config = AgentConfig.model_construct(provider="bad_provider", tools=[]) + with pytest.raises(ValueError, match="Unknown agent provider: 'bad_provider'"): + build_agent(config, clients, "sys", "p1", file_registry) + + +def test_missing_client_raises(file_registry): + config = AgentConfig(provider="anthropic", tools=[]) + with pytest.raises(ValueError, match="No client provided for agent provider 'anthropic'"): + build_agent(config, {}, "sys", "p1", file_registry) + + +def test_builds_anthropic_agent_with_correct_types_and_name(file_registry, clients): + config = AgentConfig(provider="anthropic", tools=[]) + agent, registry = build_agent(config, clients, "sys", "p1", file_registry) + assert isinstance(agent, AnthropicAgent) + assert isinstance(registry, ToolRegistry) + assert agent.name == "p1" + + +@pytest.mark.parametrize( + "model,max_tokens", + [ + ("claude-opus-4-7", 2048), + ("claude-haiku-4-5", 512), + ], +) +def test_model_and_max_tokens_forwarded(file_registry, clients, model, max_tokens): + config = AgentConfig(provider="anthropic", model=model, max_tokens=max_tokens, tools=[]) + agent, _ = build_agent(config, clients, "sys", "p1", file_registry) + assert agent._model == model + assert agent._max_tokens == max_tokens + + +def test_build_agent_uses_config_tools(file_registry, clients): + config = AgentConfig(provider="anthropic", tools=["read_file"]) + _, registry = build_agent(config, clients, "sys", "p1", file_registry) + assert len(registry.definitions) == 1 + assert registry.definitions[0]["name"] == "read_file" + + +# --------------------------------------------------------------------------- +# build_subagent +# --------------------------------------------------------------------------- + + +def test_build_subagent_reuses_shared_file_registry(file_registry, clients): + config = AgentConfig(provider="anthropic", tools=[]) + _, registry = build_subagent(config, clients, file_registry, "sys", "child", ["read_file", "edit_file"]) + + for tool in registry._tools.values(): + assert tool._registry is file_registry + assert tool._owner_id == "child" + + +def test_build_subagent_recursion_guard(file_registry, clients): + config = AgentConfig.model_construct(provider="anthropic", tools=[]) + with pytest.raises(ValueError): + build_subagent(config, clients, file_registry, "sys", "sub", ["spawn_subagent"]) + + +async def test_shared_registry_does_not_share_parent_read_authorization(file_registry, clients, tmp_path): + config = AgentConfig(provider="anthropic", tools=[]) + path = tmp_path / "file.txt" + path.write_text("before", encoding="utf-8") + file_registry.record("parent", str(path), "before") + + _, registry = build_subagent(config, clients, file_registry, "sys", "parent.sub.001-child", ["edit_file"]) + result = await registry.run("edit_file", {"path": str(path), "old_string": "before", "new_string": "after"}) + + assert result.success is False + assert "Not authorized" in result.error + assert path.read_text(encoding="utf-8") == "before" + + +# --------------------------------------------------------------------------- +# Closures β€” verify delegation works and signatures are correct +# --------------------------------------------------------------------------- + + +def test_make_agent_builder(file_registry, clients): + config = AgentConfig(provider="anthropic", tools=[]) + builder = make_agent_builder(config, clients, file_registry) + agent, registry = builder("sys", "p1", None, None) + assert isinstance(agent, AnthropicAgent) + assert agent.name == "p1" + + +def test_make_subagent_builder(file_registry, clients): + config = AgentConfig(provider="anthropic", tools=[]) + builder = make_subagent_builder(config, clients, file_registry) + agent, registry = builder("sys", "sub-1", []) + assert isinstance(agent, AnthropicAgent) + assert agent.name == "sub-1" diff --git a/ddev/tests/ai/callbacks/__init__.py b/ddev/tests/ai/callbacks/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/callbacks/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/callbacks/test_callbacks.py b/ddev/tests/ai/callbacks/test_callbacks.py new file mode 100644 index 0000000000000..79a4ae151fb8e --- /dev/null +++ b/ddev/tests/ai/callbacks/test_callbacks.py @@ -0,0 +1,494 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import pytest + +from ddev.ai.agent.types import AgentResponse, StopReason, TokenUsage, ToolCall +from ddev.ai.callbacks.callbacks import Callbacks, CallbackSet +from ddev.ai.react.types import ReActResult +from ddev.ai.tools.core.types import ToolResult + +# --------------------------------------------------------------------------- +# Minimal fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def response() -> AgentResponse: + return AgentResponse( + stop_reason=StopReason.END_TURN, + text="", + tool_calls=[], + usage=TokenUsage(input_tokens=10, output_tokens=5, cache_read_input_tokens=0, cache_creation_input_tokens=0), + ) + + +@pytest.fixture +def tool_call() -> ToolCall: + return ToolCall(id="tc_01", name="read_file", input={}) + + +@pytest.fixture +def react_result(response: AgentResponse) -> ReActResult: + return ReActResult( + final_response=response, + iterations=1, + total_input_tokens=10, + total_output_tokens=5, + context_usage=None, + ) + + +# --------------------------------------------------------------------------- +# Registration via decorators +# --------------------------------------------------------------------------- + + +async def test_decorator_returns_original_function() -> None: + cb = CallbackSet() + + async def handler(response: AgentResponse, iteration: int) -> None: + pass + + assert cb.on_agent_response(handler) is handler + + +async def test_decorator_registers_handler_in_internal_list() -> None: + cb = CallbackSet() + + @cb.on_agent_response + async def h1(response: AgentResponse, iteration: int) -> None: ... + + @cb.on_agent_response + async def h2(response: AgentResponse, iteration: int) -> None: ... + + assert cb._on_agent_response == [h1, h2] + + +# --------------------------------------------------------------------------- +# Dispatch ordering and isolation +# --------------------------------------------------------------------------- + + +async def test_empty_callback_set_is_noop( + response: AgentResponse, tool_call: ToolCall, react_result: ReActResult +) -> None: + cb = CallbackSet() + await cb.fire_agent_response(response, 1) + await cb.fire_tool_call(tool_call, ToolResult(success=True, data="ok"), 1) + await cb.fire_complete(react_result) + await cb.fire_error(RuntimeError("boom")) + + +async def test_multiple_handlers_same_event_all_fire(response: AgentResponse) -> None: + cb = CallbackSet() + fired: list[int] = [] + + @cb.on_agent_response + async def first(response: AgentResponse, iteration: int) -> None: + fired.append(1) + + @cb.on_agent_response + async def second(response: AgentResponse, iteration: int) -> None: + fired.append(2) + + @cb.on_agent_response + async def third(response: AgentResponse, iteration: int) -> None: + fired.append(3) + + await cb.fire_agent_response(response, 5) + + assert fired == [1, 2, 3] + + +async def test_handlers_receive_correct_arguments(response: AgentResponse) -> None: + cb = CallbackSet() + received: list[tuple] = [] + + @cb.on_agent_response + async def h(response: AgentResponse, iteration: int) -> None: + received.append((response, iteration)) + + await cb.fire_agent_response(response, 7) + + assert received == [(response, 7)] + + +# --------------------------------------------------------------------------- +# Exception-swallowing guarantee +# --------------------------------------------------------------------------- + + +async def test_fire_swallows_handler_exception(response: AgentResponse) -> None: + cb = CallbackSet() + fired: list[int] = [] + + @cb.on_agent_response + async def bad(response: AgentResponse, iteration: int) -> None: + raise RuntimeError("boom") + + @cb.on_agent_response + async def good(response: AgentResponse, iteration: int) -> None: + fired.append(iteration) + + await cb.fire_agent_response(response, 1) + assert fired == [1] + + +async def test_fire_tool_call_swallows_handler_exception(tool_call: ToolCall) -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_tool_call + async def bad(tool_call: ToolCall, result: ToolResult, iteration: int) -> None: + raise RuntimeError("boom") + + @cb.on_tool_call + async def good(tool_call: ToolCall, result: ToolResult, iteration: int) -> None: + fired.append(True) + + await cb.fire_tool_call(tool_call, ToolResult(success=True, data="ok"), 1) + assert fired == [True] + + +async def test_fire_complete_swallows_handler_exception(react_result: ReActResult) -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_complete + async def bad(result: ReActResult) -> None: + raise RuntimeError("boom") + + @cb.on_complete + async def good(result: ReActResult) -> None: + fired.append(True) + + await cb.fire_complete(react_result) + assert fired == [True] + + +async def test_fire_error_swallows_handler_exception() -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_error + async def bad(error: BaseException) -> None: + raise RuntimeError("boom") + + @cb.on_error + async def good(error: BaseException) -> None: + fired.append(True) + + await cb.fire_error(ValueError("original error")) + assert fired == [True] + + +# --------------------------------------------------------------------------- +# before_compact and after_compact +# --------------------------------------------------------------------------- + + +async def test_before_compact_registered_and_fired() -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_before_compact + async def h() -> None: + fired.append(True) + + await cb.fire_before_compact() + assert fired == [True] + + +async def test_after_compact_registered_and_fired() -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_after_compact + async def h() -> None: + fired.append(True) + + await cb.fire_after_compact() + assert fired == [True] + + +async def test_compact_callback_exception_is_swallowed() -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_before_compact + async def bad() -> None: + raise RuntimeError("boom") + + @cb.on_before_compact + async def good() -> None: + fired.append(True) + + await cb.fire_before_compact() + assert fired == [True] + + +async def test_multiple_compact_handlers_all_fired() -> None: + cb = CallbackSet() + fired: list[str] = [] + + @cb.on_before_compact + async def b1() -> None: + fired.append("before-1") + + @cb.on_before_compact + async def b2() -> None: + fired.append("before-2") + + @cb.on_after_compact + async def a1() -> None: + fired.append("after-1") + + @cb.on_after_compact + async def a2() -> None: + fired.append("after-2") + + await cb.fire_before_compact() + await cb.fire_after_compact() + assert fired == ["before-1", "before-2", "after-1", "after-2"] + + +# --------------------------------------------------------------------------- +# on_phase_start +# --------------------------------------------------------------------------- + + +async def test_phase_start_registered_and_fired() -> None: + cb = CallbackSet() + fired: list[str] = [] + + @cb.on_phase_start + async def h(phase_id: str) -> None: + fired.append(phase_id) + + await cb.fire_phase_start("my-phase") + assert fired == ["my-phase"] + + +async def test_phase_start_receives_correct_phase_id() -> None: + cb = CallbackSet() + received: list[str] = [] + + @cb.on_phase_start + async def h(phase_id: str) -> None: + received.append(phase_id) + + await cb.fire_phase_start("draft") + assert received == ["draft"] + + +async def test_phase_start_multiple_handlers_all_fire_in_order() -> None: + cb = CallbackSet() + fired: list[int] = [] + + @cb.on_phase_start + async def first(phase_id: str) -> None: + fired.append(1) + + @cb.on_phase_start + async def second(phase_id: str) -> None: + fired.append(2) + + @cb.on_phase_start + async def third(phase_id: str) -> None: + fired.append(3) + + await cb.fire_phase_start("p") + assert fired == [1, 2, 3] + + +async def test_phase_start_exception_is_swallowed() -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_phase_start + async def bad(phase_id: str) -> None: + raise RuntimeError("boom") + + @cb.on_phase_start + async def good(phase_id: str) -> None: + fired.append(True) + + await cb.fire_phase_start("p") + assert fired == [True] + + +# --------------------------------------------------------------------------- +# on_phase_finish +# --------------------------------------------------------------------------- + + +async def test_phase_finish_registered_and_fired() -> None: + cb = CallbackSet() + fired: list[str] = [] + + @cb.on_phase_finish + async def h(phase_id: str) -> None: + fired.append(phase_id) + + await cb.fire_phase_finish("my-phase") + assert fired == ["my-phase"] + + +async def test_phase_finish_receives_correct_phase_id() -> None: + cb = CallbackSet() + received: list[str] = [] + + @cb.on_phase_finish + async def h(phase_id: str) -> None: + received.append(phase_id) + + await cb.fire_phase_finish("write-code") + assert received == ["write-code"] + + +async def test_phase_finish_multiple_handlers_all_fire_in_order() -> None: + cb = CallbackSet() + fired: list[int] = [] + + @cb.on_phase_finish + async def first(phase_id: str) -> None: + fired.append(1) + + @cb.on_phase_finish + async def second(phase_id: str) -> None: + fired.append(2) + + @cb.on_phase_finish + async def third(phase_id: str) -> None: + fired.append(3) + + await cb.fire_phase_finish("p") + assert fired == [1, 2, 3] + + +async def test_phase_finish_exception_is_swallowed() -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_phase_finish + async def bad(phase_id: str) -> None: + raise RuntimeError("boom") + + @cb.on_phase_finish + async def good(phase_id: str) -> None: + fired.append(True) + + await cb.fire_phase_finish("p") + assert fired == [True] + + +# --------------------------------------------------------------------------- +# on_before_agent_send +# --------------------------------------------------------------------------- + + +async def test_before_agent_send_registered_and_fired() -> None: + cb = CallbackSet() + fired: list[int] = [] + + @cb.on_before_agent_send + async def h(iteration: int) -> None: + fired.append(iteration) + + await cb.fire_before_agent_send(3) + assert fired == [3] + + +async def test_before_agent_send_receives_correct_iteration() -> None: + cb = CallbackSet() + received: list[int] = [] + + @cb.on_before_agent_send + async def h(iteration: int) -> None: + received.append(iteration) + + await cb.fire_before_agent_send(7) + assert received == [7] + + +async def test_before_agent_send_multiple_handlers_all_fire_in_order() -> None: + cb = CallbackSet() + fired: list[int] = [] + + @cb.on_before_agent_send + async def first(iteration: int) -> None: + fired.append(1) + + @cb.on_before_agent_send + async def second(iteration: int) -> None: + fired.append(2) + + @cb.on_before_agent_send + async def third(iteration: int) -> None: + fired.append(3) + + await cb.fire_before_agent_send(1) + assert fired == [1, 2, 3] + + +async def test_before_agent_send_exception_is_swallowed() -> None: + cb = CallbackSet() + fired: list[bool] = [] + + @cb.on_before_agent_send + async def bad(iteration: int) -> None: + raise RuntimeError("boom") + + @cb.on_before_agent_send + async def good(iteration: int) -> None: + fired.append(True) + + await cb.fire_before_agent_send(1) + assert fired == [True] + + +# --------------------------------------------------------------------------- +# Callbacks container +# --------------------------------------------------------------------------- + + +async def test_callbacks_empty_is_noop(response: AgentResponse, tool_call: ToolCall, react_result: ReActResult) -> None: + callbacks = Callbacks() + await callbacks.fire_agent_response(response, 1) + await callbacks.fire_tool_call(tool_call, ToolResult(success=True, data="ok"), 1) + await callbacks.fire_complete(react_result) + await callbacks.fire_error(RuntimeError("boom")) + await callbacks.fire_before_compact() + await callbacks.fire_after_compact() + await callbacks.fire_before_agent_send(1) + await callbacks.fire_phase_start("p") + await callbacks.fire_phase_finish("p") + + +async def test_callbacks_dispatches_to_all_sets(response: AgentResponse) -> None: + fired_a: list[int] = [] + fired_b: list[int] = [] + + set_a = CallbackSet() + set_b = CallbackSet() + + @set_a.on_agent_response + async def a(response: AgentResponse, iteration: int) -> None: + fired_a.append(iteration) + + @set_b.on_agent_response + async def b(response: AgentResponse, iteration: int) -> None: + fired_b.append(iteration) + + callbacks = Callbacks([set_a, set_b]) + await callbacks.fire_agent_response(response, 3) + + assert fired_a == [3] + assert fired_b == [3] + + +async def test_callbacks_set_with_no_registered_handlers_is_noop(response: AgentResponse) -> None: + callbacks = Callbacks([CallbackSet()]) + await callbacks.fire_agent_response(response, 1) diff --git a/ddev/tests/ai/phases/__init__.py b/ddev/tests/ai/phases/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/phases/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/phases/conftest.py b/ddev/tests/ai/phases/conftest.py new file mode 100644 index 0000000000000..96149455e1385 --- /dev/null +++ b/ddev/tests/ai/phases/conftest.py @@ -0,0 +1,179 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import asyncio +from typing import Any + +import pytest + +from ddev.ai.agent.types import AgentResponse, ContextUsage, StopReason, TokenUsage, ToolResultMessage +from ddev.ai.phases.agentic_phase import AgenticPhase +from ddev.ai.phases.checkpoint import CheckpointManager +from ddev.ai.phases.config import PhaseConfig, TaskConfig +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.registry import ToolRegistry + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def make_response( + text: str = "", + input_tokens: int = 100, + output_tokens: int = 50, + context_pct: float | None = None, + stop_reason: StopReason = StopReason.END_TURN, +) -> AgentResponse: + context_usage = None + if context_pct is not None: + context_usage = ContextUsage(window_size=100_000, used_tokens=int(100_000 * context_pct / 100)) + return AgentResponse( + stop_reason=stop_reason, + text=text, + tool_calls=[], + usage=TokenUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_read_input_tokens=0, + cache_creation_input_tokens=0, + context_usage=context_usage, + ), + ) + + +# --------------------------------------------------------------------------- +# Mock helpers +# --------------------------------------------------------------------------- + + +class MockAgent: + """Agent mock that replays a fixed list of responses. + + Used via monkeypatch to replace AnthropicAgent in Phase tests. + """ + + def __init__(self, responses: list[AgentResponse]) -> None: + self._responses = list(responses) + self._index = 0 + self.send_calls: list[str | list[ToolResultMessage]] = [] + self.compact_call_count: int = 0 + self.name = "mock" + self._history: list[Any] = [] + + async def send( + self, + content: str | list[ToolResultMessage], + allowed_tools: list[str] | None = None, + ) -> AgentResponse: + self.send_calls.append(content) + response = self._responses[self._index] + self._index += 1 + return response + + def reset(self) -> None: + self._history = [] + + async def compact(self) -> AgentResponse | None: + self.compact_call_count += 1 + return None + + async def compact_preserving_last_turn(self) -> AgentResponse | None: + self.compact_call_count += 1 + return None + + +def make_agent_builder(mock_agent: MockAgent, captured_kwargs: dict[str, Any] | None = None): + """Create an agent_builder that returns the given mock and an empty ToolRegistry. + + If ``captured_kwargs`` is provided, every call records the system_prompt and + owner_id passed in β€” useful for asserting on prompt rendering. + """ + + def builder( + system_prompt: str, owner_id: str, subagent_builder=None, log_dir=None + ) -> tuple[MockAgent, ToolRegistry]: + if captured_kwargs is not None: + captured_kwargs["system_prompt"] = system_prompt + captured_kwargs["owner_id"] = owner_id + mock_agent.name = owner_id + return mock_agent, ToolRegistry([]) + + return builder + + +def make_agent_phase( + flow_dir, + mock_agent: MockAgent, + monkeypatch, + message_queue, + *, + phase_id: str = "p1", + dependencies: list[str] | None = None, + tasks: list[TaskConfig] | None = None, + checkpoint=None, + flow_variables: dict[str, str] | None = None, + runtime_variables: dict[str, str] | None = None, + context_compact_threshold_pct: int = 80, + callbacks=None, + captured_agent_kwargs: dict[str, Any] | None = None, +) -> tuple[AgenticPhase, CheckpointManager]: + """Build an AgenticPhase ready for process_message-driven tests. + + Injects a mock agent_builder so no real LLM or tools are constructed. Pass + ``captured_agent_kwargs`` (a dict) to record the rendered system_prompt and owner_id. + """ + config = PhaseConfig( + agent="writer", + tasks=tasks or [TaskConfig(name="t1", prompt="Do the work.")], + checkpoint=checkpoint, + context_compact_threshold_pct=context_compact_threshold_pct, + ) + checkpoint_manager = CheckpointManager(flow_dir / "checkpoints.yaml") + + phase = AgenticPhase( + phase_id=phase_id, + dependencies=dependencies or [], + config=config, + agent_builder=make_agent_builder(mock_agent, captured_agent_kwargs), + checkpoint_manager=checkpoint_manager, + runtime_variables=runtime_variables or {}, + flow_variables=flow_variables or {}, + config_dir=flow_dir, + file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), + callbacks=callbacks, + ) + phase.queue = message_queue + return phase, checkpoint_manager + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +def resolve_key(key: str) -> str: + """Resolver that wraps a key in 'resolved(...)' for use in template tests.""" + return f"resolved({key})" + + +@pytest.fixture +def flow_dir(tmp_path): + """Create a minimal flow directory with a system prompt.""" + prompts_dir = tmp_path / "prompts" + prompts_dir.mkdir() + (prompts_dir / "writer.md").write_text("You are a writer for ${phase_name}.") + return tmp_path + + +@pytest.fixture +def message_queue(): + """An asyncio.Queue that can be attached to a Phase for submit_message.""" + return asyncio.Queue() + + +@pytest.fixture +def file_access_policy(tmp_path) -> FileAccessPolicy: + return FileAccessPolicy(write_root=tmp_path) diff --git a/ddev/tests/ai/phases/test_agentic_phase.py b/ddev/tests/ai/phases/test_agentic_phase.py new file mode 100644 index 0000000000000..36d4701b5b353 --- /dev/null +++ b/ddev/tests/ai/phases/test_agentic_phase.py @@ -0,0 +1,466 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import json +from pathlib import Path + +import pytest + +from ddev.ai.agent.types import AgentResponse, StopReason, TokenUsage, ToolCall +from ddev.ai.callbacks.callbacks import Callbacks, CallbackSet +from ddev.ai.phases.agentic_phase import AgenticPhase, render_memory_prompt, render_task_prompt +from ddev.ai.phases.checkpoint import CheckpointManager +from ddev.ai.phases.config import AgentConfig, CheckpointConfig, FlowConfigError, PhaseConfig, TaskConfig +from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.registry import ToolRegistry + +from .conftest import MockAgent, make_agent_phase, make_response, resolve_key + + +def read_jsonl(path: Path) -> list[dict]: + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +# --------------------------------------------------------------------------- +# render_task_prompt +# --------------------------------------------------------------------------- + + +def test_render_task_prompt_from_file(tmp_path): + prompt_file = tmp_path / "task.md" + prompt_file.write_text("Hello ${name}.") + result = render_task_prompt(TaskConfig(name="t1", prompt_path="task.md"), tmp_path, {"name": "Alice"}) + assert result == "Hello Alice." + + +def test_render_task_prompt_inline(): + result = render_task_prompt(TaskConfig(name="t1", prompt="Hello ${name}."), None, {"name": "Bob"}) + assert result == "Hello Bob." + + +def test_render_task_prompt_forwards_resolver(tmp_path): + (tmp_path / "task.md").write_text("Memory: ${draft_memory}") + result = render_task_prompt(TaskConfig(name="t1", prompt_path="task.md"), tmp_path, {}, resolve_key) + assert result == "Memory: resolved(draft_memory)" + + +def test_render_task_prompt_raises_when_no_source(): + with pytest.raises(FlowConfigError, match="prompt"): + render_task_prompt(TaskConfig.model_construct(name="t1", prompt=None, prompt_path=None), None, {}) + + +# --------------------------------------------------------------------------- +# render_memory_prompt +# --------------------------------------------------------------------------- + + +def test_render_memory_prompt_from_file(tmp_path): + (tmp_path / "mem.md").write_text("List files for ${phase_name}.") + result = render_memory_prompt(CheckpointConfig(memory_prompt_path="mem.md"), tmp_path, {"phase_name": "draft"}) + assert result == "List files for draft." + + +def test_render_memory_prompt_inline(): + result = render_memory_prompt( + CheckpointConfig(memory_prompt="List files for ${phase_name}."), None, {"phase_name": "draft"} + ) + assert result == "List files for draft." + + +def test_render_memory_prompt_raises_when_no_source(): + with pytest.raises(FlowConfigError, match="memory_prompt"): + render_memory_prompt(CheckpointConfig.model_construct(memory_prompt=None, memory_prompt_path=None), None, {}) + + +# --------------------------------------------------------------------------- +# AgenticPhase.validate_config +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "config,match", + [ + (PhaseConfig(tasks=[TaskConfig(name="t1", prompt="x")]), "requires 'agent'"), + (PhaseConfig(agent="ghost", tasks=[TaskConfig(name="t1", prompt="x")]), "unknown agent"), + (PhaseConfig(agent="writer"), "at least one task"), + ], + ids=["missing_agent", "unknown_agent", "empty_tasks"], +) +def test_validate_config_rejects_invalid(config, match): + with pytest.raises(FlowConfigError, match=match): + AgenticPhase.validate_config("p1", config, {"writer": AgentConfig()}) + + +def test_validate_config_accepts_valid(): + AgenticPhase.validate_config( + "p1", PhaseConfig(agent="writer", tasks=[TaskConfig(name="t1", prompt="x")]), {"writer": AgentConfig()} + ) + + +# --------------------------------------------------------------------------- +# process_message β€” happy path +# --------------------------------------------------------------------------- + + +async def test_happy_path_single_task(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([make_response("task done", 100, 50), make_response("summary", 10, 5)]) + phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.memory_content("p1") == "summary" + checkpoint = mgr.read()["p1"] + assert checkpoint["status"] == "success" + assert checkpoint["tokens"] == {"total_input": 110, "total_output": 55} + assert mock_agent.send_calls[0] == "Do the work." + assert "Write a brief summary" in mock_agent.send_calls[1] + # checkpoint memory_path points to the written file + memory_path = Path(checkpoint["memory_path"]) + assert memory_path.is_absolute() and memory_path.exists() and memory_path.name == "p1_memory.md" + + +async def test_happy_path_two_tasks_accumulates_tokens(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent( + [ + make_response("t1 done", 100, 50), + make_response("t2 done", 200, 80), + make_response("summary", 10, 5), + ] + ) + phase, mgr = make_agent_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + tasks=[TaskConfig(name="t1", prompt="First."), TaskConfig(name="t2", prompt="Second.")], + ) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.read()["p1"]["tokens"] == {"total_input": 310, "total_output": 135} + + +# --------------------------------------------------------------------------- +# process_message β€” context compaction +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("context_pct,expect_compact", [(85, True), (50, False)], ids=["above", "below"]) +async def test_compact_between_tasks(flow_dir, monkeypatch, message_queue, context_pct, expect_compact): + mock_agent = MockAgent( + [ + make_response("t1 done", 100, 50, context_pct=context_pct), + make_response("t2 done", 200, 80), + make_response("summary", 10, 5), + ] + ) + phase, _ = make_agent_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + tasks=[TaskConfig(name="t1", prompt="First."), TaskConfig(name="t2", prompt="Second.")], + ) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert (mock_agent.compact_call_count >= 1) == expect_compact + + +# --------------------------------------------------------------------------- +# process_message β€” before_react / after_react hooks +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("hook_name", ["before_react", "after_react"], ids=["before", "after"]) +async def test_react_hook_failure_fails_phase(flow_dir, monkeypatch, message_queue, hook_name): + mock_agent = MockAgent([make_response("done", 100, 50)]) + phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) + setattr(phase, hook_name, lambda: (_ for _ in ()).throw(RuntimeError("hook failed"))) + + with pytest.raises(RuntimeError, match="hook failed"): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.read() == {} + + +# --------------------------------------------------------------------------- +# process_message β€” template context +# --------------------------------------------------------------------------- + + +async def test_flow_variables_rendered_in_system_prompt(flow_dir, monkeypatch, message_queue): + (flow_dir / "prompts" / "writer.md").write_text("Project: ${project}") + mock_agent = MockAgent([make_response("done", 100, 50), make_response("summary", 10, 5)]) + captured: dict = {} + phase, _ = make_agent_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + flow_variables={"project": "myproj"}, + captured_agent_kwargs=captured, + ) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert captured["system_prompt"] == "Project: myproj" + + +async def test_runtime_variables_override_flow_variables(flow_dir, monkeypatch, message_queue): + (flow_dir / "prompts" / "writer.md").write_text("Project: ${project}") + mock_agent = MockAgent([make_response("done", 100, 50), make_response("summary", 10, 5)]) + captured: dict = {} + phase, _ = make_agent_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + flow_variables={"project": "flow"}, + runtime_variables={"project": "runtime"}, + captured_agent_kwargs=captured, + ) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert captured["system_prompt"] == "Project: runtime" + + +async def test_task_prompt_resolves_memory_variable(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([make_response("done", 100, 50), make_response("summary", 10, 5)]) + phase, mgr = make_agent_phase( + flow_dir, + mock_agent, + monkeypatch, + message_queue, + phase_id="review", + tasks=[TaskConfig(name="t1", prompt="Review: ${draft_memory}")], + ) + mgr.write_memory("draft", "Created file.py") + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mock_agent.send_calls[0] == "Review: Created file.py" + + +# --------------------------------------------------------------------------- +# process_message β€” failure modes +# --------------------------------------------------------------------------- + + +async def test_memory_api_failure_fails_phase(flow_dir, monkeypatch, message_queue): + # Only one response β€” IndexError when memory step tries to call agent again + phase, mgr = make_agent_phase( + flow_dir, MockAgent([make_response("task done", 100, 50)]), monkeypatch, message_queue + ) + + with pytest.raises(IndexError): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.read() == {} + + +async def test_memory_template_render_failure_fails_phase(flow_dir, monkeypatch, message_queue): + phase, mgr = make_agent_phase( + flow_dir, + MockAgent([make_response("task done", 100, 50)]), + monkeypatch, + message_queue, + checkpoint=CheckpointConfig(memory_prompt="Summarize."), + ) + monkeypatch.setattr( + "ddev.ai.phases.agentic_phase.render_memory_prompt", + lambda *a, **kw: (_ for _ in ()).throw(ValueError("bad template")), + ) + + with pytest.raises(ValueError, match="bad template"): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.read() == {} + + +async def test_disk_failure_on_write_memory_fails_phase(flow_dir, monkeypatch, message_queue): + mock_agent = MockAgent([make_response("task done", 100, 50), make_response("summary", 10, 5)]) + phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue) + monkeypatch.setattr( + "ddev.ai.phases.checkpoint.CheckpointManager.write_memory", + lambda *a, **kw: (_ for _ in ()).throw(PermissionError("read-only")), + ) + + with pytest.raises(PermissionError, match="read-only"): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.read() == {} + + +# --------------------------------------------------------------------------- +# AgenticPhase._run_memory_step +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "checkpoint,expected_user_additions", + [(None, None), (CheckpointConfig(memory_prompt="anything"), "USER_ADDITIONS")], + ids=["no_checkpoint", "with_checkpoint"], +) +async def test_run_memory_step_passes_user_additions_to_build( + flow_dir, monkeypatch, message_queue, checkpoint, expected_user_additions +): + mock_agent = MockAgent([make_response("ok", 0, 0)]) + phase, mgr = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue, checkpoint=checkpoint) + monkeypatch.setattr("ddev.ai.phases.agentic_phase.render_memory_prompt", lambda *a, **kw: "USER_ADDITIONS") + build_calls: list = [] + monkeypatch.setattr( + mgr, "build_memory_prompt", lambda user_additions: build_calls.append(user_additions) or "PROMPT" + ) + + await phase._run_memory_step(mock_agent, {}) + + assert build_calls == [expected_user_additions] + + +async def test_run_memory_step_sends_built_prompt_with_no_tools(flow_dir, monkeypatch, message_queue): + captured: dict = {} + + class CapturingAgent(MockAgent): + async def send(self, content, allowed_tools=None): + captured.update({"content": content, "allowed_tools": allowed_tools}) + return await super().send(content, allowed_tools) + + agent = CapturingAgent([make_response("ok", 0, 0)]) + phase, mgr = make_agent_phase(flow_dir, agent, monkeypatch, message_queue) + monkeypatch.setattr(mgr, "build_memory_prompt", lambda _: "BUILT") + + await phase._run_memory_step(agent, {}) + + assert captured == {"content": "BUILT", "allowed_tools": []} + + +async def test_run_memory_step_returns_response_data_and_fires_callbacks(flow_dir, monkeypatch, message_queue): + events: list = [] + cb_set = CallbackSet() + + @cb_set.on_before_agent_send + async def _before(iteration): + events.append(("before", iteration)) + + @cb_set.on_agent_response + async def _response(response, iteration): + events.append(("response", iteration, response.text)) + + mock_agent = MockAgent([make_response("summary text", 7, 3)]) + phase, _ = make_agent_phase(flow_dir, mock_agent, monkeypatch, message_queue, callbacks=Callbacks([cb_set])) + + result = await phase._run_memory_step(mock_agent, {}) + + assert result == ("summary text", 7, 3) + assert events == [("before", 1), ("response", 1, "summary text")] + + +# --------------------------------------------------------------------------- +# AgenticPhase with spawn_subagent β€” wiring smoke test +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("tools", "expected"), + [(["spawn_subagent"], True), (["read_file"], False), ([], False)], + ids=["spawn", "regular_tool", "no_tools"], +) +def test_extra_init_kwargs_creates_subagent_builder_from_tool_metadata( + flow_dir: Path, + tools: list[str], + expected: bool, +) -> None: + kwargs = AgenticPhase.extra_init_kwargs( + phase_id="p1", + phase_config=PhaseConfig(agent="writer", tasks=[TaskConfig(name="t1", prompt="Do the work.")]), + agents={"writer": AgentConfig(tools=tools)}, + agent_clients={}, + file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), + ) + + assert (kwargs["subagent_builder"] is not None) is expected + + +async def test_spawn_subagent_wiring(flow_dir, message_queue): + """Phase correctly passes subagent_builder + log_dir to the agent builder at execute time.""" + + def make_usage() -> TokenUsage: + return TokenUsage(input_tokens=100, output_tokens=50, cache_read_input_tokens=0, cache_creation_input_tokens=0) + + spawn_call = ToolCall( + id="tc1", + name="spawn_subagent", + input={"system_prompt": "you are a helper", "prompt": "answer 42", "tools": [], "name": "child"}, + ) + parent_agent = MockAgent( + [ + AgentResponse(stop_reason=StopReason.TOOL_USE, text="", tool_calls=[spawn_call], usage=make_usage()), + AgentResponse(stop_reason=StopReason.END_TURN, text="parent done", tool_calls=[], usage=make_usage()), + AgentResponse(stop_reason=StopReason.END_TURN, text="memory summary", tool_calls=[], usage=make_usage()), + ] + ) + + subagent_calls: list = [] + + def mock_subagent_builder(system_prompt: str, owner_id: str, tool_names: list[str]): + subagent_calls.append(system_prompt) + return MockAgent( + [AgentResponse(stop_reason=StopReason.END_TURN, text="42", tool_calls=[], usage=make_usage())] + ), ToolRegistry([]) + + from ddev.ai.tools.agents.spawn_subagent import SpawnSubagentTool + + captured_log_dirs: list[Path | None] = [] + + def agent_builder_fn( + system_prompt: str, + owner_id: str, + subagent_builder=None, + log_dir: Path | None = None, + ): + captured_log_dirs.append(log_dir) + assert subagent_builder is not None + assert log_dir is not None + parent_agent.name = owner_id + return parent_agent, ToolRegistry( + [ + SpawnSubagentTool( + owner_id=owner_id, + subagent_builder=subagent_builder, + allowed_tools=[], + log_dir=log_dir, + ) + ] + ) + + checkpoint_manager = CheckpointManager(flow_dir / "checkpoints.yaml") + phase = AgenticPhase( + phase_id="p1", + dependencies=[], + config=PhaseConfig(agent="writer", tasks=[TaskConfig(name="t1", prompt="Do the work.")]), + agent_builder=agent_builder_fn, + checkpoint_manager=checkpoint_manager, + runtime_variables={}, + flow_variables={}, + config_dir=flow_dir, + file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), + subagent_builder=mock_subagent_builder, + ) + phase.queue = message_queue + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + submitted = [message_queue.get_nowait() for _ in range(message_queue.qsize())] + assert not any(isinstance(m, PhaseFailedMessage) for m in submitted) + assert subagent_calls == ["you are a helper"] + assert captured_log_dirs == [checkpoint_manager.root / "subagents" / "p1"] + + log_file = checkpoint_manager.root / "subagents" / "p1" / "001-child.jsonl" + assert log_file.exists() + events = {e["event"] for e in read_jsonl(log_file)} + assert {"start", "finish"} <= events diff --git a/ddev/tests/ai/phases/test_base.py b/ddev/tests/ai/phases/test_base.py new file mode 100644 index 0000000000000..b776aa11448a1 --- /dev/null +++ b/ddev/tests/ai/phases/test_base.py @@ -0,0 +1,226 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from datetime import UTC, datetime + +import pytest + +from ddev.ai.phases.base import Phase, PhaseOutcome +from ddev.ai.phases.checkpoint import CheckpointManager +from ddev.ai.phases.config import PhaseConfig +from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.event_bus.exceptions import HookName, MessageProcessingError, ProcessorHookError + + +class _StubPhase(Phase): + """Concrete Phase for lifecycle tests; execute() returns a deterministic PhaseOutcome.""" + + def __init__(self, *args, outcome: PhaseOutcome | None = None, **kwargs): + super().__init__(*args, **kwargs) + self._outcome = outcome or PhaseOutcome(memory_text="stub-memory") + + async def execute(self, context): + return self._outcome + + +def _make_stub_phase( + flow_dir, + message_queue, + *, + phase_id="p1", + dependencies=None, + outcome=None, +): + checkpoint_manager = CheckpointManager(flow_dir / "checkpoints.yaml") + phase = _StubPhase( + phase_id=phase_id, + dependencies=dependencies or [], + config=PhaseConfig(), + checkpoint_manager=checkpoint_manager, + runtime_variables={}, + flow_variables={}, + config_dir=flow_dir, + file_registry=FileRegistry(policy=FileAccessPolicy(write_root=flow_dir)), + outcome=outcome, + ) + phase.queue = message_queue + return phase, checkpoint_manager + + +# --------------------------------------------------------------------------- +# Phase.on_success +# --------------------------------------------------------------------------- + + +async def test_on_success_emits_finished_message(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue) + + await phase.on_success(PhaseTrigger(id="start", phase_id=None)) + + msg = message_queue.get_nowait() + assert isinstance(msg, PhaseTrigger) + assert msg.phase_id == "p1" + assert msg.id == "p1_finished" + + +# --------------------------------------------------------------------------- +# Phase.on_error +# --------------------------------------------------------------------------- + + +async def test_on_error_writes_failed_checkpoint(flow_dir, message_queue): + phase, mgr = _make_stub_phase(flow_dir, message_queue) + + wrapped = MessageProcessingError("p1", PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) + await phase.on_error(wrapped) + + checkpoint = mgr.read()["p1"] + assert checkpoint["status"] == "failed" + assert checkpoint["error"] == "boom" + assert checkpoint["started_at"] is None # not started yet + + +async def test_on_error_emits_failed_message(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue) + + wrapped = ProcessorHookError( + HookName.ON_SUCCESS, "p1", PhaseTrigger(id="start", phase_id=None), RuntimeError("boom") + ) + await phase.on_error(wrapped) + + msg = message_queue.get_nowait() + assert isinstance(msg, PhaseFailedMessage) + assert msg.phase_id == "p1" + assert msg.error == "boom" + + +async def test_on_error_writes_failed_checkpoint_after_start(flow_dir, message_queue): + phase, mgr = _make_stub_phase(flow_dir, message_queue) + phase._started_at = datetime.now(UTC) + + wrapped = MessageProcessingError("p1", PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) + await phase.on_error(wrapped) + + checkpoint = mgr.read()["p1"] + assert checkpoint["status"] == "failed" + assert checkpoint["started_at"] is not None + + +# --------------------------------------------------------------------------- +# Phase.should_process_message +# --------------------------------------------------------------------------- + + +def test_should_process_returns_true_for_initial_trigger_on_root_phase(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue) + + result = phase.should_process_message(PhaseTrigger(id="start", phase_id=None)) + + assert result is True + assert phase._executed is True + + +def test_should_process_returns_false_for_initial_trigger_on_dependent_phase(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue, dependencies=["dep1"]) + + result = phase.should_process_message(PhaseTrigger(id="start", phase_id=None)) + + assert result is False + assert phase._executed is False + + +def test_should_process_returns_false_for_unrelated_dep(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue, dependencies=["dep1"]) + + result = phase.should_process_message(PhaseTrigger(id="msg1", phase_id="other")) + + assert result is False + assert phase._executed is False + + +def test_should_process_returns_false_while_deps_pending(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue, dependencies=["dep1", "dep2"]) + + result = phase.should_process_message(PhaseTrigger(id="msg1", phase_id="dep1")) + + assert result is False + assert phase._remaining_dependencies == {"dep2"} + assert phase._executed is False + + +def test_should_process_returns_true_when_last_dep_arrives(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue, dependencies=["dep1", "dep2"]) + + phase.should_process_message(PhaseTrigger(id="msg1", phase_id="dep1")) + result = phase.should_process_message(PhaseTrigger(id="msg2", phase_id="dep2")) + + assert result is True + assert phase._executed is True + + +def test_should_process_returns_false_after_already_executed(flow_dir, message_queue): + phase, _ = _make_stub_phase(flow_dir, message_queue) + + phase.should_process_message(PhaseTrigger(id="start", phase_id=None)) + result = phase.should_process_message(PhaseTrigger(id="start2", phase_id=None)) + + assert result is False + + +# --------------------------------------------------------------------------- +# Phase lifecycle β€” memory path +# --------------------------------------------------------------------------- + + +async def test_process_message_writes_memory_and_checkpoint(flow_dir, message_queue): + """End-to-end Phase contract: memory_text is persisted, extra_checkpoint merges, + token totals land in the checkpoint, and the success metadata is recorded. + """ + outcome = PhaseOutcome( + memory_text="stub-memory-body", + total_input_tokens=123, + total_output_tokens=45, + extra_checkpoint={"custom_field": "custom_value", "count": 7}, + ) + phase, mgr = _make_stub_phase(flow_dir, message_queue, outcome=outcome) + + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.memory_content("p1") == "stub-memory-body" + + checkpoint = mgr.read()["p1"] + assert checkpoint["status"] == "success" + assert checkpoint["tokens"] == {"total_input": 123, "total_output": 45} + assert checkpoint["memory_path"] == str(mgr.memory_path("p1")) + assert checkpoint["custom_field"] == "custom_value" + assert checkpoint["count"] == 7 + assert checkpoint["started_at"] + assert checkpoint["finished_at"] + + +@pytest.mark.parametrize( + "reserved_key", + ["status", "started_at", "finished_at", "tokens", "memory_path"], +) +async def test_extra_checkpoint_cannot_override_reserved_keys(flow_dir, message_queue, reserved_key): + outcome = PhaseOutcome(memory_text="m", extra_checkpoint={reserved_key: "evil"}) + phase, mgr = _make_stub_phase(flow_dir, message_queue, outcome=outcome) + + with pytest.raises(ValueError, match=f"reserved keys.*{reserved_key}"): + await phase.process_message(PhaseTrigger(id="start", phase_id=None)) + + assert mgr.read() == {} + assert not mgr.memory_path("p1").exists() + + +async def test_failed_phase_omits_memory_path(flow_dir, message_queue): + phase, mgr = _make_stub_phase(flow_dir, message_queue) + + wrapped = MessageProcessingError("p1", PhaseTrigger(id="start", phase_id=None), RuntimeError("boom")) + await phase.on_error(wrapped) + + checkpoint = mgr.read()["p1"] + assert "memory_path" not in checkpoint diff --git a/ddev/tests/ai/phases/test_checkpoint.py b/ddev/tests/ai/phases/test_checkpoint.py new file mode 100644 index 0000000000000..41ec010e3481f --- /dev/null +++ b/ddev/tests/ai/phases/test_checkpoint.py @@ -0,0 +1,143 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from pathlib import Path + +import pytest + +from ddev.ai.phases.checkpoint import CheckpointManager, CheckpointReadError + + +@pytest.fixture +def manager(tmp_path) -> CheckpointManager: + return CheckpointManager(tmp_path / "checkpoints.yaml") + + +# --------------------------------------------------------------------------- +# read +# --------------------------------------------------------------------------- + + +def test_read_returns_empty_when_file_absent(manager): + assert manager.read() == {} + + +def test_read_returns_empty_when_file_is_empty(manager): + manager._path.write_text("") + assert manager.read() == {} + + +def test_read_malformed_yaml_raises_checkpoint_read_error(manager): + manager._path.write_text(": :\n -[") + with pytest.raises(CheckpointReadError, match="checkpoints.yaml"): + manager.read() + + +def test_read_unreadable_file_raises_checkpoint_read_error(manager, monkeypatch): + manager._path.write_text("phase1:\n status: success\n") + monkeypatch.setattr("pathlib.Path.read_text", lambda *_, **__: (_ for _ in ()).throw(OSError("permission denied"))) + with pytest.raises(CheckpointReadError, match="checkpoints.yaml"): + manager.read() + + +# --------------------------------------------------------------------------- +# write_phase_checkpoint +# --------------------------------------------------------------------------- + + +def test_write_and_read_back(manager): + manager.write_phase_checkpoint("phase1", {"status": "success", "tokens": 100}) + data = manager.read() + assert data["phase1"]["status"] == "success" + assert data["phase1"]["tokens"] == 100 + + +def test_write_creates_parent_dirs(tmp_path): + manager = CheckpointManager(tmp_path / "nested" / "dir" / "checkpoints.yaml") + manager.write_phase_checkpoint("p", {"status": "success"}) + assert manager.read()["p"]["status"] == "success" + + +def test_write_multiple_phases(manager): + manager.write_phase_checkpoint("phase1", {"status": "success"}) + manager.write_phase_checkpoint("phase2", {"status": "failed"}) + data = manager.read() + assert data["phase1"]["status"] == "success" + assert data["phase2"]["status"] == "failed" + + +def test_write_overwrites_existing_phase(manager): + manager.write_phase_checkpoint("phase1", {"status": "running"}) + manager.write_phase_checkpoint("phase1", {"status": "success"}) + assert manager.read()["phase1"]["status"] == "success" + + +# --------------------------------------------------------------------------- +# build_memory_prompt +# --------------------------------------------------------------------------- + + +def test_build_memory_prompt_no_additions(manager): + result = manager.build_memory_prompt(None) + assert result == "Write a brief summary of what you accomplished in this phase." + + +def test_build_memory_prompt_with_additions(manager): + result = manager.build_memory_prompt("Also list the files you created.") + assert result.startswith("Also list the files you created.") + assert "Write a brief summary" in result + + +# --------------------------------------------------------------------------- +# write_memory / memory_content / memory_path +# --------------------------------------------------------------------------- + + +def test_write_memory_and_read_back(manager): + manager.write_memory("draft", "Created integration.py and tests.") + assert manager.memory_content("draft") == "Created integration.py and tests." + + +def test_write_memory_overwrites(manager): + manager.write_memory("draft", "first version") + manager.write_memory("draft", "second version") + assert manager.memory_content("draft") == "second version" + + +def test_memory_content_absent_returns_placeholder(manager): + assert manager.memory_content("nonexistent") == "" + + +def test_memory_path_returns_absolute_path(manager): + path = manager.memory_path("phase1") + assert isinstance(path, Path) + assert path.is_absolute() + assert path.name == "phase1_memory.md" + + +def test_memory_path_before_write(manager): + path = manager.memory_path("phase1") + assert not path.exists() + + +def test_memory_file_location(manager): + manager.write_memory("phase1", "content") + expected_path = manager._path.parent / "phase1_memory.md" + assert expected_path.exists() + assert expected_path.read_text() == "content" + assert manager.memory_path("phase1") == expected_path.resolve() + + +# --------------------------------------------------------------------------- +# resolve_template_variable +# --------------------------------------------------------------------------- + + +def test_resolve_template_variable_memory_suffix(manager): + manager.write_memory("draft", "Draft memory content.") + assert manager.resolve_template_variable("draft_memory") == "Draft memory content." + + +def test_resolve_template_variable_non_memory_key(manager): + assert manager.resolve_template_variable("some_variable") == "" diff --git a/ddev/tests/ai/phases/test_config.py b/ddev/tests/ai/phases/test_config.py new file mode 100644 index 0000000000000..79deda7b989b5 --- /dev/null +++ b/ddev/tests/ai/phases/test_config.py @@ -0,0 +1,422 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import pytest +from pydantic import ValidationError + +from ddev.ai.phases.config import ( + AgentConfig, + CheckpointConfig, + FlowConfig, + FlowConfigError, + PhaseConfig, + TaskConfig, +) + +# --------------------------------------------------------------------------- +# TaskConfig +# --------------------------------------------------------------------------- + + +def test_task_config_with_prompt(): + tc = TaskConfig(name="t1", prompt="Do it.") + assert tc.prompt == "Do it." + assert tc.prompt_path is None + + +def test_task_config_with_prompt_path(): + tc = TaskConfig(name="t1", prompt_path="prompts/task.md") + assert tc.prompt is None + assert tc.prompt_path is not None + + +def test_task_config_both_set_raises(): + with pytest.raises(ValidationError, match="Exactly one"): + TaskConfig(name="t1", prompt="Do it.", prompt_path="prompts/task.md") + + +def test_task_config_neither_set_raises(): + with pytest.raises(ValidationError, match="Exactly one"): + TaskConfig(name="t1") + + +def test_task_config_extra_field_raises(): + with pytest.raises(ValidationError, match="extra"): + TaskConfig(name="t1", prompt="Do it.", unknown_field="x") + + +# --------------------------------------------------------------------------- +# CheckpointConfig +# --------------------------------------------------------------------------- + + +def test_checkpoint_config_with_memory_prompt(): + cc = CheckpointConfig(memory_prompt="List files.") + assert cc.memory_prompt == "List files." + + +def test_checkpoint_config_with_memory_prompt_path(): + cc = CheckpointConfig(memory_prompt_path="prompts/mem.md") + assert cc.memory_prompt_path is not None + + +def test_checkpoint_config_both_set_raises(): + with pytest.raises(ValidationError, match="Exactly one"): + CheckpointConfig(memory_prompt="List files.", memory_prompt_path="prompts/mem.md") + + +def test_checkpoint_config_neither_set_raises(): + with pytest.raises(ValidationError, match="Exactly one"): + CheckpointConfig() + + +# --------------------------------------------------------------------------- +# AgentConfig +# --------------------------------------------------------------------------- + + +def test_agent_config_valid_tools(): + ac = AgentConfig(tools=["read_file", "grep"]) + assert ac.tools == ["read_file", "grep"] + + +def test_agent_config_unknown_tool_raises(): + with pytest.raises(ValidationError, match="Unknown tool names"): + AgentConfig(tools=["read_file", "teleport"]) + + +def test_agent_config_empty_tools(): + ac = AgentConfig() + assert ac.tools == [] + + +def test_agent_config_optional_fields(): + ac = AgentConfig(model="claude-opus-4-5", max_tokens=4096) + assert ac.model == "claude-opus-4-5" + assert ac.max_tokens == 4096 + + +# --------------------------------------------------------------------------- +# PhaseConfig +# --------------------------------------------------------------------------- + + +def test_phase_config_defaults(): + pc = PhaseConfig(agent="writer", tasks=[TaskConfig(name="t1", prompt="Do it.")]) + assert pc.type == "AgenticPhase" + assert pc.context_compact_threshold_pct == 80 + assert pc.checkpoint is None + + +def test_phase_config_with_checkpoint(): + pc = PhaseConfig( + agent="writer", + tasks=[TaskConfig(name="t1", prompt="Do it.")], + checkpoint=CheckpointConfig(memory_prompt="List files."), + ) + assert pc.checkpoint is not None + + +# --------------------------------------------------------------------------- +# FlowConfig cross-reference validation +# --------------------------------------------------------------------------- + + +def _minimal_config(**overrides) -> dict: + base = { + "agents": {"writer": {"tools": []}}, + "phases": {"p1": {"agent": "writer", "tasks": [{"name": "t1", "prompt": "Do it."}]}}, + "flow": [{"phase": "p1"}], + } + base.update(overrides) + return base + + +def test_flow_config_minimal_valid(): + config = FlowConfig.model_validate(_minimal_config()) + assert "p1" in config.phases + + +def test_flow_config_unknown_phase_in_flow(): + raw = _minimal_config() + raw["flow"] = [{"phase": "nonexistent"}] + with pytest.raises(ValidationError, match="unknown phase"): + FlowConfig.model_validate(raw) + + +def test_flow_config_unknown_dependency(): + raw = _minimal_config() + raw["flow"] = [{"phase": "p1", "dependencies": ["nonexistent"]}] + with pytest.raises(ValidationError, match="unknown phase"): + FlowConfig.model_validate(raw) + + +def test_flow_config_dependency_not_scheduled_in_flow(): + raw = { + "agents": {"writer": {"tools": []}}, + "phases": { + "p1": {"agent": "writer", "tasks": [{"name": "t1", "prompt": "Do it."}]}, + "p2": {"agent": "writer", "tasks": [{"name": "t2", "prompt": "Review it."}]}, + }, + "flow": [{"phase": "p2", "dependencies": ["p1"]}], + } + with pytest.raises(ValidationError, match="not scheduled in flow"): + FlowConfig.model_validate(raw) + + +def test_flow_config_duplicate_phase_raises(): + raw = _minimal_config() + raw["flow"] = [{"phase": "p1"}, {"phase": "p1"}] + with pytest.raises(ValidationError, match="Duplicate phase"): + FlowConfig.model_validate(raw) + + +def test_flow_config_unknown_agent_in_phase(): + raw = _minimal_config() + raw["phases"]["p1"]["agent"] = "nonexistent" + with pytest.raises(ValidationError, match="unknown agent"): + FlowConfig.model_validate(raw) + + +def test_flow_config_phase_without_agent_validates(): + raw = { + "agents": {"writer": {"tools": []}}, + "phases": { + "p1": {"agent": "writer", "tasks": [{"name": "t1", "prompt": "Do it."}]}, + "noop": {"type": "SomeCustomPhase"}, + }, + "flow": [ + {"phase": "p1"}, + {"phase": "noop", "dependencies": ["p1"]}, + ], + } + config = FlowConfig.model_validate(raw) + assert config.phases["noop"].agent is None + assert config.phases["noop"].tasks == [] + + +def test_flow_config_with_variables(): + raw = _minimal_config(variables={"project": "myproj"}) + config = FlowConfig.model_validate(raw) + assert config.variables["project"] == "myproj" + + +def test_flow_config_multiple_phases_and_deps(): + raw = { + "agents": {"writer": {"tools": []}}, + "phases": { + "p1": {"agent": "writer", "tasks": [{"name": "t1", "prompt": "Do it."}]}, + "p2": {"agent": "writer", "tasks": [{"name": "t2", "prompt": "Review it."}]}, + }, + "flow": [ + {"phase": "p1"}, + {"phase": "p2", "dependencies": ["p1"]}, + ], + } + config = FlowConfig.model_validate(raw) + assert len(config.flow) == 2 + assert config.flow[1].dependencies == ["p1"] + + +def test_flow_config_extra_field_raises(): + raw = _minimal_config() + raw["extra"] = "boom" + with pytest.raises(ValidationError, match="extra"): + FlowConfig.model_validate(raw) + + +# --------------------------------------------------------------------------- +# FlowConfig.from_yaml +# --------------------------------------------------------------------------- + + +def test_from_yaml_valid(tmp_path): + prompts_dir = tmp_path / "prompts" + prompts_dir.mkdir() + (prompts_dir / "writer.md").write_text("system prompt") + + flow_yaml = tmp_path / "flow.yaml" + flow_yaml.write_text( + """\ +agents: + writer: + tools: [] +phases: + p1: + agent: writer + tasks: + - name: t1 + prompt: "Do it." +flow: + - phase: p1 +""" + ) + config = FlowConfig.from_yaml(flow_yaml, tmp_path) + assert "p1" in config.phases + + +def test_from_yaml_missing_system_prompt(tmp_path): + (tmp_path / "prompts").mkdir() + + flow_yaml = tmp_path / "flow.yaml" + flow_yaml.write_text( + """\ +agents: + writer: + tools: [] +phases: + p1: + agent: writer + tasks: + - name: t1 + prompt: "Do it." +flow: + - phase: p1 +""" + ) + with pytest.raises(FlowConfigError, match="System prompt not found"): + FlowConfig.from_yaml(flow_yaml, tmp_path) + + +def test_from_yaml_missing_task_prompt_path(tmp_path): + prompts_dir = tmp_path / "prompts" + prompts_dir.mkdir() + (prompts_dir / "writer.md").write_text("system prompt") + + flow_yaml = tmp_path / "flow.yaml" + flow_yaml.write_text( + """\ +agents: + writer: + tools: [] +phases: + p1: + agent: writer + tasks: + - name: t1 + prompt_path: prompts/nonexistent.md +flow: + - phase: p1 +""" + ) + with pytest.raises(FlowConfigError, match="prompt_path not found"): + FlowConfig.from_yaml(flow_yaml, tmp_path) + + +def test_from_yaml_invalid_yaml(tmp_path): + flow_yaml = tmp_path / "flow.yaml" + flow_yaml.write_text(": invalid: yaml: [") + with pytest.raises(FlowConfigError, match="Failed to load"): + FlowConfig.from_yaml(flow_yaml, tmp_path) + + +def test_from_yaml_missing_file(tmp_path): + with pytest.raises(FlowConfigError, match="Failed to load"): + FlowConfig.from_yaml(tmp_path / "nonexistent.yaml", tmp_path) + + +# --------------------------------------------------------------------------- +# FlowConfig cycle detection via model_validate +# --------------------------------------------------------------------------- + + +def _three_phase_config() -> dict: + agent = {"tools": []} + task = {"name": "t", "prompt": "Do it."} + return { + "agents": {"writer": agent}, + "phases": { + "p1": {"agent": "writer", "tasks": [task]}, + "p2": {"agent": "writer", "tasks": [task]}, + "p3": {"agent": "writer", "tasks": [task]}, + }, + } + + +def test_flow_config_direct_cycle_raises(): + raw = _three_phase_config() + raw["flow"] = [ + {"phase": "p1", "dependencies": ["p2"]}, + {"phase": "p2", "dependencies": ["p1"]}, + ] + with pytest.raises(ValidationError, match="Cycle"): + FlowConfig.model_validate(raw) + + +def test_flow_config_three_node_cycle_raises(): + raw = _three_phase_config() + raw["flow"] = [ + {"phase": "p1", "dependencies": ["p3"]}, + {"phase": "p2", "dependencies": ["p1"]}, + {"phase": "p3", "dependencies": ["p2"]}, + ] + with pytest.raises(ValidationError, match="Cycle"): + FlowConfig.model_validate(raw) + + +def test_flow_config_acyclic_chain_ok(): + raw = _three_phase_config() + raw["flow"] = [ + {"phase": "p1"}, + {"phase": "p2", "dependencies": ["p1"]}, + {"phase": "p3", "dependencies": ["p1"]}, + ] + config = FlowConfig.model_validate(raw) + assert len(config.flow) == 3 + + +def test_flow_disjoined_graphs_ok(): + agent = {"tools": []} + task = {"name": "t", "prompt": "Do it."} + raw = { + "agents": {"writer": agent}, + "phases": { + "p1": {"agent": "writer", "tasks": [task]}, + "p2": {"agent": "writer", "tasks": [task]}, + "p3": {"agent": "writer", "tasks": [task]}, + "p4": {"agent": "writer", "tasks": [task]}, + }, + "flow": [ + {"phase": "p1"}, + {"phase": "p2", "dependencies": ["p1"]}, + {"phase": "p3"}, + {"phase": "p4", "dependencies": ["p3"]}, + ], + } + config = FlowConfig.model_validate(raw) + assert len(config.flow) == 4 + + +def test_flow_config_self_dependency_raises(): + raw = _minimal_config() + raw["flow"] = [{"phase": "p1", "dependencies": ["p1"]}] + with pytest.raises(ValidationError, match="Cycle"): + FlowConfig.model_validate(raw) + + +def test_flow_config_two_independent_cycles_reports_both(): + agent = {"tools": []} + task = {"name": "t", "prompt": "Do it."} + raw = { + "agents": {"writer": agent}, + "phases": { + "p1": {"agent": "writer", "tasks": [task]}, + "p2": {"agent": "writer", "tasks": [task]}, + "p3": {"agent": "writer", "tasks": [task]}, + "p4": {"agent": "writer", "tasks": [task]}, + }, + "flow": [ + # dependency edges: p1β†’p3β†’p2β†’p1 and p1β†’p4β†’p2β†’p1 + {"phase": "p1", "dependencies": ["p3", "p4"]}, + {"phase": "p2", "dependencies": ["p1"]}, + {"phase": "p3", "dependencies": ["p2"]}, + {"phase": "p4", "dependencies": ["p2"]}, + ], + } + with pytest.raises(ValidationError) as exc_info: + FlowConfig.model_validate(raw) + error = str(exc_info.value) + assert "Cycle" in error + assert "p1 β†’ p3 β†’ p2 β†’ p1" in error + assert "p1 β†’ p4 β†’ p2 β†’ p1" in error diff --git a/ddev/tests/ai/phases/test_orchestrator.py b/ddev/tests/ai/phases/test_orchestrator.py new file mode 100644 index 0000000000000..85c9758a95d1b --- /dev/null +++ b/ddev/tests/ai/phases/test_orchestrator.py @@ -0,0 +1,513 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import logging +from pathlib import Path +from textwrap import dedent +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from ddev.ai.phases.agentic_phase import AgenticPhase +from ddev.ai.phases.base import Phase, PhaseRegistry +from ddev.ai.phases.config import FlowConfigError +from ddev.ai.phases.messages import PhaseFailedMessage, PhaseTrigger +from ddev.ai.phases.orchestrator import PhaseOrchestrator, _discover_and_register_phases +from ddev.event_bus.exceptions import FatalProcessingError + + +@pytest.fixture +def make_orchestrator(file_access_policy): + """Factory that builds a PhaseOrchestrator with test defaults. + + Pass a ``base_dir`` to anchor ``flow.yaml`` / ``checkpoints.yaml`` (defaults to + ``/fake`` for tests that never touch disk). Any constructor kwarg can be overridden. + """ + + def _make(base_dir: Path | None = None, **overrides: Any) -> PhaseOrchestrator: + base_dir = base_dir if base_dir is not None else Path("/fake") + kwargs: dict[str, Any] = { + "flow_yaml_path": base_dir / "flow.yaml", + "checkpoint_path": base_dir / "checkpoints.yaml", + "runtime_variables": {}, + "agent_clients": {"anthropic": MagicMock()}, + "file_access_policy": file_access_policy, + **overrides, + } + return PhaseOrchestrator(**kwargs) + + return _make + + +# --------------------------------------------------------------------------- +# _discover_and_register_phases +# --------------------------------------------------------------------------- + + +def test_discover_registers_agentic_phase(): + registry = PhaseRegistry() + _discover_and_register_phases(registry) + assert "AgenticPhase" in registry.known_names() + assert registry.get("AgenticPhase") is AgenticPhase + + +def test_discover_registers_custom_subclass(tmp_path, monkeypatch): + """Discovery imports a real .py file and registers the Phase subclass it defines.""" + fake_dir = tmp_path / "fake_phases" + fake_dir.mkdir() + (fake_dir / "__init__.py").write_text("") + (fake_dir / "custom.py").write_text( + "from ddev.ai.phases.agentic_phase import AgenticPhase\nclass CustomPhase(AgenticPhase):\n pass\n" + ) + monkeypatch.syspath_prepend(str(tmp_path)) + + registry = PhaseRegistry() + _discover_and_register_phases(registry, phases_dir=fake_dir, import_prefix="fake_phases") + + assert "CustomPhase" in registry.known_names() + assert issubclass(registry.get("CustomPhase"), AgenticPhase) + + +def test_discover_ignores_module_without_phase_subclass(tmp_path, monkeypatch): + fake_dir = tmp_path / "no_phase_pkg" + fake_dir.mkdir() + (fake_dir / "__init__.py").write_text("") + (fake_dir / "helpers.py").write_text("CONSTANT = 42\n") + monkeypatch.syspath_prepend(str(tmp_path)) + + registry = PhaseRegistry() + _discover_and_register_phases(registry, phases_dir=fake_dir, import_prefix="no_phase_pkg") + + assert registry.known_names() == [] + + +def test_discover_does_not_register_imported_phase_class(tmp_path, monkeypatch): + """A module that imports Phase but defines no subclass should not register Phase itself.""" + fake_dir = tmp_path / "importer_pkg" + fake_dir.mkdir() + (fake_dir / "__init__.py").write_text("") + (fake_dir / "importer.py").write_text("from ddev.ai.phases.agentic_phase import AgenticPhase\n") + monkeypatch.syspath_prepend(str(tmp_path)) + + registry = PhaseRegistry() + _discover_and_register_phases(registry, phases_dir=fake_dir, import_prefix="importer_pkg") + + assert "AgenticPhase" not in registry.known_names() + + +def test_discover_skips_underscore_prefixed_files(tmp_path, monkeypatch): + """Classes defined in underscore-prefixed files (e.g. _private.py) are never registered.""" + fake_dir = tmp_path / "underscore_pkg" + fake_dir.mkdir() + (fake_dir / "__init__.py").write_text("") + (fake_dir / "_private.py").write_text( + "from ddev.ai.phases.agentic_phase import AgenticPhase\nclass PrivatePhase(AgenticPhase):\n pass\n" + ) + (fake_dir / "public.py").write_text( + "from ddev.ai.phases.agentic_phase import AgenticPhase\nclass PublicPhase(AgenticPhase):\n pass\n" + ) + monkeypatch.syspath_prepend(str(tmp_path)) + + registry = PhaseRegistry() + _discover_and_register_phases(registry, phases_dir=fake_dir, import_prefix="underscore_pkg") + + assert "PrivatePhase" not in registry.known_names() + assert "PublicPhase" in registry.known_names() + + +def test_discover_idempotent(): + registry = PhaseRegistry() + _discover_and_register_phases(registry) + first = registry.known_names() + _discover_and_register_phases(registry) + second = registry.known_names() + assert first == second + + +def test_registry_get_unknown_raises(): + registry = PhaseRegistry() + with pytest.raises(ValueError, match="Unknown phase type"): + registry.get("NonexistentPhase") + + +def test_imported_class_not_registered(): + """A class imported into a phases module but defined elsewhere should not be registered.""" + registry = PhaseRegistry() + _discover_and_register_phases(registry) + # BaseMessage is imported in messages.py but defined in event_bus β€” it should NOT be registered + assert "BaseMessage" not in registry.known_names() + + +def test_two_orchestrators_have_independent_registries(tmp_path, make_orchestrator): + """Each PhaseOrchestrator owns its own registry; registering in one does not affect the other.""" + o1 = make_orchestrator(tmp_path) + o2 = make_orchestrator(tmp_path) + + class ExclusivePhase(Phase): + pass + + o1._phase_registry.register("ExclusivePhase", ExclusivePhase) + assert "ExclusivePhase" in o1._phase_registry.known_names() + assert "ExclusivePhase" not in o2._phase_registry.known_names() + + +def test_discover_does_not_mutate_global_state(): + """_discover_and_register_phases only touches the registry passed to it.""" + registry = PhaseRegistry() + _discover_and_register_phases(registry) + # No module-level / class-level container should have been touched. + # Verify by checking there is no class-level _registry attribute on PhaseRegistry. + assert not hasattr(PhaseRegistry, "_registry") + + +# --------------------------------------------------------------------------- +# PhaseOrchestrator.on_message_received +# --------------------------------------------------------------------------- + + +async def test_on_message_received_fatal_on_phase_failed(make_orchestrator): + orchestrator = make_orchestrator() + msg = PhaseFailedMessage(id="f1", phase_id="p1", error="something broke") + + with pytest.raises(FatalProcessingError, match="Phase 'p1' failed"): + await orchestrator.on_message_received(msg) + + +async def test_on_message_received_ignores_other_messages(make_orchestrator): + orchestrator = make_orchestrator() + # These should not raise + await orchestrator.on_message_received(PhaseTrigger(id="start", phase_id=None)) + await orchestrator.on_message_received(PhaseTrigger(id="f1", phase_id="p1")) + + +# --------------------------------------------------------------------------- +# PhaseOrchestrator.on_initialize +# --------------------------------------------------------------------------- + + +@pytest.fixture +def minimal_flow(tmp_path): + """Two-phase flow: 'a' is root, 'b' depends on 'a'.""" + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + a: + agent: writer + tasks: + - name: task_a + prompt: task a + b: + agent: writer + tasks: + - name: task_b + prompt: task b + flow: + - phase: a + - phase: b + dependencies: [a] + """) + ) + return tmp_path + + +async def test_on_initialize_registers_all_flow_phases(minimal_flow, make_orchestrator): + orchestrator = make_orchestrator(minimal_flow) + await orchestrator.on_initialize() + + processors = orchestrator._subscribers.get(PhaseTrigger, []) + phase_names = {p.name for p in processors} + assert phase_names == {"a", "b"} + + +async def test_on_initialize_wires_dependencies(minimal_flow, make_orchestrator): + orchestrator = make_orchestrator(minimal_flow) + await orchestrator.on_initialize() + + processors = orchestrator._subscribers.get(PhaseTrigger, []) + phases_by_name = {p.name: p for p in processors} + assert phases_by_name["a"]._dependencies == set() + assert phases_by_name["b"]._dependencies == {"a"} + + +async def test_on_initialize_submits_initial_phase_trigger(minimal_flow, make_orchestrator): + orchestrator = make_orchestrator(minimal_flow) + await orchestrator.on_initialize() + + assert not orchestrator._queue.empty() + msg = orchestrator._queue.get_nowait() + assert isinstance(msg, PhaseTrigger) + assert msg.phase_id is None + + +async def test_on_initialize_unknown_phase_type_raises_flow_config_error(tmp_path, make_orchestrator): + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + a: + type: NotARealPhase + agent: writer + tasks: + - name: task_a + prompt: task a + flow: + - phase: a + """) + ) + orchestrator = make_orchestrator(tmp_path) + with pytest.raises(FlowConfigError, match="Unknown phase type"): + await orchestrator.on_initialize() + + +async def test_on_initialize_missing_agent_raises(tmp_path, make_orchestrator): + (tmp_path / "prompts").mkdir() + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + a: + agent: nonexistent_agent + tasks: + - name: task_a + prompt: task a + flow: + - phase: a + """) + ) + orchestrator = make_orchestrator(tmp_path) + with pytest.raises(FlowConfigError): + await orchestrator.on_initialize() + + +async def test_on_initialize_phases_share_file_registry(minimal_flow, make_orchestrator): + orchestrator = make_orchestrator(minimal_flow) + await orchestrator.on_initialize() + phases = orchestrator._subscribers.get(PhaseTrigger, []) + assert len(phases) >= 2 + assert all(p._file_registry is phases[0]._file_registry for p in phases[1:]) + + +# --------------------------------------------------------------------------- +# PhaseOrchestrator.on_initialize β€” orphan-phase validation +# --------------------------------------------------------------------------- + + +async def test_orphan_phase_with_unknown_type_does_not_block_init(tmp_path, make_orchestrator): + """A phase defined in phases: but absent from flow: may have an unknown type β€” no error.""" + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + real: + agent: writer + tasks: + - name: t1 + prompt: do it + orphan: + type: BogusType + agent: writer + tasks: + - name: t2 + prompt: ignored + flow: + - phase: real + """) + ) + orchestrator = make_orchestrator(tmp_path) + await orchestrator.on_initialize() + + processors = orchestrator._subscribers.get(PhaseTrigger, []) + assert {p.name for p in processors} == {"real"} + + +async def test_phase_in_flow_with_unknown_type_raises(tmp_path, make_orchestrator): + """A phase referenced from flow: with an unknown type must still raise FlowConfigError.""" + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + a: + type: NotARealPhase + agent: writer + tasks: + - name: t1 + prompt: do it + flow: + - phase: a + """) + ) + orchestrator = make_orchestrator(tmp_path) + with pytest.raises(FlowConfigError, match="Unknown phase type"): + await orchestrator.on_initialize() + + +async def test_orphan_phase_logs_warning(tmp_path, make_orchestrator, caplog): + """An orphan phase must emit a warning containing its phase id.""" + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + real: + agent: writer + tasks: + - name: t1 + prompt: do it + orphan: + agent: writer + tasks: + - name: t2 + prompt: ignored + flow: + - phase: real + """) + ) + orchestrator = make_orchestrator(tmp_path) + with caplog.at_level(logging.WARNING): + await orchestrator.on_initialize() + + assert any("orphan" in record.message for record in caplog.records) + + +# --------------------------------------------------------------------------- +# PhaseOrchestrator.on_initialize β€” validate_config invocation +# --------------------------------------------------------------------------- + + +async def test_on_initialize_invokes_validate_config(tmp_path, make_orchestrator): + """validate_config is called for each scheduled phase; raising propagates as FlowConfigError.""" + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + a: + agent: writer + tasks: [] + flow: + - phase: a + """) + ) + orchestrator = make_orchestrator(tmp_path) + with pytest.raises(FlowConfigError, match="at least one task"): + await orchestrator.on_initialize() + + +async def test_on_initialize_skips_validate_config_for_orphan(tmp_path, make_orchestrator): + """A phase defined but not in flow must not trigger its validate_config.""" + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + real: + agent: writer + tasks: + - name: t1 + prompt: do it + orphan: + agent: writer + tasks: [] + flow: + - phase: real + """) + ) + orchestrator = make_orchestrator(tmp_path) + await orchestrator.on_initialize() # must not raise + + +# --------------------------------------------------------------------------- +# PhaseOrchestrator.on_finalize +# --------------------------------------------------------------------------- + + +async def test_on_finalize_no_failure_is_noop(tmp_path, make_orchestrator): + orchestrator = make_orchestrator() + await orchestrator.on_finalize(None) # must not raise + + +async def test_on_finalize_after_phase_failed_logs(tmp_path, make_orchestrator, caplog): + orchestrator = make_orchestrator() + msg = PhaseFailedMessage(id="f1", phase_id="p1", error="boom") + exc = FatalProcessingError("Phase 'p1' failed: boom") + with pytest.raises(FatalProcessingError): + await orchestrator.on_message_received(msg) + + with caplog.at_level(logging.ERROR): + await orchestrator.on_finalize(exc) # must not raise + + assert any("Pipeline aborted" in r.message and "p1" in r.message and "boom" in r.message for r in caplog.records) + + +async def test_on_finalize_no_exception_no_log(tmp_path, make_orchestrator, caplog): + orchestrator = make_orchestrator() + msg = PhaseFailedMessage(id="f1", phase_id="p1", error="boom") + with pytest.raises(FatalProcessingError): + await orchestrator.on_message_received(msg) + + with caplog.at_level(logging.ERROR): + await orchestrator.on_finalize(None) # exception=None means clean exit β€” no log + + assert not any("Pipeline aborted" in r.message for r in caplog.records) + + +def test_run_raises_runtime_error_when_phase_fails(tmp_path, make_orchestrator): + """Full pipeline: a failing phase must cause run() to raise RuntimeError.""" + (tmp_path / "prompts").mkdir() + (tmp_path / "prompts" / "writer.md").write_text("system prompt") + (tmp_path / "flow.yaml").write_text( + dedent("""\ + agents: + writer: + tools: [] + phases: + failing: + type: FailingPhase + agent: writer + tasks: + - name: t1 + prompt: do it + flow: + - phase: failing + """) + ) + + class FailingPhase(Phase): + async def execute(self, context): + raise RuntimeError("intentional failure") + + orchestrator = make_orchestrator(tmp_path, grace_period=0.1) + orchestrator._phase_registry.register("FailingPhase", FailingPhase) + + with pytest.raises(FatalProcessingError, match="Phase 'failing' failed"): + orchestrator.run() diff --git a/ddev/tests/ai/phases/test_template.py b/ddev/tests/ai/phases/test_template.py new file mode 100644 index 0000000000000..f7cf0f9ded088 --- /dev/null +++ b/ddev/tests/ai/phases/test_template.py @@ -0,0 +1,103 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from ddev.ai.phases.template import _SafeMapping, render_inline, render_prompt + +from .conftest import resolve_key + +# --------------------------------------------------------------------------- +# _SafeMapping +# --------------------------------------------------------------------------- + + +def test_safe_mapping_key_in_context(): + mapping = _SafeMapping({"name": "Alice"}) + assert mapping["name"] == "Alice" + + +def test_safe_mapping_key_absent_with_resolver(): + mapping = _SafeMapping({}, resolve_key) + assert mapping["missing"] == "resolved(missing)" + + +def test_safe_mapping_key_absent_no_resolver(): + mapping = _SafeMapping({}) + assert mapping["missing"] == "" + + +def test_safe_mapping_context_takes_precedence_over_resolver(): + def resolver(key): + return "from_resolver" + + mapping = _SafeMapping({"key": "from_context"}, resolver) + assert mapping["key"] == "from_context" + + +def test_safe_mapping_non_string_value_converted(): + mapping = _SafeMapping({"count": 42}) + assert mapping["count"] == "42" + + +# --------------------------------------------------------------------------- +# render_prompt +# --------------------------------------------------------------------------- + + +def test_render_prompt_substitutes_variables(tmp_path): + template = tmp_path / "prompt.md" + template.write_text("Hello ${name}, you are ${role}.") + result = render_prompt(template, {"name": "Alice", "role": "writer"}) + assert result == "Hello Alice, you are writer." + + +def test_render_prompt_missing_variable_shows_placeholder(tmp_path): + template = tmp_path / "prompt.md" + template.write_text("Hello ${name}.") + result = render_prompt(template, {}) + assert result == "Hello ." + + +def test_render_prompt_uses_resolver(tmp_path): + template = tmp_path / "prompt.md" + template.write_text("Memory: ${draft_memory}") + result = render_prompt(template, {}, resolve_key) + assert result == "Memory: resolved(draft_memory)" + + +def test_render_prompt_resolver_not_called_when_key_in_context(tmp_path): + called = [] + template = tmp_path / "prompt.md" + template.write_text("Value: ${key}") + + def resolver(k): + called.append(k) + return "nope" + + render_prompt(template, {"key": "from_context"}, resolver) + assert called == [] + + +# --------------------------------------------------------------------------- +# render_inline +# --------------------------------------------------------------------------- + + +def test_render_inline_substitutes_variables(): + result = render_inline("Hello ${name}.", {"name": "Bob"}) + assert result == "Hello Bob." + + +def test_render_inline_missing_variable_shows_placeholder(): + result = render_inline("Hello ${name}.", {}) + assert result == "Hello ." + + +def test_render_inline_uses_resolver(): + result = render_inline("Memory: ${draft_memory}", {}, resolve_key) + assert result == "Memory: resolved(draft_memory)" + + +def test_render_inline_escaped_dollar(): + result = render_inline("Price: $$5", {}) + assert result == "Price: $5" diff --git a/ddev/tests/ai/react/__init__.py b/ddev/tests/ai/react/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/react/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/react/test_process.py b/ddev/tests/ai/react/test_process.py new file mode 100644 index 0000000000000..e29e6af80b70a --- /dev/null +++ b/ddev/tests/ai/react/test_process.py @@ -0,0 +1,646 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import asyncio +from typing import Any + +import pytest + +from ddev.ai.agent.base import BaseAgent +from ddev.ai.agent.exceptions import AgentConnectionError +from ddev.ai.agent.types import AgentResponse, ContextUsage, StopReason, TokenUsage, ToolCall, ToolResultMessage +from ddev.ai.callbacks.callbacks import Callbacks, CallbackSet +from ddev.ai.react.process import ReActProcess +from ddev.ai.react.types import ReActResult +from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.registry import ToolRegistry + +_TOOL_RESULT_DATA: str = "ok" + +# --------------------------------------------------------------------------- +# Mock helpers +# --------------------------------------------------------------------------- + + +class MockAgent(BaseAgent[Any]): + """Minimal BaseAgent implementation that replays a fixed list of responses.""" + + def __init__(self, responses: list[AgentResponse]) -> None: + super().__init__(name="mock", system_prompt="", tools=ToolRegistry([])) + self._responses = iter(responses) + self.send_calls: list[str | list[ToolResultMessage]] = [] + self.compact_calls: int = 0 + self.compact_preserving_turn_calls: int = 0 + self.compact_response: AgentResponse | None = None + self.compact_token_response: AgentResponse | None = None + self.reset_calls: int = 0 + + async def send( + self, + content: str | list[ToolResultMessage], + allowed_tools: list[str] | None = None, + ) -> AgentResponse: + self.send_calls.append(content) + return next(self._responses) + + async def compact(self) -> AgentResponse | None: + self.compact_calls += 1 + return self.compact_response + + async def compact_preserving_last_turn(self) -> AgentResponse | None: + self.compact_preserving_turn_calls += 1 + return self.compact_token_response + + def reset(self) -> None: + super().reset() + self.reset_calls += 1 + + +class MockToolRegistry: + """Minimal tool registry that always returns a configurable ToolResult.""" + + def __init__(self, result: ToolResult | None = None) -> None: + self._result = result or ToolResult(success=True, data=_TOOL_RESULT_DATA) + self.run_calls: list[tuple[str, dict]] = [] + + async def run(self, name: str, raw: dict[str, object]) -> ToolResult: + self.run_calls.append((name, raw)) + return self._result + + +class RaisingToolRegistry: + """Registry that always raises a given exception from run().""" + + def __init__(self, exc: BaseException) -> None: + self._exc = exc + self.run_calls: list[tuple[str, dict]] = [] + + async def run(self, name: str, raw: dict[str, object]) -> ToolResult: + self.run_calls.append((name, raw)) + raise self._exc + + +class PerToolRegistry: + """Registry that dispatches per tool name, raising or returning per configured behavior.""" + + def __init__(self, behaviors: dict[str, ToolResult | BaseException]) -> None: + self._behaviors = behaviors + self.run_calls: list[tuple[str, dict]] = [] + + async def run(self, name: str, raw: dict[str, object]) -> ToolResult: + self.run_calls.append((name, raw)) + behavior = self._behaviors[name] + if isinstance(behavior, BaseException): + raise behavior + return behavior + + +class CallbackRecorder: + """Test helper that wires a CallbackSet to record all lifecycle events.""" + + def __init__(self) -> None: + self.agent_responses: list[tuple[AgentResponse, int]] = [] + self.tool_calls_seen: list[tuple[ToolCall, ToolResult, int]] = [] + self.complete_results: list[ReActResult] = [] + self.errors: list[BaseException] = [] + self.before_compacts: int = 0 + self.after_compacts: int = 0 + + self.callback_set = CallbackSet() + + @self.callback_set.on_agent_response + async def _record_response(response: AgentResponse, iteration: int) -> None: + self.agent_responses.append((response, iteration)) + + @self.callback_set.on_tool_call + async def _record_tool_call(tool_call: ToolCall, result: ToolResult, iteration: int) -> None: + self.tool_calls_seen.append((tool_call, result, iteration)) + + @self.callback_set.on_complete + async def _record_complete(result: ReActResult) -> None: + self.complete_results.append(result) + + @self.callback_set.on_error + async def _record_error(error: BaseException) -> None: + self.errors.append(error) + + @self.callback_set.on_before_compact + async def _record_before_compact() -> None: + self.before_compacts += 1 + + @self.callback_set.on_after_compact + async def _record_after_compact() -> None: + self.after_compacts += 1 + + +def make_response( + stop_reason: StopReason, + tool_calls: list[ToolCall] | None = None, + input_tokens: int = 10, + output_tokens: int = 5, + context_usage: ContextUsage | None = None, +) -> AgentResponse: + return AgentResponse( + stop_reason=stop_reason, + text="", + tool_calls=tool_calls or [], + usage=TokenUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_read_input_tokens=0, + cache_creation_input_tokens=0, + context_usage=context_usage, + ), + ) + + +def make_tool_call( + call_id: str = "tc_01", name: str = "read_file", tool_input: dict[str, Any] | None = None +) -> ToolCall: + return ToolCall(id=call_id, name=name, input=tool_input or {}) + + +def make_process( + agent: MockAgent, + registry: MockToolRegistry | None = None, + callbacks: Callbacks | None = None, + compact_threshold_pct: float | None = None, +) -> ReActProcess: + return ReActProcess( + agent=agent, + tool_registry=registry or MockToolRegistry(), + callbacks=callbacks, + compact_threshold_pct=compact_threshold_pct, + ) + + +# --------------------------------------------------------------------------- +# Stop reasons β€” parametrized single-response cases +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("stop_reason", [StopReason.END_TURN, StopReason.MAX_TOKENS, StopReason.OTHER]) +async def test_stop_reason_single_response(stop_reason: StopReason) -> None: + agent = MockAgent([make_response(stop_reason)]) + + result = await make_process(agent).start("Hi") + + assert result.final_response.stop_reason == stop_reason + assert result.iterations == 1 + assert len(agent.send_calls) == 1 + assert agent.send_calls[0] == "Hi" + + +# --------------------------------------------------------------------------- +# Single tool call +# --------------------------------------------------------------------------- + + +async def test_single_tool_call_executes_tool_and_returns() -> None: + tc = make_tool_call("tc_01", "read_file") + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN), + ] + registry = MockToolRegistry() + agent = MockAgent(responses) + + result = await make_process(agent, registry=registry).start("Do something") + + assert result.final_response.stop_reason == StopReason.END_TURN + assert result.iterations == 2 + assert len(registry.run_calls) == 1 + assert registry.run_calls[0][0] == "read_file" + assert len(agent.send_calls) == 2 + assert agent.send_calls[0] == "Do something" + assert isinstance(agent.send_calls[1], list) + assert agent.send_calls[1][0].tool_call_id == "tc_01" + assert agent.send_calls[1][0].result.data == _TOOL_RESULT_DATA + + +# --------------------------------------------------------------------------- +# Multi-tool parallel dispatch +# --------------------------------------------------------------------------- + + +async def test_multi_tool_parallel_dispatches_all() -> None: + tool_calls = [ + make_tool_call("tc_01", "a"), + make_tool_call("tc_02", "b"), + make_tool_call("tc_03", "c"), + ] + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=tool_calls), + make_response(StopReason.END_TURN), + ] + registry = MockToolRegistry() + agent = MockAgent(responses) + + await make_process(agent, registry=registry).start("Do three things") + + assert len(registry.run_calls) == 3 + assert {name for name, _ in registry.run_calls} == {"a", "b", "c"} + assert len(agent.send_calls[1]) == 3 + + +# --------------------------------------------------------------------------- +# Tool exception resilience +# --------------------------------------------------------------------------- + + +async def test_tool_exception_loop_continues_with_failure_result() -> None: + """(a) A raising tool must not abort the loop. (b) Its ToolResultMessage must have success=False.""" + tc = make_tool_call("tc_01", "read_file") + agent = MockAgent( + [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN), + ] + ) + + result = await ReActProcess( + agent=agent, + tool_registry=RaisingToolRegistry(RuntimeError("disk error")), + ).start("Do something") + + assert result.iterations == 2 + assert result.final_response.stop_reason == StopReason.END_TURN + sent_back = agent.send_calls[1] + assert isinstance(sent_back, list) + assert sent_back[0].result.success is False + + +async def test_tool_exception_on_tool_call_callback_fires_with_error_result() -> None: + """(c) on_tool_call must fire even when the tool raised, carrying the failure ToolResult.""" + tc = make_tool_call("tc_01", "read_file") + agent = MockAgent( + [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN), + ] + ) + recorder = CallbackRecorder() + + await ReActProcess( + agent=agent, + tool_registry=RaisingToolRegistry(ValueError("oops")), + callbacks=Callbacks([recorder.callback_set]), + ).start("x") + + assert len(recorder.tool_calls_seen) == 1 + _, error_result, _ = recorder.tool_calls_seen[0] + assert error_result.success is False + + +@pytest.mark.parametrize( + "exc,expected_error", + [ + (RuntimeError("disk error"), "RuntimeError: disk error"), + (ValueError("bad input"), "ValueError: bad input"), + (OSError("file not found"), "OSError: file not found"), + ], +) +async def test_tool_exception_error_message_format(exc: BaseException, expected_error: str) -> None: + """Error string in the failure result must be formatted as 'ExceptionType: message'.""" + tc = make_tool_call() + agent = MockAgent( + [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN), + ] + ) + + await ReActProcess( + agent=agent, + tool_registry=RaisingToolRegistry(exc), + ).start("x") + + sent_back: list[ToolResultMessage] = agent.send_calls[1] + assert sent_back[0].result.error == expected_error + + +async def test_partial_batch_failure_only_affects_raising_tool() -> None: + """In a multi-tool batch, only the raising tool gets success=False; successful tools are unaffected.""" + tc_ok = make_tool_call("tc_01", "read_file") + tc_bad = make_tool_call("tc_02", "write_file") + agent = MockAgent( + [ + make_response(StopReason.TOOL_USE, tool_calls=[tc_ok, tc_bad]), + make_response(StopReason.END_TURN), + ] + ) + registry = PerToolRegistry( + { + "read_file": ToolResult(success=True, data="contents"), + "write_file": RuntimeError("permission denied"), + } + ) + + result = await ReActProcess(agent=agent, tool_registry=registry).start("Do both") + + assert result.iterations == 2 + sent_back: list[ToolResultMessage] = agent.send_calls[1] + assert len(sent_back) == 2 + results = {msg.tool_call_id: msg.result for msg in sent_back} + assert results["tc_01"].success is True + assert results["tc_01"].data == "contents" + assert results["tc_02"].success is False + assert "RuntimeError" in (results["tc_02"].error or "") + + +# --------------------------------------------------------------------------- +# Callbacks fired correctly +# --------------------------------------------------------------------------- + + +async def test_callbacks_invoked_correct_counts() -> None: + tool_calls = [make_tool_call("tc_01"), make_tool_call("tc_02", "grep")] + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=tool_calls), + make_response(StopReason.END_TURN), + ] + expected_result = ToolResult(success=True, data="ok") + registry = MockToolRegistry(result=expected_result) + recorder = CallbackRecorder() + agent = MockAgent(responses) + + result = await make_process(agent, registry=registry, callbacks=Callbacks([recorder.callback_set])).start( + "Run tools" + ) + + assert len(recorder.agent_responses) == 2 + assert recorder.agent_responses[0][1] == 1 + assert recorder.agent_responses[1][1] == 2 + assert len(recorder.tool_calls_seen) == 2 + assert all(iteration == 1 for _, _, iteration in recorder.tool_calls_seen) + assert recorder.tool_calls_seen[0][0] is tool_calls[0] + assert recorder.tool_calls_seen[1][0] is tool_calls[1] + assert recorder.tool_calls_seen[0][1] is expected_result + assert recorder.tool_calls_seen[1][1] is expected_result + assert len(recorder.complete_results) == 1 + assert recorder.complete_results[0] is result + assert len(recorder.errors) == 0 + + +async def test_two_callback_sets_both_notified() -> None: + agent = MockAgent([make_response(StopReason.END_TURN)]) + rec_a, rec_b = CallbackRecorder(), CallbackRecorder() + await make_process(agent, callbacks=Callbacks([rec_a.callback_set, rec_b.callback_set])).start("x") + assert len(rec_a.complete_results) == 1 + assert len(rec_b.complete_results) == 1 + + +# --------------------------------------------------------------------------- +# Error path +# --------------------------------------------------------------------------- + + +class ErrorAgent(BaseAgent[Any]): + def __init__(self) -> None: + super().__init__(name="error", system_prompt="", tools=ToolRegistry([])) + + async def send( + self, content: str | list[ToolResultMessage], allowed_tools: list[str] | None = None + ) -> AgentResponse: + raise AgentConnectionError("network down") + + +async def test_agent_error_notifies_and_reraises() -> None: + recorder = CallbackRecorder() + process = ReActProcess( + agent=ErrorAgent(), + tool_registry=MockToolRegistry(), + callbacks=Callbacks([recorder.callback_set]), + ) + + with pytest.raises(AgentConnectionError): + await process.start("Anything") + + assert len(recorder.errors) == 1 + assert isinstance(recorder.errors[0], AgentConnectionError) + assert len(recorder.complete_results) == 0 + assert len(recorder.agent_responses) == 0 + + +class InterruptAgent(BaseAgent[Any]): + def __init__(self) -> None: + super().__init__(name="interrupt", system_prompt="", tools=ToolRegistry([])) + + async def send( + self, content: str | list[ToolResultMessage], allowed_tools: list[str] | None = None + ) -> AgentResponse: + raise KeyboardInterrupt + + +async def test_keyboard_interrupt_notifies_and_reraises() -> None: + recorder = CallbackRecorder() + process = ReActProcess( + agent=InterruptAgent(), + tool_registry=MockToolRegistry(), + callbacks=Callbacks([recorder.callback_set]), + ) + + with pytest.raises(KeyboardInterrupt): + await process.start("Anything") + + assert len(recorder.errors) == 1 + assert isinstance(recorder.errors[0], KeyboardInterrupt) + assert len(recorder.complete_results) == 0 + + +class CancelledAgent(BaseAgent[Any]): + def __init__(self) -> None: + super().__init__(name="cancelled", system_prompt="", tools=ToolRegistry([])) + + async def send( + self, content: str | list[ToolResultMessage], allowed_tools: list[str] | None = None + ) -> AgentResponse: + raise asyncio.CancelledError + + +async def test_cancelled_error_notifies_and_reraises() -> None: + recorder = CallbackRecorder() + process = ReActProcess( + agent=CancelledAgent(), + tool_registry=MockToolRegistry(), + callbacks=Callbacks([recorder.callback_set]), + ) + + with pytest.raises(asyncio.CancelledError): + await process.start("Anything") + + assert len(recorder.errors) == 1 + assert isinstance(recorder.errors[0], asyncio.CancelledError) + assert len(recorder.complete_results) == 0 + + +# --------------------------------------------------------------------------- +# Token accumulation +# --------------------------------------------------------------------------- + + +async def test_total_tokens_summed_across_iterations() -> None: + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[make_tool_call()], input_tokens=100, output_tokens=50), + make_response(StopReason.END_TURN, input_tokens=200, output_tokens=80), + ] + agent = MockAgent(responses) + + result = await make_process(agent).start("Task") + + assert result.total_input_tokens == 300 + assert result.total_output_tokens == 130 + assert result.iterations == 2 + + +async def test_tool_result_tokens_included_in_total_tokens() -> None: + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[make_tool_call()], input_tokens=100, output_tokens=50), + make_response(StopReason.END_TURN, input_tokens=200, output_tokens=80), + ] + agent = MockAgent(responses) + registry = MockToolRegistry(ToolResult(success=True, data="ok", total_input_tokens=30, total_output_tokens=10)) + + result = await make_process(agent, registry=registry).start("Task") + + assert result.total_input_tokens == 330 + assert result.total_output_tokens == 140 + + +# --------------------------------------------------------------------------- +# Context usage propagation β€” parametrized None vs present +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "context_usage", + [ + None, + ContextUsage(window_size=200_000, used_tokens=50_000), + ], +) +async def test_context_usage_propagated(context_usage: ContextUsage | None) -> None: + agent = MockAgent([make_response(StopReason.END_TURN, context_usage=context_usage)]) + + result = await make_process(agent).start("Hi") + + assert result.context_usage is context_usage + assert result.final_response.usage.context_usage is context_usage + + +# --------------------------------------------------------------------------- +# reset() and compact() β€” delegation +# --------------------------------------------------------------------------- + + +async def test_reset_delegates_to_agent() -> None: + agent = MockAgent([]) + make_process(agent).reset() + assert agent.reset_calls == 1 + assert agent.history == [] + + +async def test_compact_delegates_to_agent_returns_zero_when_no_op() -> None: + agent = MockAgent([]) # compact_response is None β€” no compaction occurred + compact_in, compact_out = await make_process(agent).compact() + assert agent.compact_calls == 1 + assert compact_in == 0 + assert compact_out == 0 + + +async def test_compact_returns_tokens_when_compaction_occurs() -> None: + agent = MockAgent([]) + agent.compact_response = make_response(StopReason.END_TURN, input_tokens=40, output_tokens=15) + compact_in, compact_out = await make_process(agent).compact() + assert compact_in == 40 + assert compact_out == 15 + + +async def test_compact_fires_before_and_after_callbacks() -> None: + agent = MockAgent([]) + recorder = CallbackRecorder() + await make_process(agent, callbacks=Callbacks([recorder.callback_set])).compact() + assert recorder.before_compacts == 1 + assert recorder.after_compacts == 1 + + +# --------------------------------------------------------------------------- +# Auto-compact inside the ReAct loop +# --------------------------------------------------------------------------- + + +def make_context_usage(pct: float, window: int = 200_000) -> ContextUsage: + return ContextUsage(window_size=window, used_tokens=int(window * pct / 100)) + + +async def test_auto_compact_triggers_when_threshold_exceeded() -> None: + tc = make_tool_call() + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN, context_usage=make_context_usage(80.0)), + ] + agent = MockAgent(responses) + await make_process(agent, compact_threshold_pct=75.0).start("task") + assert agent.compact_calls == 1 + + +async def test_auto_compact_fires_callbacks() -> None: + tc = make_tool_call() + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN, context_usage=make_context_usage(80.0)), + ] + agent = MockAgent(responses) + recorder = CallbackRecorder() + await make_process(agent, callbacks=Callbacks([recorder.callback_set]), compact_threshold_pct=75.0).start("task") + assert recorder.before_compacts == 1 + assert recorder.after_compacts == 1 + + +async def test_auto_compact_does_not_trigger_below_threshold() -> None: + tc = make_tool_call() + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN, context_usage=make_context_usage(50.0)), + ] + agent = MockAgent(responses) + await make_process(agent, compact_threshold_pct=75.0).start("task") + assert agent.compact_preserving_turn_calls == 0 + + +async def test_auto_compact_disabled_when_threshold_is_none() -> None: + tc = make_tool_call() + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN, context_usage=make_context_usage(99.9)), + ] + agent = MockAgent(responses) + await make_process(agent, compact_threshold_pct=None).start("task") + assert agent.compact_preserving_turn_calls == 0 + + +async def test_auto_compact_skipped_when_context_usage_is_none() -> None: + tc = make_tool_call() + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[tc]), + make_response(StopReason.END_TURN, context_usage=None), + ] + agent = MockAgent(responses) + await make_process(agent, compact_threshold_pct=75.0).start("task") + assert agent.compact_preserving_turn_calls == 0 + + +async def test_auto_compact_tokens_included_in_result() -> None: + tc = make_tool_call() + responses = [ + make_response(StopReason.TOOL_USE, tool_calls=[tc], input_tokens=100, output_tokens=50), + make_response(StopReason.END_TURN, context_usage=make_context_usage(80.0), input_tokens=200, output_tokens=80), + ] + agent = MockAgent(responses) + agent.compact_response = make_response(StopReason.END_TURN, input_tokens=30, output_tokens=10) + + result = await make_process(agent, compact_threshold_pct=75.0).start("task") + + assert result.total_input_tokens == 100 + 200 + 30 + assert result.total_output_tokens == 50 + 80 + 10 diff --git a/ddev/tests/ai/tools/__init__.py b/ddev/tests/ai/tools/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/tools/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/tools/agents/__init__.py b/ddev/tests/ai/tools/agents/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/tools/agents/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/tools/agents/test_agent_logger.py b/ddev/tests/ai/tools/agents/test_agent_logger.py new file mode 100644 index 0000000000000..6f15343fad07c --- /dev/null +++ b/ddev/tests/ai/tools/agents/test_agent_logger.py @@ -0,0 +1,133 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import json + +import pytest + +from ddev.ai.agent.types import AgentResponse, StopReason, TokenUsage, ToolCall +from ddev.ai.tools.agents.agent_logger import AgentLogger +from ddev.ai.tools.core.types import ToolResult + + +def make_response(text: str = "", stop_reason: StopReason = StopReason.END_TURN) -> AgentResponse: + return AgentResponse( + stop_reason=stop_reason, + text=text, + tool_calls=[], + usage=TokenUsage(input_tokens=10, output_tokens=5, cache_read_input_tokens=0, cache_creation_input_tokens=0), + ) + + +def read_events(log_path) -> list[dict]: + return [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +# --------------------------------------------------------------------------- +# File mechanics +# --------------------------------------------------------------------------- + + +def test_log_entries_are_valid_jsonl_with_timestamp(tmp_path): + log_path = tmp_path / "log.jsonl" + logger = AgentLogger(log_path) + logger.log_start(system_prompt="sys", prompt="go", tools=["read_file"]) + logger.log_finish(success=True, iterations=1) + logger.close() + + events = read_events(log_path) + assert len(events) == 2 + assert all("ts" in e for e in events) + assert events[0]["event"] == "start" + assert events[1]["event"] == "finish" + + +def test_flush_after_each_write(tmp_path): + log_path = tmp_path / "log.jsonl" + logger = AgentLogger(log_path) + logger.log_start(system_prompt="s", prompt="p", tools=[]) + # A second file handle reads the line without closing the logger first + assert log_path.read_text(encoding="utf-8").strip() != "" + logger.close() + + +def test_close_is_idempotent_and_prevents_further_writes(tmp_path): + log_path = tmp_path / "log.jsonl" + logger = AgentLogger(log_path) + logger.log_start(system_prompt="s", prompt="p", tools=[]) + logger.close() + logger.close() # must not raise + logger.log_finish(success=False) # must not write + assert len(read_events(log_path)) == 1 + + +def test_reopening_same_path_appends_start_run_delimiter(tmp_path): + log_path = tmp_path / "log.jsonl" + + logger = AgentLogger(log_path) + logger.log_start(system_prompt="s", prompt="first", tools=[]) + logger.log_finish(success=True) + logger.close() + + logger = AgentLogger(log_path) + logger.log_start(system_prompt="s", prompt="second", tools=[]) + logger.log_finish(success=True) + logger.close() + + events = read_events(log_path) + assert [event["event"] for event in events] == ["start", "finish", "start", "finish"] + assert events[0]["prompt"] == "first" + assert events[2]["prompt"] == "second" + + +def test_constructor_requires_existing_parent(tmp_path): + with pytest.raises(OSError): + AgentLogger(tmp_path / "doesnotexist" / "log.jsonl") + + +def test_non_serializable_values_use_str_repr(tmp_path): + log_path = tmp_path / "log.jsonl" + logger = AgentLogger(log_path) + + class Unserializable: + def __repr__(self): + return "" + + logger.log_finish(success=True, extra=Unserializable()) + logger.close() + + assert read_events(log_path)[0]["extra"] == "" + + +# --------------------------------------------------------------------------- +# Callbacks wiring +# --------------------------------------------------------------------------- + + +async def test_build_callbacks_fires_all_event_types(tmp_path): + log_path = tmp_path / "log.jsonl" + logger = AgentLogger(log_path) + callbacks = logger.build_callbacks() + + tool_call = ToolCall(id="tc1", name="read_file", input={"path": "/f"}) + tool_result = ToolResult(success=True, data="content") + + await callbacks.fire_before_agent_send(2) + await callbacks.fire_agent_response(make_response("hi"), 2) + await callbacks.fire_tool_call(tool_call, tool_result, 2) + await callbacks.fire_before_compact() + await callbacks.fire_after_compact() + await callbacks.fire_error(ValueError("oops")) + logger.close() + + events = {e["event"]: e for e in read_events(log_path)} + + assert events["before_agent_send"]["iter"] == 2 + assert events["agent_response"]["text"] == "hi" + assert events["agent_response"]["iter"] == 2 + assert events["tool_call"]["tool_call_id"] == "tc1" + assert events["tool_call"]["result"]["success"] is True + assert "before_compact" in events + assert "after_compact" in events + assert "ValueError" in events["error"]["exception"] diff --git a/ddev/tests/ai/tools/agents/test_spawn_subagent.py b/ddev/tests/ai/tools/agents/test_spawn_subagent.py new file mode 100644 index 0000000000000..f3881b539b805 --- /dev/null +++ b/ddev/tests/ai/tools/agents/test_spawn_subagent.py @@ -0,0 +1,378 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import asyncio +import json +from pathlib import Path +from typing import Any + +import pytest +from anthropic.types import ToolParam + +from ddev.ai.agent.base import BaseAgent +from ddev.ai.agent.build import SubagentBuilder +from ddev.ai.agent.exceptions import AgentError +from ddev.ai.agent.types import AgentResponse, StopReason, TokenUsage, ToolCall, ToolResultMessage +from ddev.ai.tools.agents.spawn_subagent import SpawnSubagentInput, SpawnSubagentTool +from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.registry import ToolRegistry + +# --------------------------------------------------------------------------- +# Mock helpers +# --------------------------------------------------------------------------- + + +class MockAgent(BaseAgent[Any]): + def __init__(self, responses: list[AgentResponse]) -> None: + super().__init__("mock", "", ToolRegistry([])) + self._responses = list(responses) + self._index = 0 + + async def send( + self, + content: str | list[ToolResultMessage], + allowed_tools: list[str] | None = None, + ) -> AgentResponse: + response = self._responses[self._index] + self._index += 1 + return response + + def reset(self) -> None: + self._history = [] + + async def compact(self) -> AgentResponse | None: + return None + + async def compact_preserving_last_turn(self) -> AgentResponse | None: + return None + + +class _RaisingAgent(BaseAgent[Any]): + """Raises a fixed exception on every send() call.""" + + def __init__(self, exc: BaseException) -> None: + super().__init__("raising", "", ToolRegistry([])) + self._exc = exc + + async def send( + self, + content: str | list[ToolResultMessage], + allowed_tools: list[str] | None = None, + ) -> AgentResponse: + raise self._exc + + def reset(self) -> None: + self._history = [] + + async def compact(self) -> AgentResponse | None: + return None + + async def compact_preserving_last_turn(self) -> AgentResponse | None: + return None + + +class MockToolRegistry(ToolRegistry): + def __init__(self, result: ToolResult | None = None) -> None: + super().__init__([]) + self._result = result or ToolResult(success=True, data="ok") + + @property + def definitions(self) -> list[ToolParam]: + return [] + + async def run(self, name: str, raw: dict[str, object]) -> ToolResult: + return self._result + + +def make_response( + text: str = "", + stop_reason: StopReason = StopReason.END_TURN, + tool_calls: list[ToolCall] | None = None, +) -> AgentResponse: + return AgentResponse( + stop_reason=stop_reason, + text=text, + tool_calls=tool_calls or [], + usage=TokenUsage(input_tokens=10, output_tokens=5, cache_read_input_tokens=0, cache_creation_input_tokens=0), + ) + + +def make_builder(responses: list[AgentResponse], tool_result: ToolResult | None = None) -> SubagentBuilder: + """Return a builder closure that replays fixed responses.""" + tr = tool_result or ToolResult(success=True, data="ok") + + def builder(system_prompt: str, owner_id: str, tool_names: list[str]) -> tuple[BaseAgent[Any], ToolRegistry]: + return MockAgent(list(responses)), MockToolRegistry(tr) + + return builder + + +def make_tool( + log_dir: Path, + builder: SubagentBuilder, + allowed_tools: list[str] | None = None, + owner_id: str = "parent", +) -> SpawnSubagentTool: + return SpawnSubagentTool( + owner_id=owner_id, + subagent_builder=builder, + allowed_tools=allowed_tools if allowed_tools is not None else ["read_file", "edit_file"], + log_dir=log_dir, + ) + + +def read_events(log_path: Path) -> list[dict]: + return [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +# --------------------------------------------------------------------------- +# Input validation β€” fails before any log file is opened +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "tools,allowed,error_fragment", + [ + (["spawn_subagent"], ["read_file"], "spawn further subagents"), + (["read_file", "edit_file"], ["read_file"], "edit_file"), + ], + ids=["recursive", "disallowed"], +) +async def test_input_validation_fails_before_logging(tmp_path, tools, allowed, error_fragment): + tool = make_tool(tmp_path, make_builder([make_response()]), allowed_tools=allowed) + result = await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=tools, name="x")) + + assert result.success is False + assert error_fragment in result.error + assert "x" in result.error + assert list(tmp_path.glob("*.jsonl")) == [] + assert tool._counter == 0 + + +# --------------------------------------------------------------------------- +# mkdir failure β€” after validation, before counter advances +# --------------------------------------------------------------------------- + + +async def test_mkdir_failure(tmp_path): + blocker = tmp_path / "blocked" + blocker.write_text("I am a file") + log_dir = blocker / "subagents" + + tool = make_tool(log_dir, make_builder([make_response()])) + result = await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="x")) + + assert result.success is False + assert "x" in result.error + assert str(log_dir) in result.error + assert tool._counter == 0 + + +async def test_logger_open_failure_returns_tool_result(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def builder( + system_prompt: str, + owner_id: str, + tool_names: list[str], + ) -> tuple[BaseAgent[Any], ToolRegistry]: + raise AssertionError("builder should not be called") + + def failing_logger(log_path: Path) -> None: + raise OSError("permission denied") + + monkeypatch.setattr("ddev.ai.tools.agents.spawn_subagent.AgentLogger", failing_logger) + + tool = make_tool(tmp_path, builder, allowed_tools=[]) + result = await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="x")) + + assert result.success is False + assert "x" in result.error + assert "cannot open log file" in result.error + assert str(tmp_path / "001-x.jsonl") in result.error + assert "permission denied" in result.error + assert list(tmp_path.glob("*.jsonl")) == [] + + +# --------------------------------------------------------------------------- +# Happy path +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("name", "expected_log_name"), + [ + ("worker", "001-worker.jsonl"), + (None, "001-unnamed.jsonl"), + ("", "001-unnamed.jsonl"), + ], + ids=["named", "none", "empty"], +) +async def test_happy_path(tmp_path: Path, name: str | None, expected_log_name: str) -> None: + tool = make_tool(tmp_path, make_builder([make_response(text="ok")])) + result = await tool(SpawnSubagentInput(system_prompt="sys", prompt="do it", tools=[], name=name)) + + assert result.success is True + assert result.data == "ok" + assert result.total_input_tokens == 10 + assert result.total_output_tokens == 5 + + events = read_events(tmp_path / expected_log_name) + assert events[0]["event"] == "start" + assert events[-1]["event"] == "finish" + assert events[-1]["success"] is True + + +async def test_multi_iteration_wires_callbacks(tmp_path): + """Proves AgentLogger callbacks are wired: a subagent tool call produces a tool_call log event.""" + tool_call = ToolCall(id="tc1", name="read_file", input={"path": "/f"}) + tool = make_tool( + tmp_path, + make_builder( + [make_response(stop_reason=StopReason.TOOL_USE, tool_calls=[tool_call]), make_response(text="done")], + tool_result=ToolResult(success=True, data="content"), + ), + allowed_tools=["read_file"], + ) + + result = await tool(SpawnSubagentInput(system_prompt="sys", prompt="go", tools=["read_file"])) + + assert result.success is True + assert result.data == "done" + assert "tool_call" in [e["event"] for e in read_events(tmp_path / "001-unnamed.jsonl")] + + +async def test_max_tokens_response_prefixed(tmp_path): + tool = make_tool(tmp_path, make_builder([make_response(text="partial", stop_reason=StopReason.MAX_TOKENS)])) + result = await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="mt")) + + assert result.success is True + assert result.data.startswith("[SUBAGENT HIT MAX_TOKENS β€” RESPONSE MAY BE TRUNCATED]") + assert "partial" in result.data + + finish = next(e for e in read_events(tmp_path / "001-mt.jsonl") if e["event"] == "finish") + assert finish["stop_reason"] == "max_tokens" + + +# --------------------------------------------------------------------------- +# Failure paths +# --------------------------------------------------------------------------- + + +async def test_builder_failure(tmp_path): + def failing_builder( + system_prompt: str, + owner_id: str, + tool_names: list[str], + ) -> tuple[BaseAgent[Any], ToolRegistry]: + raise ValueError("boom") + + tool = SpawnSubagentTool(owner_id="parent", subagent_builder=failing_builder, allowed_tools=[], log_dir=tmp_path) + result = await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="fail")) + + assert result.success is False + assert "fail" in result.error and "ValueError" in result.error and "boom" in result.error + + events = read_events(tmp_path / "001-fail.jsonl") + assert [e["event"] for e in events] == ["start", "finish"] + assert events[-1]["success"] is False + + +async def test_react_process_failure(tmp_path): + def builder( + system_prompt: str, + owner_id: str, + tool_names: list[str], + ) -> tuple[BaseAgent[Any], ToolRegistry]: + return _RaisingAgent(AgentError("rate limit")), MockToolRegistry() + + tool = make_tool(tmp_path, builder) + result = await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="rl")) + + assert result.success is False + assert "rl" in result.error and "AgentError" in result.error + + names = [e["event"] for e in read_events(tmp_path / "001-rl.jsonl")] + assert "error" in names and "finish" in names + assert names.index("error") < names.index("finish") + assert next(e for e in read_events(tmp_path / "001-rl.jsonl") if e["event"] == "finish")["success"] is False + + +async def test_finally_close_runs_on_base_exception(tmp_path): + """KeyboardInterrupt propagates but logger.close() still runs via finally.""" + + def builder( + system_prompt: str, + owner_id: str, + tool_names: list[str], + ) -> tuple[BaseAgent[Any], ToolRegistry]: + return _RaisingAgent(KeyboardInterrupt()), MockToolRegistry() + + tool = make_tool(tmp_path, builder) + + with pytest.raises(KeyboardInterrupt): + await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="ki")) + + names = [e["event"] for e in read_events(tmp_path / "001-ki.jsonl")] + assert "error" in names + assert "finish" not in names + + +# --------------------------------------------------------------------------- +# Counter and log file naming +# --------------------------------------------------------------------------- + + +async def test_counter_increments_per_invocation(tmp_path): + tool = make_tool(tmp_path, make_builder([make_response(text="r1"), make_response(text="r2")])) + + await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="a")) + await tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name="b")) + + assert (tmp_path / "001-a.jsonl").exists() + assert (tmp_path / "002-b.jsonl").exists() + + +async def test_parallel_spawns_get_distinct_counters(tmp_path): + owner_ids: list[str] = [] + + def recording_builder( + system_prompt: str, + owner_id: str, + tool_names: list[str], + ) -> tuple[BaseAgent[Any], ToolRegistry]: + owner_ids.append(owner_id) + return MockAgent([make_response(text="ok")]), MockToolRegistry() + + tool = SpawnSubagentTool(owner_id="parent", subagent_builder=recording_builder, allowed_tools=[], log_dir=tmp_path) + results = await asyncio.gather( + *[tool(SpawnSubagentInput(system_prompt="s", prompt="p", tools=[], name=n)) for n in ["x", "y", "z"]] + ) + + assert all(r.success for r in results) + assert len(list(tmp_path.glob("*.jsonl"))) == 3 + assert len(set(owner_ids)) == 3 + + +# --------------------------------------------------------------------------- +# Pydantic input validation (via BaseTool.run) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "raw", + [ + {"prompt": "x"}, + {"system_prompt": "s", "prompt": "p", "tools": [], "bad_field": True}, + {"system_prompt": "s", "prompt": "p", "tools": [], "name": "../oops"}, + ], + ids=["missing_field", "extra_field", "unsafe_name"], +) +async def test_pydantic_rejects_invalid_input(raw: dict[str, object]) -> None: + tool = SpawnSubagentTool( + owner_id="p", + subagent_builder=make_builder([make_response()]), + allowed_tools=[], + log_dir=Path("/tmp"), + ) + result = await tool.run(raw) + assert result.success is False diff --git a/ddev/tests/ai/tools/core/__init__.py b/ddev/tests/ai/tools/core/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/tools/core/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/tools/core/test_base.py b/ddev/tests/ai/tools/core/test_base.py new file mode 100644 index 0000000000000..35e94f750a69e --- /dev/null +++ b/ddev/tests/ai/tools/core/test_base.py @@ -0,0 +1,226 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +import pytest +from pydantic import Field + +from ddev.ai.tools.core.base import BaseTool, BaseToolInput, _get_input_type +from ddev.ai.tools.core.types import ToolResult + +# --------------------------------------------------------------------------- +# Minimal concrete tools used across tests +# --------------------------------------------------------------------------- + + +class SimpleInput(BaseToolInput): + message: Annotated[str, Field(description="A message to echo")] + + +class FullInput(BaseToolInput): + required_str: Annotated[str, Field(description="A required string")] + optional_int: Annotated[int | None, Field(description="An optional integer")] = None + flag: Annotated[bool, Field(description="A boolean flag")] = False + + +class EchoTool(BaseTool[SimpleInput]): + """Echo the message back.""" + + @property + def name(self) -> str: + return "echo" + + async def __call__(self, tool_input: SimpleInput) -> ToolResult: + return ToolResult(success=True, data=tool_input.message) + + +class FailingTool(BaseTool[SimpleInput]): + """A tool that always raises.""" + + @property + def name(self) -> str: + return "failing" + + async def __call__(self, tool_input: SimpleInput) -> ToolResult: + raise RuntimeError("something went wrong") + + +class FullInputTool(BaseTool[FullInput]): + """Tool using FullInput.""" + + @property + def name(self) -> str: + return "full" + + async def __call__(self, tool_input: FullInput) -> ToolResult: + return ToolResult(success=True) + + +# --------------------------------------------------------------------------- +# BaseToolInput.to_input_schema() +# --------------------------------------------------------------------------- + + +def test_to_input_schema_type_and_required(): + schema = SimpleInput.to_input_schema() + assert schema["type"] == "object" + assert schema["required"] == ["message"] + + +def test_to_input_schema_field_description(): + schema = SimpleInput.to_input_schema() + assert schema["properties"]["message"]["description"] == "A message to echo" + assert schema["properties"]["message"]["type"] == "string" + + +def test_to_input_schema_no_title_keys(): + schema = FullInput.to_input_schema() + assert "title" not in schema + for prop in schema["properties"].values(): + assert "title" not in prop + + +def test_to_input_schema_additional_properties_false(): + assert SimpleInput.to_input_schema().get("additionalProperties") is False + + +def test_to_input_schema_optional_fields_not_required(): + schema = FullInput.to_input_schema() + assert "required_str" in schema["required"] + assert "optional_int" not in schema["required"] + assert "flag" not in schema["required"] + + +def test_to_input_schema_anyof_flattened_for_optional_int(): + schema = FullInput.to_input_schema() + prop = schema["properties"]["optional_int"] + assert "anyOf" not in prop + assert prop["type"] == "integer" + + +def test_to_input_schema_all_optional_no_required_key(): + class AllOptional(BaseToolInput): + x: Annotated[str, Field(description="x")] = "default" + y: Annotated[int, Field(description="y")] = 0 + + schema = AllOptional.to_input_schema() + assert "required" not in schema + + +# --------------------------------------------------------------------------- +# _get_input_type +# --------------------------------------------------------------------------- + + +def test_get_input_type_returns_correct_input_type(): + class ChildTool(EchoTool): + pass + + assert _get_input_type(EchoTool) is SimpleInput + assert _get_input_type(FullInputTool) is FullInput + assert _get_input_type(ChildTool) is SimpleInput + + +def test_get_input_type_unparameterized_subclass(): + # A class that extends BaseTool without a type argument cannot be resolved + class BareSubclass(BaseTool): # type: ignore[type-arg] + @property + def name(self) -> str: + return "bare" + + async def __call__(self, tool_input) -> ToolResult: # type: ignore[override] + return ToolResult(success=True) + + with pytest.raises(TypeError, match="BareSubclass"): + _get_input_type(BareSubclass) + + +def test_resolves_through_intermediate_generic(): + # Simulates the CmdTool[TInput] -> BaseTool[TInput] pattern + class IntermediateTool[T](BaseTool[T]): + @property + def name(self) -> str: + return "intermediate" + + async def __call__(self, tool_input: T) -> ToolResult: # type: ignore[override] + return ToolResult(success=True) + + class ConcreteTool(IntermediateTool[SimpleInput]): + pass + + assert _get_input_type(ConcreteTool) is SimpleInput + + +# --------------------------------------------------------------------------- +# BaseTool +# --------------------------------------------------------------------------- + + +@pytest.fixture +def echo_tool() -> EchoTool: + return EchoTool() + + +@pytest.fixture +def failing_tool() -> FailingTool: + return FailingTool() + + +def test_build_tool(echo_tool: EchoTool): + assert echo_tool.name == "echo" + assert echo_tool.description == "Echo the message back." + assert echo_tool.input_schema == SimpleInput.to_input_schema() + assert echo_tool.definition == { + "name": "echo", + "description": "Echo the message back.", + "input_schema": SimpleInput.to_input_schema(), + } + + +def test_build_tool_no_docstring(): + class NoDocstringTool(BaseTool[SimpleInput]): + @property + def name(self) -> str: + return "nodoc" + + async def __call__(self, tool_input: SimpleInput) -> ToolResult: + return ToolResult(success=True) + + assert NoDocstringTool().description == "" + + +# --- run(): happy path --- + + +async def test_run_valid_input_returns_success(echo_tool: EchoTool): + result = await echo_tool.run({"message": "hello"}) + assert result.success is True + assert result.data == "hello" + + +# --- run(): input validation failures --- + + +@pytest.mark.parametrize( + "raw", + [ + {}, + {"message": "hi", "extra": "oops"}, + ], +) +async def test_run_invalid_input_returns_failure(echo_tool: EchoTool, raw: dict): + result = await echo_tool.run(raw) + assert result.success is False + assert result.error is not None + + +# --- run(): __call__ exception handling --- + + +async def test_run_captures_exception_from_call(failing_tool: FailingTool): + result = await failing_tool.run({"message": "boom"}) + assert isinstance(result, ToolResult) + assert result.success is False + assert "RuntimeError" in result.error + assert "something went wrong" in result.error diff --git a/ddev/tests/ai/tools/core/test_truncation.py b/ddev/tests/ai/tools/core/test_truncation.py new file mode 100644 index 0000000000000..b4f1478dd486f --- /dev/null +++ b/ddev/tests/ai/tools/core/test_truncation.py @@ -0,0 +1,176 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ddev.ai.tools.core.truncation import extract_error_lines, truncate + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def make_content(n_chars: int, char: str = "x") -> str: + """Build a string of exactly n_chars characters.""" + return char * n_chars + + +def make_content_with_error(total: int, error_line: str = "ERROR: something failed") -> str: + """Build a string longer than MAX_CHARS with an error line in the middle.""" + half = total // 2 + padding = "x" * 80 + "\n" + before = padding * (half // len(padding) + 1) + after = padding * (half // len(padding) + 1) + return before[:half] + "\n" + error_line + "\n" + after[:half] + + +# --------------------------------------------------------------------------- +# extract_error_lines +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "keyword", + [ + "ERROR", + "FAILED", + "Exception", + "Traceback", + "fatal", + "panic", + "error", + "failed", + "exception", + "traceback", + "FATAL", + "PANIC", + ], +) +def test_detects_each_error_keyword_case_insensitive(keyword: str): + lines = ["normal line", f"this is a {keyword} here", "another normal"] + result = extract_error_lines(lines) + assert len(result) == 1 + + +def test_extract_error_lines_returns_correct_index(): + lines = ["ok", "ok", "ERROR: boom", "ok"] + result = extract_error_lines(lines) + assert result[0][0] == 2 + + +def test_extract_error_lines_returns_multiple_matching_lines(): + lines = ["ERROR: first", "normal", "Traceback: second"] + result = extract_error_lines(lines) + assert len(result) == 2 + + +def test_extract_error_lines_clean_content_returns_empty(): + lines = ["everything", "is", "fine"] + assert extract_error_lines(lines) == [] + + +def test_extract_error_lines_empty_input_returns_empty(): + assert extract_error_lines([]) == [] + + +# --------------------------------------------------------------------------- +# truncate β€” max_char limit works as expected +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "content,expected_truncated", + [ + ("hello world", False), + (make_content(50), False), + (make_content(50 + 1), True), + ], +) +def test_max_char_limit(content, expected_truncated): + result = truncate(content, max_chars=50) + assert result.truncated is expected_truncated + if not expected_truncated: + assert result.meta is None + + +# --------------------------------------------------------------------------- +# truncate β€” basic head+tail (no errors) +# --------------------------------------------------------------------------- + + +@pytest.fixture +def content() -> str: + return make_content(500) + + +@pytest.fixture +def max_chars() -> int: + return 200 + + +def test_truncate_basic_head_tail_no_errors(content: str, max_chars: int): + result = truncate(content, max_chars=max_chars) + assert len(result.output) <= max_chars + 150 # gap marker adds ~50 chars + assert result.truncated is True + + assert result.meta is not None + assert result.meta.total_size == len(content) + assert result.meta.shown_size == len(result.output) + assert result.meta.truncated_size == result.meta.total_size - result.meta.shown_size + + assert "[..." in result.output and "characters removed" in result.output + + assert str(result.meta.shown_size) in result.meta.hint + assert str(result.meta.total_size) in result.meta.hint + + +def test_truncate_starts_and_ends_with_content(content: str, max_chars: int): + content = "START" + content + "END" + result = truncate(content, max_chars=max_chars) + assert result.output.startswith("START") + assert result.output.endswith("END") + + +# --------------------------------------------------------------------------- +# truncate β€” error-aware preservation +# --------------------------------------------------------------------------- + + +@pytest.fixture +def content_with_error() -> str: + padding = "x" * 80 + "\n" + middle_error = "ERROR: critical failure detected\n" + return padding * 5 + middle_error + padding * 5 + + +def test_error_aware_preservation(content_with_error: str, max_chars: int): + result = truncate(content_with_error, max_chars=max_chars) + assert "ERROR: critical failure detected" in result.output + assert "errors preserved" in result.output + assert "could not be preserved" not in result.meta.hint + + +@pytest.mark.parametrize("keyword", ["FAILED", "Exception", "Traceback", "fatal", "panic"]) +def test_each_error_keyword_triggers_preservation(keyword: str): + padding = "y" * 80 + "\n" + content = padding * 5 + f"{keyword}: something bad\n" + padding * 5 + result = truncate(content, max_chars=200) + assert keyword in result.output + + +# --------------------------------------------------------------------------- +# truncate β€” errors too large to preserve (fallback to plain head+tail) +# --------------------------------------------------------------------------- + + +def test_falls_back_to_plain_truncation_when_error_snippet_exceeds_budget(): + max_chars = 200 + error_lines = "\n".join([f"ERROR: line {i}" for i in range(50)]) # ~700 chars of errors + padding = "x" * 80 + "\n" + content = padding * 5 + error_lines + padding * 5 + + result = truncate(content, max_chars=max_chars) + + assert result.truncated is True + assert "could not be preserved" in result.meta.hint + assert "errors preserved" not in result.output diff --git a/ddev/tests/ai/tools/fs/__init__.py b/ddev/tests/ai/tools/fs/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/tools/fs/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/tools/fs/conftest.py b/ddev/tests/ai/tools/fs/conftest.py new file mode 100644 index 0000000000000..defedae7d9269 --- /dev/null +++ b/ddev/tests/ai/tools/fs/conftest.py @@ -0,0 +1,63 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import pytest + +from ddev.ai.tools.fs.append_file import AppendFileTool +from ddev.ai.tools.fs.create_file import CreateFileTool +from ddev.ai.tools.fs.edit_file import EditFileTool +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.fs.mkdir import MkdirTool +from ddev.ai.tools.fs.read_file import ReadFileTool + +OWNER_ID = "test-agent" + + +@pytest.fixture +def owner_id() -> str: + return OWNER_ID + + +@pytest.fixture +def permissive_policy(tmp_path) -> FileAccessPolicy: + return FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + + +@pytest.fixture +def registry(permissive_policy: FileAccessPolicy) -> FileRegistry: + return FileRegistry(policy=permissive_policy) + + +@pytest.fixture +def read_tool(registry: FileRegistry, owner_id: str) -> ReadFileTool: + return ReadFileTool(registry, owner_id) + + +@pytest.fixture +def create_tool(registry: FileRegistry, owner_id: str) -> CreateFileTool: + return CreateFileTool(registry, owner_id) + + +@pytest.fixture +def edit_tool(registry: FileRegistry, owner_id: str) -> EditFileTool: + return EditFileTool(registry, owner_id) + + +@pytest.fixture +def append_tool(registry: FileRegistry, owner_id: str) -> AppendFileTool: + return AppendFileTool(registry, owner_id) + + +@pytest.fixture +def mkdir_tool(permissive_policy: FileAccessPolicy) -> MkdirTool: + return MkdirTool(permissive_policy) + + +@pytest.fixture +async def known_file(tmp_path, create_tool: CreateFileTool): + """A temp file registered in the registry via create.""" + f = tmp_path / "file.txt" + await create_tool.run({"path": str(f), "content": "line one\nline two\nline three\n"}) + return f diff --git a/ddev/tests/ai/tools/fs/test_append_file.py b/ddev/tests/ai/tools/fs/test_append_file.py new file mode 100644 index 0000000000000..7b0b6cfb33d91 --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_append_file.py @@ -0,0 +1,94 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from unittest.mock import patch + +import pytest + +from ddev.ai.tools.fs.append_file import AppendFileTool +from ddev.ai.tools.fs.create_file import CreateFileTool +from ddev.ai.tools.fs.file_registry import FileRegistry + +from .conftest import OWNER_ID + + +def test_tool_name(registry: FileRegistry) -> None: + assert AppendFileTool(registry, OWNER_ID).name == "append_file" + + +@pytest.mark.parametrize( + "content,expected_in,expected_not_in", + [ + ("line four\n", "line four\n", None), + ("appended", "three\nappended", None), + ("A\r\nB\r\n", "A\nB\n", "\r"), + ], +) +async def test_append_file_success( + append_tool: AppendFileTool, known_file, content, expected_in, expected_not_in +) -> None: + result = await append_tool.run({"path": str(known_file), "content": content}) + + assert result.success is True + text = known_file.read_text(encoding="utf-8") + assert expected_in in text + if expected_not_in is not None: + assert expected_not_in not in text + + +async def test_append_file_fails_for_unregistered_file(append_tool: AppendFileTool, tmp_path) -> None: + f = tmp_path / "unread.txt" + f.write_text("content", encoding="utf-8") + + result = await append_tool.run({"path": str(f), "content": "more"}) + + assert result.success is False + assert "Not authorized" in result.error + + +@pytest.mark.parametrize( + "initial,appended,expected", + [ + ("no newline", "appended", "no newline\nappended"), + ("", "first line", "first line"), + ], +) +async def test_append_file_separator( + append_tool: AppendFileTool, create_tool: CreateFileTool, tmp_path, initial, appended, expected +) -> None: + f = tmp_path / "file.txt" + await create_tool.run({"path": str(f), "content": initial}) + + result = await append_tool.run({"path": str(f), "content": appended}) + + assert result.success is True + assert f.read_text(encoding="utf-8") == expected + + +async def test_append_file_fails_if_file_changed_externally(append_tool: AppendFileTool, known_file) -> None: + known_file.write_text("externally modified\n", encoding="utf-8") + + result = await append_tool.run({"path": str(known_file), "content": "more"}) + + assert result.success is False + assert "Re-read and retry" in result.error + + +async def test_append_file_updates_registry(append_tool: AppendFileTool, registry: FileRegistry, known_file) -> None: + await append_tool.run({"path": str(known_file), "content": "extra\n"}) + + new_content = known_file.read_text(encoding="utf-8") + assert registry.verify(OWNER_ID, str(known_file), new_content) is True + + +async def test_append_file_oserror_on_write(append_tool: AppendFileTool, registry: FileRegistry, known_file) -> None: + original_content = known_file.read_text(encoding="utf-8") + + with patch("pathlib.Path.write_text", side_effect=PermissionError("permission denied")): + result = await append_tool.run({"path": str(known_file), "content": "new line"}) + + assert result.success is False + assert result.error is not None + # File must be untouched and registry must still reflect the original content + assert known_file.read_text(encoding="utf-8") == original_content + assert registry.verify(OWNER_ID, str(known_file), original_content) is True diff --git a/ddev/tests/ai/tools/fs/test_base.py b/ddev/tests/ai/tools/fs/test_base.py new file mode 100644 index 0000000000000..d7183726c0779 --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_base.py @@ -0,0 +1,157 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Annotated + +import pytest +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.fs.base import FileRegistryTool +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry + +OWNER_ID = "test-agent" + +# --------------------------------------------------------------------------- +# Minimal concrete subclass for testing +# --------------------------------------------------------------------------- + + +class DummyInput(BaseToolInput): + path: Annotated[str, Field(description="Path")] + + +class DummyTool(FileRegistryTool[DummyInput]): + """Dummy tool to test TextEdit base behavior.""" + + @property + def name(self) -> str: + return "dummy" + + async def __call__(self, tool_input: DummyInput) -> ToolResult: + return ToolResult(success=True) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def registry(tmp_path) -> FileRegistry: + return FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) + + +@pytest.fixture +def tool(registry: FileRegistry) -> DummyTool: + return DummyTool(registry, OWNER_ID) + + +# --------------------------------------------------------------------------- +# _read_verified +# --------------------------------------------------------------------------- + + +def test_read_verified_fails_if_not_known(tool: DummyTool, tmp_path) -> None: + path = str(tmp_path / "file.txt") + content, error = tool._read_verified(path) + + assert content == "" + assert error is not None + assert error.success is False + assert "Not authorized" in error.error + + +def test_read_verified_fails_if_file_changed_externally(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: + f = tmp_path / "file.txt" + f.write_text("original", encoding="utf-8") + registry.record(OWNER_ID, str(f), "original") + + f.write_text("modified", encoding="utf-8") + + content, error = tool._read_verified(str(f)) + + assert content == "" + assert error is not None + assert error.success is False + assert "Re-read and retry" in error.error + + +def test_read_verified_succeeds_if_content_matches(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: + f = tmp_path / "file.txt" + f.write_text("hello", encoding="utf-8") + registry.record(OWNER_ID, str(f), "hello") + + content, error = tool._read_verified(str(f)) + + assert error is None + assert content == "hello" + + +def test_read_verified_handles_oserror(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "ghost.txt") + registry.record(OWNER_ID, path, "anything") + + content, error = tool._read_verified(path) + + assert content == "" + assert error is not None + assert error.success is False + + +def test_read_verified_handles_unicode_decode_error(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: + f = tmp_path / "binary.bin" + f.write_bytes(b"\xff\xfe invalid utf-8") + registry.record(OWNER_ID, str(f), "anything") + + content, error = tool._read_verified(str(f)) + + assert content == "" + assert error is not None + assert error.success is False + + +def test_read_verified_is_isolated_between_agents(registry: FileRegistry, tmp_path) -> None: + """A file registered by agent A cannot be read-verified by agent B.""" + f = tmp_path / "file.txt" + f.write_text("hello", encoding="utf-8") + registry.record("agent-a", str(f), "hello") + + tool_b = DummyTool(registry, "agent-b") + content, error = tool_b._read_verified(str(f)) + + assert content == "" + assert error is not None + assert "Not authorized" in error.error + + +# --------------------------------------------------------------------------- +# _register +# --------------------------------------------------------------------------- + + +def test_register_registers_path(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + tool._register(path, "written") + + assert registry.is_known(OWNER_ID, path) is True + assert registry.verify(OWNER_ID, path, "written") is True + + +def test_register_updates_hash_after_register(tool: DummyTool, registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + tool._register(path, "old") + tool._register(path, "new") + + assert registry.verify(OWNER_ID, path, "new") is True + assert registry.verify(OWNER_ID, path, "old") is False + + +def test_register_scopes_to_the_tools_agent(registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + DummyTool(registry, "agent-a")._register(path, "x") + + assert registry.is_known("agent-a", path) is True + assert registry.is_known("agent-b", path) is False diff --git a/ddev/tests/ai/tools/fs/test_create_file.py b/ddev/tests/ai/tools/fs/test_create_file.py new file mode 100644 index 0000000000000..226a45a9d732d --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_create_file.py @@ -0,0 +1,86 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from unittest.mock import patch + +from ddev.ai.tools.fs.create_file import CreateFileTool +from ddev.ai.tools.fs.file_registry import FileRegistry + +from .conftest import OWNER_ID + + +def test_tool_name(registry: FileRegistry) -> None: + assert CreateFileTool(registry, OWNER_ID).name == "create_file" + + +async def test_create_file_success(create_tool: CreateFileTool, tmp_path) -> None: + f = tmp_path / "new.txt" + + result = await create_tool.run({"path": str(f), "content": "hello"}) + + assert result.success is True + assert f.read_text(encoding="utf-8") == "hello" + + +async def test_create_file_default_empty_content(create_tool: CreateFileTool, tmp_path) -> None: + f = tmp_path / "empty.txt" + + result = await create_tool.run({"path": str(f)}) + + assert result.success is True + assert f.read_text(encoding="utf-8") == "" + + +async def test_create_file_creates_missing_parent_dirs(create_tool: CreateFileTool, tmp_path) -> None: + f = tmp_path / "a" / "b" / "c" / "file.txt" + + result = await create_tool.run({"path": str(f), "content": "nested"}) + + assert result.success is True + assert f.exists() + assert f.read_text(encoding="utf-8") == "nested" + + +async def test_create_file_fails_if_file_already_exists( + create_tool: CreateFileTool, registry: FileRegistry, tmp_path +) -> None: + f = tmp_path / "existing.txt" + f.write_text("original", encoding="utf-8") + + result = await create_tool.run({"path": str(f), "content": "new"}) + + assert result.success is False + assert "File already exists" in result.error + assert f.read_text(encoding="utf-8") == "original" + assert not registry.is_known(OWNER_ID, str(f)) + + +async def test_create_tool_registers_in_registry(create_tool: CreateFileTool, registry: FileRegistry, tmp_path) -> None: + f = tmp_path / "file.txt" + await create_tool.run({"path": str(f), "content": "hi"}) + + assert registry.is_known(OWNER_ID, str(f)) is True + assert registry.verify(OWNER_ID, str(f), "hi") is True + + +async def test_create_file_oserror_on_mkdir(create_tool: CreateFileTool, registry: FileRegistry, tmp_path) -> None: + f = tmp_path / "a" / "b" / "new.txt" + + with patch("pathlib.Path.mkdir", side_effect=PermissionError("permission denied")): + result = await create_tool.run({"path": str(f), "content": "hi"}) + + assert result.success is False + assert result.error is not None + assert not f.exists() + assert not registry.is_known(OWNER_ID, str(f)) + + +async def test_create_file_oserror_on_write(create_tool: CreateFileTool, registry: FileRegistry, tmp_path) -> None: + f = tmp_path / "new.txt" + + with patch("builtins.open", side_effect=PermissionError("permission denied")): + result = await create_tool.run({"path": str(f), "content": "hi"}) + + assert result.success is False + assert result.error is not None + assert not registry.is_known(OWNER_ID, str(f)) diff --git a/ddev/tests/ai/tools/fs/test_edit_file.py b/ddev/tests/ai/tools/fs/test_edit_file.py new file mode 100644 index 0000000000000..078b46b8bb3f3 --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_edit_file.py @@ -0,0 +1,113 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from unittest.mock import patch + +import pytest + +from ddev.ai.tools.fs.create_file import CreateFileTool +from ddev.ai.tools.fs.edit_file import EditFileTool +from ddev.ai.tools.fs.file_registry import FileRegistry + +from .conftest import OWNER_ID + + +def test_tool_name(registry: FileRegistry) -> None: + assert EditFileTool(registry, OWNER_ID).name == "edit_file" + + +async def test_edit_file_replaces_string(edit_tool: EditFileTool, known_file) -> None: + result = await edit_tool.run({"path": str(known_file), "old_string": "line two", "new_string": "line TWO"}) + + assert result.success is True + content = known_file.read_text(encoding="utf-8") + assert "line TWO" in content + assert "line two" not in content + + +async def test_edit_file_deletes_line(edit_tool: EditFileTool, known_file) -> None: + result = await edit_tool.run({"path": str(known_file), "old_string": "line two\n", "new_string": ""}) + + assert result.success is True + assert "line two" not in known_file.read_text(encoding="utf-8") + + +async def test_edit_file_fails_for_unregistered_file(edit_tool: EditFileTool, tmp_path) -> None: + f = tmp_path / "unread.txt" + f.write_text("content", encoding="utf-8") + + result = await edit_tool.run({"path": str(f), "old_string": "content", "new_string": "new"}) + + assert result.success is False + assert "Not authorized" in result.error + + +@pytest.mark.parametrize("old_string", ["does not exist", ""]) +async def test_edit_file_fails_if_old_string_not_found_or_empty( + edit_tool: EditFileTool, known_file, old_string +) -> None: + result = await edit_tool.run({"path": str(known_file), "old_string": old_string, "new_string": "x"}) + + assert result.success is False + + +async def test_edit_file_fails_if_old_string_ambiguous( + edit_tool: EditFileTool, create_tool: CreateFileTool, tmp_path +) -> None: + f = tmp_path / "dup.txt" + await create_tool.run({"path": str(f), "content": "foo\nfoo\nfoo\n"}) + + result = await edit_tool.run({"path": str(f), "old_string": "foo", "new_string": "bar"}) + + assert result.success is False + assert "3" in result.error + assert result.hint is not None + + +async def test_edit_file_fails_if_file_changed_externally(edit_tool: EditFileTool, known_file) -> None: + known_file.write_text("externally modified\n", encoding="utf-8") + + result = await edit_tool.run({"path": str(known_file), "old_string": "line one", "new_string": "x"}) + + assert result.success is False + assert "Re-read and retry" in result.error + + +async def test_edit_file_updates_registry(edit_tool: EditFileTool, registry: FileRegistry, known_file) -> None: + await edit_tool.run({"path": str(known_file), "old_string": "line one", "new_string": "LINE ONE"}) + + new_content = known_file.read_text(encoding="utf-8") + assert registry.verify(OWNER_ID, str(known_file), new_content) is True + assert registry.verify(OWNER_ID, str(known_file), "line one\nline two\nline three\n") is False + + +@pytest.mark.parametrize( + "file_content,old_string,new_string,expected", + [ + ("line one\nline two\n", "line one\r\nline two", "replaced", "replaced\n"), # CRLF in old_string + ("line one\n", "line one", "A\r\nB", "A\nB\n"), # CRLF in new_string + ], +) +async def test_edit_file_normalizes_crlf( + edit_tool: EditFileTool, create_tool: CreateFileTool, tmp_path, file_content, old_string, new_string, expected +) -> None: + f = tmp_path / "file.txt" + await create_tool.run({"path": str(f), "content": file_content}) + + result = await edit_tool.run({"path": str(f), "old_string": old_string, "new_string": new_string}) + + assert result.success is True + assert f.read_text(encoding="utf-8") == expected + + +async def test_edit_file_oserror_on_write(edit_tool: EditFileTool, registry: FileRegistry, known_file) -> None: + original_content = known_file.read_text(encoding="utf-8") + + with patch("pathlib.Path.write_text", side_effect=PermissionError("permission denied")): + result = await edit_tool.run({"path": str(known_file), "old_string": "line one", "new_string": "x"}) + + assert result.success is False + assert result.error is not None + # File must be untouched and registry must still reflect the original content + assert known_file.read_text(encoding="utf-8") == original_content + assert registry.verify(OWNER_ID, str(known_file), original_content) is True diff --git a/ddev/tests/ai/tools/fs/test_file_access_policy.py b/ddev/tests/ai/tools/fs/test_file_access_policy.py new file mode 100644 index 0000000000000..01371aad4a307 --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_file_access_policy.py @@ -0,0 +1,263 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ddev.ai.tools.fs.file_access_policy import FileAccessError, FileAccessPolicy, canonicalize_path + +# --------------------------------------------------------------------------- +# canonicalize_path +# --------------------------------------------------------------------------- + + +def test_canonicalize_path_expands_tilde(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) # Windows uses USERPROFILE, not HOME + assert canonicalize_path("~/foo") == tmp_path / "foo" + + +def test_canonicalize_path_is_idempotent(tmp_path) -> None: + p = tmp_path / "sub" / "file.txt" + assert canonicalize_path(str(p)) == canonicalize_path(canonicalize_path(str(p))) + + +def test_canonicalize_path_accepts_path_object(tmp_path) -> None: + assert canonicalize_path(tmp_path / "x.txt") == tmp_path / "x.txt" + + +# --------------------------------------------------------------------------- +# assert_* return canonical path +# --------------------------------------------------------------------------- + + +def test_assert_readable_returns_canonical_path(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + returned = policy.assert_readable(str(tmp_path / "file.txt")) + assert returned == tmp_path / "file.txt" + + +def test_assert_writable_returns_canonical_path(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + returned = policy.assert_writable(str(tmp_path / "file.txt")) + assert returned == tmp_path / "file.txt" + + +def test_assert_readable_expands_tilde(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) # Windows uses USERPROFILE, not HOME + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + returned = policy.assert_readable("~/file.txt") + assert returned == tmp_path / "file.txt" + + +# --------------------------------------------------------------------------- +# write_root enforcement +# --------------------------------------------------------------------------- + + +def test_write_inside_root_allowed(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + policy.assert_writable(str(tmp_path / "sub" / "file.txt")) + + +def test_write_outside_root_denied(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + with pytest.raises(FileAccessError, match="outside write root"): + policy.assert_writable(str(tmp_path.parent / "outside.txt")) + + +def test_write_traversal_denied(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + with pytest.raises(FileAccessError, match="outside write root"): + policy.assert_writable(str(tmp_path / ".." / "escape.txt")) + + +def test_write_symlink_escaping_root_denied(tmp_path) -> None: + outside = tmp_path.parent / "outside_target" + outside.mkdir(exist_ok=True) + link = tmp_path / "link_to_outside" + link.symlink_to(outside) + + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + with pytest.raises(FileAccessError, match="outside write root"): + policy.assert_writable(str(link / "file.txt")) + + +# --------------------------------------------------------------------------- +# Inside write_root: deny patterns are bypassed for both reads and writes +# --------------------------------------------------------------------------- + + +def test_read_denied_basename_inside_write_root_is_allowed(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=(".env",)) + policy.assert_readable(str(tmp_path / ".env")) + + +def test_read_denied_path_pattern_inside_write_root_is_allowed(tmp_path) -> None: + secrets = tmp_path / "secrets" + secrets.mkdir() + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=(f"{secrets}/*",)) + policy.assert_readable(str(secrets / "key.txt")) + + +def test_write_denied_basename_inside_write_root_is_allowed(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=(".env",)) + policy.assert_writable(str(tmp_path / ".env")) + + +def test_write_denied_path_pattern_inside_write_root_is_allowed(tmp_path) -> None: + secrets = tmp_path / "secrets" + secrets.mkdir() + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=(f"{secrets}/*",)) + policy.assert_writable(str(secrets / "x.txt")) + + +# --------------------------------------------------------------------------- +# Outside write_root: deny patterns still apply to reads +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "filename", + [".env", ".env.local", ".envrc", ".netrc", "secret.pem", "private.key"], +) +def test_basename_pattern_denies_read_outside_write_root(tmp_path, filename) -> None: + write_root = tmp_path / "sandbox" + policy = FileAccessPolicy(write_root=write_root) # default patterns + with pytest.raises(FileAccessError, match="Read denied"): + policy.assert_readable(str(tmp_path / filename)) + + +@pytest.mark.parametrize("filename", ["app.py", "README.md", "config.yaml", "env.txt"]) +def test_basename_pattern_allows_unrelated_outside_write_root(tmp_path, filename) -> None: + write_root = tmp_path / "sandbox" + policy = FileAccessPolicy(write_root=write_root) + policy.assert_readable(str(tmp_path / filename)) + + +def test_custom_basename_pattern_denies_outside_write_root(tmp_path) -> None: + write_root = tmp_path / "sandbox" + policy = FileAccessPolicy(write_root=write_root, deny_patterns=("*.secret",)) + with pytest.raises(FileAccessError): + policy.assert_readable(str(tmp_path / "api.secret")) + policy.assert_readable(str(tmp_path / "api.public")) + + +# --------------------------------------------------------------------------- +# Path pattern semantics β€” match against full canonical path string +# --------------------------------------------------------------------------- + + +def test_path_pattern_denies_outside_write_root(tmp_path) -> None: + write_root = tmp_path / "sandbox" + denied = tmp_path / "secrets" + denied.mkdir() + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{denied}/*",)) + with pytest.raises(FileAccessError): + policy.assert_readable(str(denied / "x.txt")) + # fnmatch's '*' is greedy across '/', so subpaths are also denied + with pytest.raises(FileAccessError): + policy.assert_readable(str(denied / "sub" / "deep.txt")) + + +def test_path_pattern_allows_siblings_outside_write_root(tmp_path) -> None: + write_root = tmp_path / "sandbox" + denied = tmp_path / "secrets" + denied.mkdir() + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{denied}/*",)) + policy.assert_readable(str(tmp_path / "public.txt")) + + +def test_specific_path_pattern_denies_only_that_file(tmp_path) -> None: + write_root = tmp_path / "sandbox" + (tmp_path / "secrets").mkdir() + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{tmp_path}/secrets/credentials",)) + with pytest.raises(FileAccessError): + policy.assert_readable(str(tmp_path / "secrets" / "credentials")) + # same name elsewhere is fine + policy.assert_readable(str(tmp_path / "credentials")) + + +def test_path_pattern_with_glob_in_middle(tmp_path) -> None: + write_root = tmp_path / "sandbox" + base = tmp_path / "dir" + base.mkdir() + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{base}/*credentials*",)) + with pytest.raises(FileAccessError): + policy.assert_readable(str(base / "my_credentials_file")) + # '*' spans '/', so a deeper file with 'credentials' in the name is still denied + with pytest.raises(FileAccessError): + policy.assert_readable(str(base / "sub" / "credentials.txt")) + + +def test_path_pattern_resolves_symlinked_root(tmp_path) -> None: + """Pattern's static prefix is resolved at __init__ so symlinks can't bypass.""" + write_root = tmp_path / "sandbox" + real = tmp_path / "real_secrets" + real.mkdir() + (real / "key").write_text("x") + link = tmp_path / "link_secrets" + link.symlink_to(real) + + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{link}/*",)) + # accessing via the real path is denied + with pytest.raises(FileAccessError): + policy.assert_readable(str(real / "key")) + # accessing via the symlinked path is also denied (same resolved target) + with pytest.raises(FileAccessError): + policy.assert_readable(str(link / "key")) + + +def test_symlink_to_denied_target_is_blocked(tmp_path) -> None: + """A symlink in an allowed dir pointing into a denied tree is still denied.""" + write_root = tmp_path / "sandbox" + denied = tmp_path / "secrets" + denied.mkdir() + target = denied / "key" + target.write_text("x") + public = tmp_path / "innocent_link" + public.symlink_to(target) + + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{denied}/*",)) + with pytest.raises(FileAccessError): + policy.assert_readable(str(public)) + + +def test_traversal_does_not_bypass(tmp_path) -> None: + write_root = tmp_path / "sandbox" + denied = tmp_path / "secrets" + denied.mkdir() + (denied / "key").write_text("x") + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{denied}/*",)) + with pytest.raises(FileAccessError): + policy.assert_readable(str(tmp_path / "public" / ".." / "secrets" / "key")) + + +# --------------------------------------------------------------------------- +# Properties +# --------------------------------------------------------------------------- + + +def test_deny_patterns_property_preserves_input(tmp_path) -> None: + patterns = ("*.pem", "~/.ssh/*", ".env") + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=patterns) + assert policy.deny_patterns == patterns + + +def test_basename_patterns_filters_to_basename_only(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=("*.pem", "~/.ssh/*", ".env")) + assert set(policy.basename_patterns) == {"*.pem", ".env"} + + +# --------------------------------------------------------------------------- +# DEFAULT_DENY_PATTERNS β€” coverage for the rooted secret directories +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("root", ["~/.aws", "~/.kube", "~/.gnupg", "~/.docker", "~/.config/gcloud", "~/.ssh"]) +def test_read_denied_by_default_path_pattern(tmp_path, root) -> None: + write_root = tmp_path / "sandbox" + policy = FileAccessPolicy(write_root=write_root) + resolved_root = canonicalize_path(root) + with pytest.raises(FileAccessError, match="Read denied"): + policy.assert_readable(str(resolved_root / "config")) diff --git a/ddev/tests/ai/tools/fs/test_file_registry.py b/ddev/tests/ai/tools/fs/test_file_registry.py new file mode 100644 index 0000000000000..8be0779b5951b --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_file_registry.py @@ -0,0 +1,149 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry + +OWNER_A = "agent-a" +OWNER_B = "agent-b" + + +@pytest.fixture +def registry(tmp_path) -> FileRegistry: + return FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) + + +# --------------------------------------------------------------------------- +# is_known +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "record,expected", + [ + (False, False), + (True, True), + ], +) +def test_is_known(registry: FileRegistry, tmp_path, record, expected) -> None: + path = str(tmp_path / "file.txt") + if record: + registry.record(OWNER_A, path, "hello") + assert registry.is_known(OWNER_A, path) is expected + + +def test_is_known_different_path(registry: FileRegistry, tmp_path) -> None: + registry.record(OWNER_A, str(tmp_path / "other.txt"), "hello") + assert registry.is_known(OWNER_A, str(tmp_path / "file.txt")) is False + + +def test_is_known_is_scoped_to_owner(registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + registry.record(OWNER_A, path, "hello") + assert registry.is_known(OWNER_A, path) is True + assert registry.is_known(OWNER_B, path) is False + + +# --------------------------------------------------------------------------- +# verify +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "recorded_content,verify_content,expected", + [ + ("hello", "hello", True), + ("hello", "world", False), + (None, "any content", False), + ], +) +def test_verify(registry: FileRegistry, tmp_path, recorded_content, verify_content, expected) -> None: + path = str(tmp_path / "file.txt") + if recorded_content is not None: + registry.record(OWNER_A, path, recorded_content) + assert registry.verify(OWNER_A, path, verify_content) is expected + + +def test_verify_fails_for_different_agent(registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + registry.record(OWNER_A, path, "hello") + assert registry.verify(OWNER_B, path, "hello") is False + + +# --------------------------------------------------------------------------- +# record overwrites +# --------------------------------------------------------------------------- + + +def test_record_overwrites_previous_hash_within_agent(registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + registry.record(OWNER_A, path, "old") + registry.record(OWNER_A, path, "new") + + assert registry.verify(OWNER_A, path, "new") is True + assert registry.verify(OWNER_A, path, "old") is False + + +def test_record_does_not_cross_agents(registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + registry.record(OWNER_A, path, "from-a") + registry.record(OWNER_B, path, "from-b") + + assert registry.verify(OWNER_A, path, "from-a") is True + assert registry.verify(OWNER_A, path, "from-b") is False + assert registry.verify(OWNER_B, path, "from-b") is True + assert registry.verify(OWNER_B, path, "from-a") is False + + +# --------------------------------------------------------------------------- +# path normalization +# --------------------------------------------------------------------------- + + +def test_normalize_relative_and_absolute_are_same_key(registry: FileRegistry, tmp_path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + + abs_path = str(tmp_path / "file.txt") + rel_path = "file.txt" + + registry.record(OWNER_A, abs_path, "hello") + assert registry.is_known(OWNER_A, rel_path) is True + assert registry.verify(OWNER_A, rel_path, "hello") is True + + +def test_normalize_tilde_and_absolute_are_same_key(registry: FileRegistry, tmp_path, monkeypatch) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) # Windows uses USERPROFILE, not HOME + + tilde_path = "~/foo.txt" + abs_path = str(tmp_path / "foo.txt") + + registry.record(OWNER_A, tilde_path, "hello") + assert registry.is_known(OWNER_A, abs_path) is True + assert registry.is_known(OWNER_A, tilde_path) is True + assert registry.verify(OWNER_A, abs_path, "hello") is True + + +# --------------------------------------------------------------------------- +# lock_for β€” shared across agents so concurrent writes serialize on the path +# --------------------------------------------------------------------------- + + +def test_lock_for_same_path_returns_same_instance(registry: FileRegistry, tmp_path) -> None: + path = str(tmp_path / "file.txt") + assert registry.lock_for(path) is registry.lock_for(path) + + +def test_lock_for_different_paths_return_different_instances(registry: FileRegistry, tmp_path) -> None: + path_a = str(tmp_path / "a.txt") + path_b = str(tmp_path / "b.txt") + assert registry.lock_for(path_a) is not registry.lock_for(path_b) + + +def test_lock_for_normalizes_path(registry: FileRegistry, tmp_path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + abs_path = str(tmp_path / "file.txt") + rel_path = "file.txt" + assert registry.lock_for(abs_path) is registry.lock_for(rel_path) diff --git a/ddev/tests/ai/tools/fs/test_mkdir.py b/ddev/tests/ai/tools/fs/test_mkdir.py new file mode 100644 index 0000000000000..830a555ee207d --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_mkdir.py @@ -0,0 +1,47 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from unittest.mock import patch + +from ddev.ai.tools.fs.mkdir import MkdirTool + + +def test_tool_name(mkdir_tool: MkdirTool) -> None: + assert mkdir_tool.name == "mkdir" + + +async def test_mkdir_creates_directory(mkdir_tool: MkdirTool, tmp_path) -> None: + d = tmp_path / "new_dir" + + result = await mkdir_tool.run({"path": str(d)}) + + assert result.success is True + assert d.is_dir() + + +async def test_mkdir_creates_nested_directories(mkdir_tool: MkdirTool, tmp_path) -> None: + d = tmp_path / "a" / "b" / "c" + + result = await mkdir_tool.run({"path": str(d)}) + + assert result.success is True + assert d.is_dir() + + +async def test_mkdir_is_idempotent(mkdir_tool: MkdirTool, tmp_path) -> None: + d = tmp_path / "existing" + d.mkdir() + + result = await mkdir_tool.run({"path": str(d)}) + + assert result.success is True + + +async def test_mkdir_oserror_returns_failure(mkdir_tool: MkdirTool, tmp_path) -> None: + d = tmp_path / "denied" + + with patch("pathlib.Path.mkdir", side_effect=PermissionError("permission denied")): + result = await mkdir_tool.run({"path": str(d)}) + + assert result.success is False + assert result.error is not None diff --git a/ddev/tests/ai/tools/fs/test_policy_enforcement.py b/ddev/tests/ai/tools/fs/test_policy_enforcement.py new file mode 100644 index 0000000000000..57189debd05f9 --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_policy_enforcement.py @@ -0,0 +1,336 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +"""End-to-end policy enforcement: tools must respect the two-zone read/write model.""" + +from unittest.mock import AsyncMock, patch + +import pytest + +from ddev.ai.tools.fs.append_file import AppendFileTool +from ddev.ai.tools.fs.create_file import CreateFileTool +from ddev.ai.tools.fs.edit_file import EditFileTool +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.fs.mkdir import MkdirTool +from ddev.ai.tools.fs.read_file import ReadFileTool +from ddev.ai.tools.shell.grep import GrepTool + +OWNER_ID = "test-agent" + + +@pytest.fixture +def sandbox(tmp_path): + """Write root β€” a subdirectory of tmp_path so files at the tmp_path level are outside it.""" + s = tmp_path / "sandbox" + s.mkdir() + return s + + +@pytest.fixture +def sandboxed_registry(sandbox) -> FileRegistry: + return FileRegistry(policy=FileAccessPolicy(write_root=sandbox)) + + +# --------------------------------------------------------------------------- +# Write tools refuse paths outside write_root +# --------------------------------------------------------------------------- + + +async def test_create_file_refuses_outside_write_root(tmp_path, sandboxed_registry) -> None: + tool = CreateFileTool(sandboxed_registry, OWNER_ID) + outside = tmp_path / "outside.txt" + result = await tool.run({"path": str(outside), "content": "x"}) + assert result.success is False + assert "outside write root" in result.error + assert not outside.exists() + + +async def test_create_file_allows_inside_write_root(sandbox, sandboxed_registry) -> None: + tool = CreateFileTool(sandboxed_registry, OWNER_ID) + target = sandbox / "nested" / "file.txt" + result = await tool.run({"path": str(target), "content": "x"}) + assert result.success is True + assert target.read_text() == "x" + + +async def test_edit_file_refuses_outside_write_root(tmp_path, sandboxed_registry) -> None: + outside = tmp_path / "outside.txt" + outside.write_text("old") + sandboxed_registry.record(OWNER_ID, str(outside), "old") + + tool = EditFileTool(sandboxed_registry, OWNER_ID) + result = await tool.run({"path": str(outside), "old_string": "old", "new_string": "new"}) + assert result.success is False + assert "outside write root" in result.error + assert outside.read_text() == "old" + + +async def test_append_file_refuses_outside_write_root(tmp_path, sandboxed_registry) -> None: + outside = tmp_path / "outside.txt" + outside.write_text("hello") + sandboxed_registry.record(OWNER_ID, str(outside), "hello") + + tool = AppendFileTool(sandboxed_registry, OWNER_ID) + result = await tool.run({"path": str(outside), "content": " world"}) + assert result.success is False + assert "outside write root" in result.error + + +async def test_mkdir_refuses_outside_write_root(tmp_path, sandboxed_registry) -> None: + tool = MkdirTool(sandboxed_registry.policy) + outside = tmp_path / "outside_dir" + result = await tool.run({"path": str(outside)}) + assert result.success is False + assert "outside write root" in result.error + assert not outside.exists() + + +async def test_mkdir_allows_inside_write_root(sandbox, sandboxed_registry) -> None: + tool = MkdirTool(sandboxed_registry.policy) + target = sandbox / "a" / "b" / "c" + result = await tool.run({"path": str(target)}) + assert result.success is True + assert target.is_dir() + + +# --------------------------------------------------------------------------- +# Inside write_root: deny patterns are bypassed for reads and writes +# --------------------------------------------------------------------------- + + +async def test_write_denied_name_inside_write_root_is_allowed(sandbox, sandboxed_registry) -> None: + """Agents must be able to write .env and similar files inside their sandbox.""" + tool = CreateFileTool(sandboxed_registry, OWNER_ID) + target = sandbox / ".env" + result = await tool.run({"path": str(target), "content": "SECRET=1"}) + assert result.success is True + assert target.exists() + assert target.read_text() == "SECRET=1" + + +async def test_read_denied_name_inside_write_root_is_allowed(sandbox, sandboxed_registry) -> None: + """Agents must be able to read back files they created inside their sandbox.""" + target = sandbox / ".env" + target.write_text("SECRET=1") + sandboxed_registry.record(OWNER_ID, str(target), "SECRET=1") + + tool = ReadFileTool(sandboxed_registry, OWNER_ID) + result = await tool.run({"path": str(target)}) + assert result.success is True + + +# --------------------------------------------------------------------------- +# Outside write_root: read denylist still applies +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("filename", [".env", ".envrc", ".netrc", "api.pem", "private.key"]) +async def test_read_file_refuses_denied_names_outside_write_root(tmp_path, sandboxed_registry, filename) -> None: + # Files sit at tmp_path level, which is outside sandbox (the write_root). + target = tmp_path / filename + target.write_text("secret") + + tool = ReadFileTool(sandboxed_registry, OWNER_ID) + result = await tool.run({"path": str(target)}) + assert result.success is False + assert "Read denied" in result.error + + +async def test_read_file_allows_normal_files(tmp_path) -> None: + registry = FileRegistry(policy=FileAccessPolicy(write_root=tmp_path, deny_patterns=())) + target = tmp_path / "data.txt" + target.write_text("ok") + + tool = ReadFileTool(registry, OWNER_ID) + result = await tool.run({"path": str(target)}) + assert result.success is True + + +# --------------------------------------------------------------------------- +# Per-agent isolation: one agent's read does not authorize another agent's write +# --------------------------------------------------------------------------- + + +async def test_read_by_one_agent_does_not_authorize_another_to_edit(sandbox) -> None: + """Agent A reads a file; agent B tries to edit it without reading β€” must fail.""" + policy = FileAccessPolicy(write_root=sandbox, deny_patterns=()) + registry = FileRegistry(policy=policy) + target = sandbox / "shared.txt" + target.write_text("hello") + + reader_a = ReadFileTool(registry, "agent-a") + result = await reader_a.run({"path": str(target)}) + assert result.success is True + + editor_b = EditFileTool(registry, "agent-b") + result = await editor_b.run({"path": str(target), "old_string": "hello", "new_string": "world"}) + assert result.success is False + assert "Not authorized" in result.error + assert target.read_text() == "hello" + + +async def test_each_agent_can_edit_after_its_own_read(sandbox) -> None: + policy = FileAccessPolicy(write_root=sandbox, deny_patterns=()) + registry = FileRegistry(policy=policy) + target = sandbox / "shared.txt" + target.write_text("one") + + # Agent A reads, then edits β€” ok. + await ReadFileTool(registry, "agent-a").run({"path": str(target)}) + result = await EditFileTool(registry, "agent-a").run( + {"path": str(target), "old_string": "one", "new_string": "two"} + ) + assert result.success is True + + # Agent B must read first to refresh its own view, then may edit. + await ReadFileTool(registry, "agent-b").run({"path": str(target)}) + result = await EditFileTool(registry, "agent-b").run( + {"path": str(target), "old_string": "two", "new_string": "three"} + ) + assert result.success is True + assert target.read_text() == "three" + + +# --------------------------------------------------------------------------- +# GrepTool policy enforcement +# --------------------------------------------------------------------------- + + +async def test_grep_refuses_denied_root(tmp_path) -> None: + # write_root is a subdirectory; the search path at tmp_path level is outside it and denied. + write_root = tmp_path / "sandbox" + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{tmp_path}/*",)) + tool = GrepTool(policy) + with patch("ddev.ai.tools.shell.grep.run_command", new=AsyncMock()) as mock_run: + result = await tool.run({"pattern": "secret", "path": str(tmp_path / "foo")}) + assert result.success is False + assert "Read denied" in result.error + mock_run.assert_not_called() + + +async def test_grep_refuses_denied_name(tmp_path) -> None: + write_root = tmp_path / "sandbox" + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(".env",)) + tool = GrepTool(policy) + with patch("ddev.ai.tools.shell.grep.run_command", new=AsyncMock()) as mock_run: + result = await tool.run({"pattern": "SECRET", "path": str(tmp_path / ".env")}) + assert result.success is False + assert "Read denied" in result.error + mock_run.assert_not_called() + + +async def test_grep_allows_normal_path(tmp_path) -> None: + target = tmp_path / "data.txt" + target.write_text("hello world") + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + tool = GrepTool(policy) + result = await tool.run({"pattern": "hello", "path": str(target)}) + assert result.success is True + + +async def test_grep_non_recursive_returns_file_matches(tmp_path) -> None: + """Non-recursive grep on a single file returns actual matches (no post-filter applied).""" + target = tmp_path / "data.txt" + target.write_text("hello world\n") + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + tool = GrepTool(policy) + result = await tool.run({"pattern": "hello", "path": str(target), "recursive": False}) + assert result.success is True + assert "hello" in (result.data or "") + + +async def test_grep_inside_write_root_returns_denied_name_files(tmp_path) -> None: + """Recursive grep inside write_root returns .env and other denied-name files.""" + sandbox = tmp_path / "sandbox" + sandbox.mkdir() + (sandbox / ".env").write_text("SECRET=hello\n") + policy = FileAccessPolicy(write_root=sandbox, deny_patterns=(".env",)) + tool = GrepTool(policy) + result = await tool.run({"pattern": "hello", "path": str(sandbox), "recursive": True}) + assert result.success is True + assert ".env" in (result.data or "") + + +async def test_grep_post_filter_strips_denied_path_pattern_matches(tmp_path) -> None: + """Denied path-pattern files are stripped from grep output even when grep walks them.""" + write_root = tmp_path / "sandbox" + project = tmp_path / "project" + secrets = tmp_path / "secrets" + project.mkdir() + secrets.mkdir() + (project / "ok.txt").write_text("hello world\n") + (secrets / "leak.txt").write_text("hello world\n") + + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{secrets}/*",)) + tool = GrepTool(policy) + result = await tool.run({"pattern": "hello", "path": str(tmp_path), "recursive": True}) + assert result.success is True + assert "ok.txt" in result.data + assert "leak.txt: Read denied by policy" in result.data + + +async def test_grep_post_filter_strips_symlink_to_denied(tmp_path) -> None: + """A symlink in the search root resolving into a denied tree is filtered out.""" + write_root = tmp_path / "sandbox" + project = tmp_path / "project" + secrets = tmp_path / "secrets" + project.mkdir() + secrets.mkdir() + (secrets / "key.txt").write_text("hello world\n") + (project / "link.txt").symlink_to(secrets / "key.txt") + + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{secrets}/*",)) + tool = GrepTool(policy) + result = await tool.run({"pattern": "hello", "path": str(project), "recursive": True}) + assert result.success is True + # link.txt may appear as a denial notice but must not appear as a match line. + assert "link.txt" not in result.data + assert result.data + + +async def test_grep_excludes_basename_pattern_matches(tmp_path) -> None: + """Basename patterns ride on grep's --exclude flag; verify denied files are absent.""" + write_root = tmp_path / "sandbox" + project = tmp_path / "project" + project.mkdir() + (project / "config.py").write_text("token=abc\n") + (project / ".env").write_text("token=abc\n") + + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(".env",)) + tool = GrepTool(policy) + result = await tool.run({"pattern": "token", "path": str(project), "recursive": True}) + assert result.success is True + assert "config.py" in result.data + assert ".env" not in result.data + + +# --------------------------------------------------------------------------- +# Tilde-path canonicalization for write tools +# --------------------------------------------------------------------------- + + +async def test_create_file_with_tilde_path_writes_to_home_when_authorized(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) # Windows uses USERPROFILE, not HOME + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + registry = FileRegistry(policy=policy) + tool = CreateFileTool(registry, OWNER_ID) + + result = await tool.run({"path": "~/x.txt", "content": "hello"}) + + assert result.success is True + assert (tmp_path / "x.txt").read_text() == "hello" + + +async def test_create_file_with_tilde_path_refused_when_outside_write_root(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + policy = FileAccessPolicy(write_root=tmp_path / "sub", deny_patterns=()) + registry = FileRegistry(policy=policy) + tool = CreateFileTool(registry, OWNER_ID) + + result = await tool.run({"path": "~/x.txt", "content": "hello"}) + + assert result.success is False + assert "outside write root" in result.error + assert not (tmp_path / "x.txt").exists() diff --git a/ddev/tests/ai/tools/fs/test_read_file.py b/ddev/tests/ai/tools/fs/test_read_file.py new file mode 100644 index 0000000000000..2d37eba14c987 --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_read_file.py @@ -0,0 +1,106 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from unittest.mock import patch + +import pytest + +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.fs.read_file import ReadFileTool + +from .conftest import OWNER_ID + + +def test_tool_name(registry: FileRegistry) -> None: + assert ReadFileTool(registry, OWNER_ID).name == "read_file" + + +async def test_read_file_success(read_tool: ReadFileTool, tmp_path) -> None: + f = tmp_path / "config.txt" + f.write_text("hello\nworld\n", encoding="utf-8") + + result = await read_tool.run({"path": str(f)}) + + assert result.success is True + assert result.data == "0: hello\n1: world\n" + + +async def test_read_registers_unknown_file(read_tool: ReadFileTool, registry: FileRegistry, tmp_path) -> None: + f = tmp_path / "file.txt" + f.write_text("content", encoding="utf-8") + await read_tool.run({"path": str(f)}) + + assert registry.is_known(OWNER_ID, str(f)) is True + + +async def test_read_file_missing_file(read_tool: ReadFileTool, tmp_path) -> None: + result = await read_tool.run({"path": str(tmp_path / "ghost.txt")}) + + assert result.success is False + assert str(tmp_path / "ghost.txt") in result.error + + +async def test_read_file_permission_error(read_tool: ReadFileTool, tmp_path) -> None: + f = tmp_path / "secret.txt" + f.write_text("secret", encoding="utf-8") + + with patch("pathlib.Path.read_text", side_effect=PermissionError("permission denied")): + result = await read_tool.run({"path": str(f)}) + + assert result.success is False + assert result.error is not None + + +async def test_read_file_binary_file(read_tool: ReadFileTool, tmp_path) -> None: + f = tmp_path / "binary.bin" + f.write_bytes(b"\xff\xfe\x00binary") + + result = await read_tool.run({"path": str(f)}) + + assert result.success is False + assert result.error is not None + + +@pytest.mark.parametrize( + "offset, limit, expected", + [ + (1, None, "1: b\n2: c\n"), + (0, 2, "0: a\n1: b\n"), + (1, 2, "1: b\n2: c\n"), + (1, 1, "1: b\n"), + (2, 10, "2: c\n"), # limit exceeds remaining lines + (100, None, ""), # offset beyond EOF + ], +) +async def test_read_file_with_offset_and_limit(read_tool: ReadFileTool, tmp_path, offset, limit, expected) -> None: + f = tmp_path / "file.txt" + f.write_text("a\nb\nc\n", encoding="utf-8") + + result = await read_tool.run({"path": str(f), "offset": offset, "limit": limit}) + + assert result.success is True + assert result.data == expected + + +async def test_read_file_truncated(read_tool: ReadFileTool, tmp_path) -> None: + from ddev.ai.tools.core.truncation import MAX_CHARS + + f = tmp_path / "large.txt" + f.write_text("x" * (MAX_CHARS + 1000), encoding="utf-8") + + result = await read_tool.run({"path": str(f)}) + + assert result.success is True + assert result.truncated is True + assert result.total_size is not None + assert result.hint is not None + + +async def test_read_file_no_trailing_newline(read_tool: ReadFileTool, tmp_path) -> None: + f = tmp_path / "file.txt" + f.write_text("no newline at end", encoding="utf-8") + + result = await read_tool.run({"path": str(f)}) + + assert result.success is True + assert result.data == "0: no newline at end" diff --git a/ddev/tests/ai/tools/fs/test_workflow.py b/ddev/tests/ai/tools/fs/test_workflow.py new file mode 100644 index 0000000000000..aef70a306747d --- /dev/null +++ b/ddev/tests/ai/tools/fs/test_workflow.py @@ -0,0 +1,64 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from ddev.ai.tools.fs.append_file import AppendFileTool +from ddev.ai.tools.fs.create_file import CreateFileTool +from ddev.ai.tools.fs.edit_file import EditFileTool +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.fs.read_file import ReadFileTool + + +async def test_workflow_create_read_edit_append( + create_tool: CreateFileTool, + read_tool: ReadFileTool, + edit_tool: EditFileTool, + append_tool: AppendFileTool, + registry: FileRegistry, + tmp_path, +) -> None: + f = tmp_path / "workflow.txt" + + # Step 1: create + r = await create_tool.run({"path": str(f), "content": "version: 1\n"}) + assert r.success is True + + # Step 2: read (registers current content) + r = await read_tool.run({"path": str(f)}) + assert r.success is True + + # Step 3: edit + r = await edit_tool.run({"path": str(f), "old_string": "version: 1", "new_string": "version: 2"}) + assert r.success is True + assert "version: 2" in f.read_text(encoding="utf-8") + + # Step 4: append + r = await append_tool.run({"path": str(f), "content": "# updated\n"}) + assert r.success is True + assert f.read_text(encoding="utf-8").endswith("# updated\n") + + # Registry must reflect the final state for this agent + from .conftest import OWNER_ID + + assert registry.verify(OWNER_ID, str(f), f.read_text(encoding="utf-8")) is True + + +async def test_workflow_stale_file( + create_tool: CreateFileTool, + read_tool: ReadFileTool, + edit_tool: EditFileTool, + tmp_path, +) -> None: + f = tmp_path / "shared.txt" + await create_tool.run({"path": str(f), "content": "original\n"}) + f.write_text("updated externally\n", encoding="utf-8") + + result = await edit_tool.run({"path": str(f), "old_string": "original", "new_string": "my edit"}) + assert result.success is False + assert "Re-read and retry" in result.error + + await read_tool.run({"path": str(f)}) + + result = await edit_tool.run({"path": str(f), "old_string": "updated externally", "new_string": "final"}) + assert result.success is True + assert f.read_text(encoding="utf-8") == "final\n" diff --git a/ddev/tests/ai/tools/http/__init__.py b/ddev/tests/ai/tools/http/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/tools/http/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/tools/http/test_http_get.py b/ddev/tests/ai/tools/http/test_http_get.py new file mode 100644 index 0000000000000..2cb871bdfd62a --- /dev/null +++ b/ddev/tests/ai/tools/http/test_http_get.py @@ -0,0 +1,129 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +from ddev.ai.tools.http.http_get import HttpGetTool + +# --------------------------------------------------------------------------- +# Fixtures / helpers +# --------------------------------------------------------------------------- + + +@pytest.fixture +def http_tool() -> HttpGetTool: + return HttpGetTool() + + +def fake_response(status_code: int, text: str = "") -> MagicMock: + """Fake a HTTP response.""" + resp = MagicMock() + resp.status_code = status_code + resp.text = text + resp.is_success = 200 <= status_code < 300 + return resp + + +def patch_httpx(response=None, *, side_effect=None): + """Patch httpx.AsyncClient so tests never hit the network.""" + mock_get = AsyncMock(return_value=response, side_effect=side_effect) + mock_client = AsyncMock() + mock_client.__aenter__.return_value.get = mock_get + return patch("ddev.ai.tools.http.http_get.httpx.AsyncClient", return_value=mock_client) + + +# --------------------------------------------------------------------------- +# Metadata +# --------------------------------------------------------------------------- + + +def test_tool_meta(http_tool: HttpGetTool) -> None: + assert http_tool.name == "http_get" + + +# --------------------------------------------------------------------------- +# URL validation +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("url", ["ftp://example.com", "example.com", "", "//example.com"]) +async def test_invalid_url(http_tool: HttpGetTool, url: str) -> None: + result = await http_tool.run({"url": url}) + + assert result.success is False + assert "http" in result.error and "https" in result.error + + +# --------------------------------------------------------------------------- +# HTTP responses +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "status_code,body", + [ + (200, "# HELP requests_total counter\nrequests_total 42"), + (201, "created"), + (204, ""), + ], +) +async def test_request_success(http_tool: HttpGetTool, status_code: int, body: str) -> None: + with patch_httpx(fake_response(status_code, body)): + result = await http_tool.run({"url": "http://localhost:9090/metrics"}) + + assert result.success is True + assert f"Status: {status_code}" in result.data + assert body in result.data + + +@pytest.mark.parametrize("status_code", [400, 404, 500, 503]) +async def test_request_non_success_status(http_tool: HttpGetTool, status_code: int) -> None: + with patch_httpx(fake_response(status_code, "error body")): + result = await http_tool.run({"url": "http://localhost:9090/metrics"}) + + assert result.success is True + assert f"Status: {status_code}" in result.data + + +# --------------------------------------------------------------------------- +# Network errors +# --------------------------------------------------------------------------- + + +async def test_request_timeout(http_tool: HttpGetTool) -> None: + with patch_httpx(side_effect=httpx.TimeoutException("timed out")): + result = await http_tool.run({"url": "http://localhost:9090/metrics", "timeout": 1.0}) + + assert result.success is False + assert "timed out after 1.0s" in result.error + + +async def test_request_error(http_tool: HttpGetTool) -> None: + with patch_httpx(side_effect=httpx.RequestError("connection refused")): + result = await http_tool.run({"url": "http://localhost:9090/metrics"}) + + assert result.success is False + assert "Request failed" in result.error + + +# --------------------------------------------------------------------------- +# Truncation +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("status_code", [200, 500]) +async def test_response_truncated(http_tool: HttpGetTool, status_code: int) -> None: + from ddev.ai.tools.core.truncation import MAX_CHARS + + large_body = "x" * (MAX_CHARS + 1000) + with patch_httpx(fake_response(status_code, large_body)): + result = await http_tool.run({"url": "http://localhost:9090/metrics"}) + + assert result.success is True + assert result.truncated is True + assert result.total_size is not None + assert result.hint is not None + assert f"Status: {status_code}" in result.data diff --git a/ddev/tests/ai/tools/shell/__init__.py b/ddev/tests/ai/tools/shell/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/tools/shell/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/tools/shell/ddev/__init__.py b/ddev/tests/ai/tools/shell/ddev/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/ddev/tests/ai/tools/shell/ddev/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/tests/ai/tools/shell/ddev/test_ddev_tools.py b/ddev/tests/ai/tools/shell/ddev/test_ddev_tools.py new file mode 100644 index 0000000000000..fa7f30378225e --- /dev/null +++ b/ddev/tests/ai/tools/shell/ddev/test_ddev_tools.py @@ -0,0 +1,191 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest +from pydantic import ValidationError + +from ddev.ai.tools.shell.ddev.create import CreateInput, DdevCreateTool +from ddev.ai.tools.shell.ddev.ddev_test import DdevTestInput, DdevTestTool +from ddev.ai.tools.shell.ddev.env_show import DdevEnvShowTool, EnvShowInput +from ddev.ai.tools.shell.ddev.env_start import DdevEnvStartTool, EnvStartInput +from ddev.ai.tools.shell.ddev.env_stop import DdevEnvStopTool, EnvStopInput +from ddev.ai.tools.shell.ddev.env_test import DdevEnvTestTool, EnvTestInput +from ddev.ai.tools.shell.ddev.release_changelog import DdevReleaseChangelogTool, ReleaseChangelogInput +from ddev.ai.tools.shell.ddev.validate import DdevValidateInput, DdevValidateTool + +# --- ddev create --- + + +def test_create_cmd_basic(): + tool = DdevCreateTool() + assert tool.cmd(CreateInput(integration="my_check", integration_type="check")) == [ + "ddev", + "--no-interactive", + "create", + "--type", + "check", + "--skip-manifest", + "My_check", + ] + + +@pytest.mark.parametrize( + "integration_type", ["check", "check_only", "event", "jmx", "logs", "metrics_crawler", "snmp_tile", "tile"] +) +def test_create_cmd_all_types(integration_type: str): + cmd = DdevCreateTool().cmd(CreateInput(integration="my_check", integration_type=integration_type)) + assert cmd[cmd.index("--type") + 1] == integration_type + + +def test_create_invalid_type_raises(): + with pytest.raises(ValidationError): + CreateInput(integration="my_check", integration_type="custom") + + +# --- ddev test --- + + +def test_ddev_test_cmd_no_flags(): + cmd = DdevTestTool().cmd(DdevTestInput(integration="mycheck")) + assert "--no-interactive" in cmd + assert "-s" not in cmd + assert "-fs" not in cmd + + +def test_ddev_test_cmd_lint_only(): + cmd = DdevTestTool().cmd(DdevTestInput(integration="mycheck", lint=True)) + assert "-s" in cmd + assert "-fs" not in cmd + + +def test_ddev_test_cmd_fmt_only(): + cmd = DdevTestTool().cmd(DdevTestInput(integration="mycheck", fmt=True)) + assert "-fs" in cmd + assert "-s" not in cmd + + +def test_ddev_test_cmd_fmt_and_lint(): + cmd = DdevTestTool().cmd(DdevTestInput(integration="mycheck", fmt=True, lint=True)) + assert "-fs" in cmd + assert "-s" in cmd + + +def test_ddev_test_cmd_integration_last(): + cmd = DdevTestTool().cmd(DdevTestInput(integration="mycheck", fmt=True, lint=True)) + assert cmd[-1] == "mycheck" + + +def test_ddev_test_cmd_pytest_args(): + cmd = DdevTestTool().cmd(DdevTestInput(integration="mycheck", pytest_args=["-k", "test_my_func", "-s"])) + separator_idx = cmd.index("--") + assert cmd[separator_idx + 1 :] == ["-k", "test_my_func", "-s"] + assert cmd[separator_idx - 1] == "mycheck" + + +def test_ddev_test_cmd_no_pytest_args_omits_separator(): + cmd = DdevTestTool().cmd(DdevTestInput(integration="mycheck")) + assert "--" not in cmd + + +# --- ddev env show --- + + +def test_env_show_cmd(): + assert DdevEnvShowTool().cmd(EnvShowInput(integration="mycheck")) == [ + "ddev", + "--no-interactive", + "env", + "show", + "mycheck", + ] + + +# --- ddev env start --- + + +@pytest.mark.parametrize( + "dev,expected", + [ + (False, ["ddev", "--no-interactive", "env", "start", "mycheck", "py3.11-1.23"]), + (True, ["ddev", "--no-interactive", "env", "start", "--dev", "mycheck", "py3.11-1.23"]), + ], +) +def test_env_start_cmd(dev, expected): + assert DdevEnvStartTool().cmd(EnvStartInput(integration="mycheck", environment="py3.11-1.23", dev=dev)) == expected + + +# --- ddev env test --- + + +@pytest.mark.parametrize( + "dev,expected", + [ + (False, ["ddev", "--no-interactive", "env", "test", "mycheck", "py3.11-1.23"]), + (True, ["ddev", "--no-interactive", "env", "test", "--dev", "mycheck", "py3.11-1.23"]), + ], +) +def test_env_test_cmd(dev, expected): + assert DdevEnvTestTool().cmd(EnvTestInput(integration="mycheck", environment="py3.11-1.23", dev=dev)) == expected + + +# --- ddev env stop --- + + +def test_env_stop_cmd(): + assert DdevEnvStopTool().cmd(EnvStopInput(integration="mycheck", environment="py3.11-1.23")) == [ + "ddev", + "--no-interactive", + "env", + "stop", + "mycheck", + "py3.11-1.23", + ] + + +# --- ddev release changelog --- + + +@pytest.mark.parametrize("change_type", ["fixed", "added", "changed"]) +def test_release_changelog_cmd_change_type(change_type: str): + cmd = DdevReleaseChangelogTool().cmd( + ReleaseChangelogInput(change_type=change_type, integration="mycheck", message="msg") + ) + assert cmd[4] == change_type + + +def test_release_changelog_cmd_message_placement(): + cmd = DdevReleaseChangelogTool().cmd( + ReleaseChangelogInput(change_type="fixed", integration="mycheck", message="Some message") + ) + assert cmd[-2] == "-m" + assert cmd[-1] == "Some message" + + +def test_release_changelog_invalid_change_type_raises(): + with pytest.raises(ValidationError): + ReleaseChangelogInput(change_type="patch", integration="mycheck", message="Some message") + + +# --- ddev validate --- + + +@pytest.mark.parametrize("subcommand", ["config", "models", "metadata", "all"]) +def test_validate_cmd_all_subcommands(subcommand: str): + cmd = DdevValidateTool().cmd(DdevValidateInput(subcommand=subcommand, integration="mycheck")) + assert cmd == ["ddev", "--no-interactive", "validate", subcommand, "mycheck"] + + +@pytest.mark.parametrize("subcommand", ["config", "models", "metadata"]) +def test_validate_cmd_sync_flag_per_subcommand(subcommand: str): + cmd = DdevValidateTool().cmd(DdevValidateInput(subcommand=subcommand, integration="mycheck", sync=True)) + assert cmd == ["ddev", "--no-interactive", "validate", subcommand, "--sync", "mycheck"] + + +def test_validate_cmd_all_uses_fix_flag(): + cmd = DdevValidateTool().cmd(DdevValidateInput(subcommand="all", integration="mycheck", sync=True)) + assert cmd == ["ddev", "--no-interactive", "validate", "all", "--fix", "mycheck"] + + +def test_validate_invalid_subcommand_raises(): + with pytest.raises(ValidationError): + DdevValidateInput(subcommand="lint", integration="mycheck") diff --git a/ddev/tests/ai/tools/shell/test_base.py b/ddev/tests/ai/tools/shell/test_base.py new file mode 100644 index 0000000000000..3568170b9092d --- /dev/null +++ b/ddev/tests/ai/tools/shell/test_base.py @@ -0,0 +1,209 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import asyncio +from typing import Annotated +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from pydantic import Field + +from ddev.ai.tools.core.base import BaseToolInput +from ddev.ai.tools.core.truncation import MAX_CHARS +from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.shell.base import CmdTool, run_command + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def make_proc(returncode: int = 0, stdout: bytes = b"", stderr: bytes = b"") -> MagicMock: + proc = MagicMock() + proc.returncode = returncode + proc.communicate = AsyncMock(return_value=(stdout, stderr)) + proc.kill = MagicMock() + return proc + + +def patch_proc(proc: MagicMock): + return patch("asyncio.create_subprocess_exec", new=AsyncMock(return_value=proc)) + + +async def _raise_timeout(coro, *args, **kwargs): + coro.close() + raise asyncio.TimeoutError() + + +# --------------------------------------------------------------------------- +# Minimal CmdTool subclass for testing +# --------------------------------------------------------------------------- + + +class GreetInput(BaseToolInput): + name: Annotated[str, Field(description="Name to greet")] + + +class GreetTool(CmdTool[GreetInput]): + """Greet someone.""" + + @property + def name(self) -> str: + return "greet" + + def cmd(self, tool_input: GreetInput) -> list[str]: + return ["echo", f"hello {tool_input.name}"] + + +class SlowGreetTool(GreetTool): + timeout = 60 + + +@pytest.fixture +def proc() -> MagicMock: + return make_proc(returncode=0, stdout=b"hello\n") + + +@pytest.fixture +def greet_tool() -> GreetTool: + return GreetTool() + + +@pytest.fixture +def slow_greet_tool() -> SlowGreetTool: + return SlowGreetTool() + + +# --------------------------------------------------------------------------- +# run_command β€” output and exit code handling +# --------------------------------------------------------------------------- + + +async def test_run_command_success(proc): + with patch_proc(proc): + result = await run_command(["echo", "hello"]) + assert result.success is True + assert result.data == "hello\n" + assert result.truncated is False + + +async def test_run_command_failure_combines_stdout_and_stderr(): + proc = make_proc(returncode=1, stdout=b"partial\n", stderr=b"error\n") + with patch_proc(proc): + result = await run_command(["cmd"]) + assert result.success is False + assert "partial" in result.data + assert "error" in result.data + + +async def test_run_command_failure_stderr_only_when_no_stdout(): + proc = make_proc(returncode=1, stdout=b"", stderr=b"fatal error\n") + with patch_proc(proc): + result = await run_command(["cmd"]) + assert result.success is False and result.data == "fatal error\n" + + +async def test_run_command_ignores_stderr_on_zero_exit(): + proc = make_proc(returncode=0, stdout=b"out\n", stderr=b"warning\n") + with patch_proc(proc): + result = await run_command(["cmd"]) + assert result.success is True + assert "warning" not in result.data + + +async def test_run_command_stderr_included_when_stdout_empty_on_success(): + proc = make_proc(returncode=0, stdout=b"", stderr=b"info: initialized\n") + with patch_proc(proc): + result = await run_command(["cmd"]) + assert result.success is True + assert result.data == "info: initialized\n" + + +@pytest.mark.parametrize( + "returncode,stdout,stderr", + [ + (0, b"", b""), + (0, b" \n ", b""), + (1, b"", b""), + ], +) +async def test_run_command_empty_output(returncode, stdout, stderr): + proc = make_proc(returncode=returncode, stdout=stdout, stderr=stderr) + with patch_proc(proc): + result = await run_command(["cmd"]) + assert result.data == "(no output)" + + +# --------------------------------------------------------------------------- +# run_command β€” exceptions +# --------------------------------------------------------------------------- + + +async def test_run_command_not_found(): + with patch("asyncio.create_subprocess_exec", side_effect=FileNotFoundError()): + result = await run_command(["nonexistent"]) + assert result.success is False + assert "Command not found" in result.error + assert "nonexistent" in result.error + + +async def test_run_command_timeout(): + proc = make_proc() + with patch_proc(proc): + with patch("asyncio.wait_for", new=_raise_timeout): + result = await run_command(["sleep", "100"], timeout=5) + assert result.success is False + assert "5s" in result.error + proc.kill.assert_called_once() + + +async def test_run_command_unexpected_exception(): + with patch("asyncio.create_subprocess_exec", side_effect=OSError("permission denied")): + result = await run_command(["cmd"]) + assert result.success is False + assert "OSError" in result.error + assert "permission denied" in result.error + + +# --------------------------------------------------------------------------- +# run_command β€” truncation +# --------------------------------------------------------------------------- + + +async def test_run_command_truncation(): + large = ("x" * 80 + "\n") * 700 + proc = make_proc(stdout=large.encode()) + with patch_proc(proc): + result = await run_command(["cmd"]) + assert result.truncated is True + assert result.total_size == len(large) + assert result.shown_size == len(result.data) + assert result.hint is not None + + +async def test_run_command_no_truncation_at_limit(): + proc = make_proc(stdout=("x" * MAX_CHARS).encode()) + with patch_proc(proc): + result = await run_command(["cmd"]) + assert result.truncated is False + assert result.total_size is None + assert result.hint is None + + +# --------------------------------------------------------------------------- +# CmdTool +# --------------------------------------------------------------------------- + + +def test_cmd_tool_timeouts(greet_tool: GreetTool, slow_greet_tool: SlowGreetTool): + assert GreetTool.timeout == 10 # default timeout + assert SlowGreetTool.timeout == 60 # custom timeout + + +async def test_cmd_tool_dispatches_with_correct_timeout(greet_tool: GreetTool, slow_greet_tool: SlowGreetTool): + for tool, expected_timeout in [(greet_tool, 10), (slow_greet_tool, 60)]: + with patch( + "ddev.ai.tools.shell.base.run_command", new=AsyncMock(return_value=ToolResult(success=True)) + ) as mock_run: + await tool.run({"name": "world"}) + mock_run.assert_called_once_with(["echo", "hello world"], timeout=expected_timeout) diff --git a/ddev/tests/ai/tools/shell/test_tools.py b/ddev/tests/ai/tools/shell/test_tools.py new file mode 100644 index 0000000000000..f58ab68fe1a81 --- /dev/null +++ b/ddev/tests/ai/tools/shell/test_tools.py @@ -0,0 +1,220 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from unittest.mock import AsyncMock, patch + +import pytest + +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.shell.grep import GrepInput, GrepTool +from ddev.ai.tools.shell.list_files import ListFilesInput, ListFilesTool + +# --------------------------------------------------------------------------- +# Tool metadata +# --------------------------------------------------------------------------- + + +def test_grep_tool_meta(tmp_path) -> None: + tool = GrepTool(FileAccessPolicy(write_root=tmp_path)) + assert tool.name == "grep" + assert GrepTool.timeout == 30 + + +def test_list_files_tool_meta() -> None: + tool = ListFilesTool() + assert tool.name == "list_files" + assert ListFilesTool.timeout == 30 + + +# --------------------------------------------------------------------------- +# GrepTool +# --------------------------------------------------------------------------- + + +@pytest.fixture +def grep_tool(tmp_path) -> GrepTool: + return GrepTool(FileAccessPolicy(write_root=tmp_path, deny_patterns=())) + + +def test_grep_cmd_full_command(grep_tool: GrepTool): + # deny_patterns=() so no --exclude= flags; paths outside write_root still produce no flags. + assert grep_tool.cmd(GrepInput(pattern="ERROR", path="/var/log", recursive=True)) == [ + "grep", + "-n", + "-E", + "--null", + "-I", + "--no-messages", + "-r", + "--", + "ERROR", + "/var/log", + ] + assert grep_tool.cmd(GrepInput(pattern="ERROR", path="/var/log", recursive=False)) == [ + "grep", + "-n", + "-E", + "--null", + "-I", + "--no-messages", + "--", + "ERROR", + "/var/log", + ] + + +def test_grep_cmd_pattern_and_path_placement(grep_tool: GrepTool): + # pattern is always second-to-last, path is always last + pattern = r"^\d+\.\d+\.\d+" + cmd = grep_tool.cmd(GrepInput(pattern=pattern, path="/my dir/sub dir")) + assert cmd[-2] == pattern + assert cmd[-1] == "/my dir/sub dir" + + +def test_grep_cmd_recursive_outside_write_root_adds_basename_excludes(tmp_path) -> None: + """--exclude= flags are added only for basename patterns when search is outside write_root.""" + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=(".env", "*.pem", f"{tmp_path}/secrets/*")) + tool = GrepTool(policy) + # /project is outside tmp_path (write_root) + cmd = tool.cmd(GrepInput(pattern="SECRET", path="/project", recursive=True)) + flags_before_sep = cmd[: cmd.index("--")] + assert "--exclude=.env" in flags_before_sep + assert "--exclude=*.pem" in flags_before_sep + # Path patterns must NOT become flags β€” they ride on the post-filter. + assert not any(f.startswith("--exclude-dir") for f in flags_before_sep) + assert not any("secrets" in f for f in flags_before_sep) + assert cmd[-2] == "SECRET" + assert cmd[-1] == "/project" + + +def test_grep_cmd_recursive_inside_write_root_no_excludes(tmp_path) -> None: + """No --exclude= flags when search path is inside write_root.""" + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=(".env", "*.pem")) + tool = GrepTool(policy) + # Search inside write_root β€” deny patterns are bypassed, so no excludes. + cmd = tool.cmd(GrepInput(pattern="SECRET", path=str(tmp_path / "project"), recursive=True)) + assert not any(arg.startswith("--exclude") for arg in cmd) + + +def test_grep_cmd_recursive_spanning_write_root_no_excludes(tmp_path) -> None: + """No --exclude= flags when write_root is inside the search path (mixed zone).""" + write_root = tmp_path / "sandbox" + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(".env", "*.pem")) + tool = GrepTool(policy) + # Search starts at tmp_path which is a parent of write_root β€” spanning case. + cmd = tool.cmd(GrepInput(pattern="SECRET", path=str(tmp_path), recursive=True)) + assert not any(arg.startswith("--exclude") for arg in cmd) + + +def test_grep_cmd_non_recursive_no_exclude_flags(tmp_path) -> None: + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=(".env", "*.pem", f"{tmp_path}/secrets/*")) + tool = GrepTool(policy) + cmd = tool.cmd(GrepInput(pattern="SECRET", path="/project/file.txt", recursive=False)) + assert not any(arg.startswith("--exclude") for arg in cmd) + + +# --------------------------------------------------------------------------- +# GrepTool post-filter β€” unit tests on the parsing/decision logic +# --------------------------------------------------------------------------- + + +def test_filter_stdout_keeps_allowed_lines(tmp_path) -> None: + f = tmp_path / "ok.txt" + f.write_text("x") + tool = GrepTool(FileAccessPolicy(write_root=tmp_path, deny_patterns=())) + raw = f"{f}\x0042:hello\n{f}\x0043:world\n" + out = tool._filter_stdout(raw) + assert out == f"{f}:42:hello\n{f}:43:world" + + +def test_filter_stdout_filters_denied_path_lines(tmp_path) -> None: + write_root = tmp_path / "sandbox" + secrets = tmp_path / "secrets" + secrets.mkdir() + leak = secrets / "leak.txt" + leak.write_text("x") + public = tmp_path / "ok.txt" + public.write_text("x") + + # write_root is a subdirectory; secrets/ and ok.txt are outside it, so deny patterns apply. + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{secrets}/*",)) + tool = GrepTool(policy) + raw = f"{leak}\x001:hit\n{public}\x002:hit\n" + out = tool._filter_stdout(raw) + assert f"{leak}: Read denied by policy" in out + assert f"{public}:2:hit" in out + + +def test_filter_stdout_drops_lines_without_nul(tmp_path) -> None: + """Defensive: stderr noise / malformed output is dropped, not passed through.""" + tool = GrepTool(FileAccessPolicy(write_root=tmp_path, deny_patterns=())) + assert tool._filter_stdout("grep: something: Permission denied\n") == "" + + +def test_filter_stdout_caches_per_filename(tmp_path, monkeypatch) -> None: + f = tmp_path / "ok.txt" + f.write_text("x") + policy = FileAccessPolicy(write_root=tmp_path, deny_patterns=()) + calls = {"n": 0} + real = policy.assert_readable + + def counting(p): + calls["n"] += 1 + return real(p) + + monkeypatch.setattr(policy, "assert_readable", counting) + tool = GrepTool(policy) + raw = "".join(f"{f}\x00{i}:line\n" for i in range(10)) + tool._filter_stdout(raw) + assert calls["n"] == 1 + + +def test_filter_stdout_resolves_symlink_to_denied(tmp_path) -> None: + write_root = tmp_path / "sandbox" + secrets = tmp_path / "secrets" + secrets.mkdir() + target = secrets / "real.txt" + target.write_text("x") + link = tmp_path / "link.txt" + link.symlink_to(target) + + # secrets/ is outside write_root, so its deny pattern applies. + policy = FileAccessPolicy(write_root=write_root, deny_patterns=(f"{secrets}/*",)) + tool = GrepTool(policy) + raw = f"{link}\x001:hit\n" + out = tool._filter_stdout(raw) + assert out == f"{link}: Read denied by policy" + + +async def test_grep_no_matches_returns_success(grep_tool: GrepTool): + from ddev.ai.tools.core.types import ToolResult + + no_match_result = ToolResult(success=False, data="(no output)", error=None) + with patch("ddev.ai.tools.shell.grep.run_command", new=AsyncMock(return_value=no_match_result)): + result = await grep_tool(GrepInput(pattern="nomatch", path="/tmp")) + assert result.success is True + assert result.data == "(no output)" + + +# --------------------------------------------------------------------------- +# ListFilesTool +# --------------------------------------------------------------------------- + + +@pytest.fixture +def list_files_tool() -> ListFilesTool: + return ListFilesTool() + + +def test_list_files_cmd_non_recursive(list_files_tool: ListFilesTool): + # non-recursive by default β€” maxdepth 1 present, mindepth before maxdepth + cmd_default = list_files_tool.cmd(ListFilesInput(path="/tmp")) + cmd_explicit = list_files_tool.cmd(ListFilesInput(path="/var", recursive=False)) + + assert cmd_default == ["find", "/tmp", "-mindepth", "1", "-maxdepth", "1"] + assert cmd_explicit == ["find", "/var", "-mindepth", "1", "-maxdepth", "1"] + + +def test_list_files_cmd_recursive(list_files_tool: ListFilesTool): + cmd = list_files_tool.cmd(ListFilesInput(path="/var", recursive=True)) + assert cmd == ["find", "/var", "-mindepth", "1"] diff --git a/ddev/tests/ai/tools/test_registry.py b/ddev/tests/ai/tools/test_registry.py new file mode 100644 index 0000000000000..a8e3f4f40ea3e --- /dev/null +++ b/ddev/tests/ai/tools/test_registry.py @@ -0,0 +1,224 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import pytest + +from ddev.ai.tools.core.types import ToolResult +from ddev.ai.tools.fs.file_access_policy import FileAccessPolicy +from ddev.ai.tools.fs.file_registry import FileRegistry +from ddev.ai.tools.registry import ToolRegistry + +# --------------------------------------------------------------------------- +# Fake tools β€” implement ToolProtocol without depending on BaseTool +# --------------------------------------------------------------------------- + + +class FakeTool: + """Minimal ToolProtocol implementation for registry tests.""" + + def __init__(self, name: str, result: ToolResult | None = None) -> None: + self._name = name + self._result = result or ToolResult(success=True, data=f"{name} ok") + self.last_raw: dict[str, object] | None = None + + @property + def name(self) -> str: + return self._name + + @property + def description(self) -> str: + return f"Fake tool {self._name}" + + @property + def definition(self) -> dict: + return {"name": self._name, "description": self.description, "input_schema": {}} + + async def run(self, raw: dict[str, object]) -> ToolResult: + self.last_raw = raw + return self._result + + +# --------------------------------------------------------------------------- +# ToolRegistry.__init__ +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "tools,expected_names", + [ + ([FakeTool("alpha")], {"alpha"}), + ([], set()), + ([FakeTool("a"), FakeTool("b"), FakeTool("c")], {"a", "b", "c"}), + ], +) +def test_registry_registers_tools(tools, expected_names): + registry = ToolRegistry(tools) + assert set(registry._tools.keys()) == expected_names + + +def test_duplicate_name_last_one_wins(): + # This design is intentional to allow for tool overrides. + first = FakeTool("dup") + second = FakeTool("dup") + registry = ToolRegistry([first, second]) + assert registry._tools["dup"] is second + + +# --------------------------------------------------------------------------- +# ToolRegistry.definitions +# --------------------------------------------------------------------------- + + +def test_empty_registry_returns_empty_list(): + assert ToolRegistry([]).definitions == [] + + +def test_tool_registry_definitions_returns_all_tool_definitions(): + registry = ToolRegistry([FakeTool("a"), FakeTool("b")]) + assert len(registry.definitions) == 2 + + +def test_definition_contains_tool_name(): + registry = ToolRegistry([FakeTool("mytool")]) + assert registry.definitions[0]["name"] == "mytool" + + +# --------------------------------------------------------------------------- +# ToolRegistry.run +# --------------------------------------------------------------------------- + + +async def test_run_dispatches_to_correct_tool(): + tool_a = FakeTool("a", ToolResult(success=True, data="from a")) + tool_b = FakeTool("b", ToolResult(success=True, data="from b")) + registry = ToolRegistry([tool_a, tool_b]) + + result = await registry.run("b", {}) + assert result.success is True + assert result.data == "from b" + + +async def test_passes_raw_dict_to_tool_unchanged(): + tool = FakeTool("t") + registry = ToolRegistry([tool]) + raw = {"key": "value", "num": 42} + + await registry.run("t", raw) + assert tool.last_raw == raw + + +async def test_returns_tool_result_on_tool_failure(): + registry = ToolRegistry([FakeTool("t", ToolResult(success=False, error="bad input"))]) + result = await registry.run("t", {}) + assert result.success is False + assert result.error == "bad input" + + +async def test_unknown_tool_returns_failure(): + registry = ToolRegistry([FakeTool("known_tool")]) + result = await registry.run("unknown_tool", {}) + assert result.success is False + assert "Unknown tool: 'unknown_tool'" in result.error + + +async def test_empty_registry_always_returns_unknown_error(): + registry = ToolRegistry([]) + result = await registry.run("anything", {}) + assert result.success is False + assert result.error is not None + + +# --------------------------------------------------------------------------- +# ToolRegistry.available_tool_names +# --------------------------------------------------------------------------- + + +def test_available_tool_names_returns_non_empty_list(): + names = ToolRegistry.available_tool_names() + assert isinstance(names, list) + assert len(names) > 0 + + +def test_available_tool_names_returns_fresh_copy(): + a = ToolRegistry.available_tool_names() + b = ToolRegistry.available_tool_names() + assert a == b + assert a is not b + + +# --------------------------------------------------------------------------- +# ToolRegistry.from_names +# --------------------------------------------------------------------------- + + +OWNER_ID = "test-agent" +TOOLS_WITHOUT_EXTRA_DEPS = [n for n in ToolRegistry.available_tool_names() if n != "spawn_subagent"] + + +def test_from_names_empty(tmp_path): + registry = ToolRegistry.from_names( + [], owner_id=OWNER_ID, file_registry=FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) + ) + assert registry.definitions == [] + + +def test_from_names_unknown_raises(tmp_path): + with pytest.raises(ValueError, match="Unknown tool name: 'teleport'"): + ToolRegistry.from_names( + ["teleport"], owner_id=OWNER_ID, file_registry=FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) + ) + + +@pytest.mark.parametrize("name", TOOLS_WITHOUT_EXTRA_DEPS) +def test_from_names_each_known_tool(name, tmp_path): + registry = ToolRegistry.from_names( + [name], owner_id=OWNER_ID, file_registry=FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) + ) + assert len(registry.definitions) == 1 + assert registry.definitions[0]["name"] == name + + +def test_from_names_all_at_once(tmp_path): + all_names = TOOLS_WITHOUT_EXTRA_DEPS + registry = ToolRegistry.from_names( + all_names, owner_id=OWNER_ID, file_registry=FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) + ) + built_names = {d["name"] for d in registry.definitions} + assert built_names == set(all_names) + + +def test_from_names_spawn_subagent_without_deps_raises(tmp_path): + with pytest.raises(ValueError, match="requires both 'subagent_builder' and 'log_dir'"): + ToolRegistry.from_names( + ["spawn_subagent"], + owner_id=OWNER_ID, + file_registry=FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)), + ) + + +def test_from_names_fs_tools_share_file_registry(tmp_path): + """All tools that use the file registry in the same ToolRegistry share a single instance.""" + all_names = TOOLS_WITHOUT_EXTRA_DEPS + registry = ToolRegistry.from_names( + all_names, owner_id=OWNER_ID, file_registry=FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) + ) + fs_tools = [t for t in registry._tools.values() if hasattr(t, "_registry")] + if len(fs_tools) < 2: + pytest.skip("Need at least 2 fs tools to test shared registry") + registries = [t._registry for t in fs_tools] + assert all(r is registries[0] for r in registries) + + +def test_from_names_reuses_supplied_file_registry(tmp_path): + """Multiple ToolRegistries can share one FileRegistry; tools carry their own owner_id.""" + shared = FileRegistry(policy=FileAccessPolicy(write_root=tmp_path)) + reg_a = ToolRegistry.from_names(["read_file", "create_file"], owner_id="a", file_registry=shared) + reg_b = ToolRegistry.from_names(["read_file", "create_file"], owner_id="b", file_registry=shared) + + for tool in reg_a._tools.values(): + assert tool._registry is shared + assert tool._owner_id == "a" + for tool in reg_b._tools.values(): + assert tool._registry is shared + assert tool._owner_id == "b" diff --git a/ddev/tests/cli/dep/conftest.py b/ddev/tests/cli/dep/conftest.py new file mode 100644 index 0000000000000..8758a8cbc804c --- /dev/null +++ b/ddev/tests/cli/dep/conftest.py @@ -0,0 +1,32 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +import logging +from collections.abc import Generator +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from ddev.config.file import ConfigFileWithOverrides + + +@pytest.fixture(autouse=True) +def configure_github_credentials(config_file: ConfigFileWithOverrides) -> None: + """Provide github credentials so commands that touch app.github do not abort.""" + config_file.model.github = {'user': 'test-user', 'token': 'test-token'} + config_file.save() + + +@pytest.fixture +def httpx_at_debug() -> Generator[logging.Logger, None, None]: + """Force the httpx logger to DEBUG and restore its previous level on teardown.""" + logger = logging.getLogger('httpx') + previous_level = logger.level + logger.setLevel(logging.DEBUG) + try: + yield logger + finally: + logger.setLevel(previous_level) diff --git a/ddev/tests/cli/dep/test_promote.py b/ddev/tests/cli/dep/test_promote.py new file mode 100644 index 0000000000000..2d55db0660c63 --- /dev/null +++ b/ddev/tests/cli/dep/test_promote.py @@ -0,0 +1,79 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import logging + +import pytest + +RUN_DETAILS = { + 'workflow_run_id': 999, + 'run_url': 'https://api.github.com/repos/DataDog/integrations-core/actions/runs/999', + 'html_url': 'https://github.com/DataDog/integrations-core/actions/runs/999', +} + + +def test_promote_dispatches_workflow_and_prints_run_url(ddev, mocker): + mocker.patch('ddev.utils.github.GitHubManager.get_pr_head', return_value=('deadbeef', 'feature-branch')) + dispatch = mocker.patch('ddev.utils.github.GitHubManager.dispatch_workflow', return_value=RUN_DETAILS) + + result = ddev('dep', 'promote', 'https://github.com/DataDog/integrations-core/pull/12345') + + assert result.exit_code == 0, result.output + dispatch.assert_called_once_with( + workflow_id='dependency-wheel-promotion.yaml', + ref='master', + inputs={'pr_number': '12345', 'head_sha': 'deadbeef'}, + return_run_details=True, + ) + assert 'PR #12345' in result.output + assert 'feature-branch' in result.output + assert 'deadbeef' in result.output + assert RUN_DETAILS['html_url'] in result.output + assert 'Recent runs' not in result.output + assert 'query=event%3Aworkflow_dispatch' not in result.output + + +def test_promote_invalid_pr_url_aborts(ddev): + result = ddev('dep', 'promote', 'https://example.invalid/not-a-pr') + + assert result.exit_code != 0 + assert 'Could not extract a PR number' in result.output + + +def test_promote_aborts_when_no_run_details_returned(ddev, mocker): + mocker.patch('ddev.utils.github.GitHubManager.get_pr_head', return_value=('deadbeef', 'feature-branch')) + mocker.patch('ddev.utils.github.GitHubManager.dispatch_workflow', return_value=None) + + result = ddev('dep', 'promote', 'https://github.com/DataDog/integrations-core/pull/12345') + + assert result.exit_code != 0 + assert 'no run details were returned' in result.output + assert 'Promote workflow dispatched' not in result.output + + +def test_promote_suppresses_httpx_logs_and_restores_level(ddev, mocker, httpx_at_debug): + captured_levels = [] + + def capture_level(*_args, **_kwargs): + captured_levels.append(httpx_at_debug.level) + return ('deadbeef', 'feature-branch') + + mocker.patch('ddev.utils.github.GitHubManager.get_pr_head', side_effect=capture_level) + mocker.patch('ddev.utils.github.GitHubManager.dispatch_workflow', return_value=RUN_DETAILS) + + result = ddev('dep', 'promote', 'https://github.com/DataDog/integrations-core/pull/12345') + + assert result.exit_code == 0, result.output + assert captured_levels == [logging.WARNING] + assert httpx_at_debug.level == logging.DEBUG + + +def test_promote_restores_httpx_log_level_on_failure(ddev, mocker, httpx_at_debug): + """Ensure the finally branch restores the previous httpx logger level even when an API call raises.""" + mocker.patch('ddev.utils.github.GitHubManager.get_pr_head', side_effect=RuntimeError('boom')) + mocker.patch('ddev.utils.github.GitHubManager.dispatch_workflow') + + with pytest.raises(RuntimeError, match='boom'): + ddev('dep', 'promote', 'https://github.com/DataDog/integrations-core/pull/12345') + + assert httpx_at_debug.level == logging.DEBUG diff --git a/ddev/tests/cli/release/agent/conftest.py b/ddev/tests/cli/release/agent/conftest.py index 35172b2b57118..d0a72f41395ce 100644 --- a/ddev/tests/cli/release/agent/conftest.py +++ b/ddev/tests/cli/release/agent/conftest.py @@ -1,9 +1,12 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +from collections.abc import Callable + import pytest from ddev.repo.core import Repository +from ddev.utils.fs import Path @pytest.fixture @@ -55,6 +58,16 @@ def commit(msg): yield repo +@pytest.fixture +def write_repo_config() -> Callable[[Path, str], None]: + def write_config(repo_path: Path, contents: str) -> None: + config_dir = repo_path / '.ddev' + config_dir.mkdir(exist_ok=True) + (config_dir / 'config.toml').write_text(contents) + + return write_config + + def write_agent_requirements(repo_path, requirements): with open(repo_path / 'requirements-agent-release.txt', 'w') as req_file: req_file.write('\n'.join(requirements)) diff --git a/ddev/tests/cli/release/agent/test_changelog.py b/ddev/tests/cli/release/agent/test_changelog.py index d4b1396895e3f..3ddff8fdc7fd9 100644 --- a/ddev/tests/cli/release/agent/test_changelog.py +++ b/ddev/tests/cli/release/agent/test_changelog.py @@ -90,6 +90,32 @@ def test_new_integration_with_non_initial_version(repo_with_new_integration_patc assert mock_fetch_tags.call_count == 1 +def test_changelog_skips_unreleased_integrations(repo_with_history, config_file, ddev, mocker, write_repo_config): + config_file.model.repos['core'] = str(repo_with_history.path) + config_file.save() + write_repo_config( + repo_with_history.path, + """ +[overrides.release.agent.unreleased-integrations.by-integration] +bar = ["7.38.0"] +""", + ) + mock_fetch_tags = mocker.patch('ddev.utils.git.GitRepository.fetch_tags') + + result = ddev('release', 'agent', 'changelog', '--since', '7.37.0', '--to', '7.38.0') + assert result.exit_code == 0 + + expected_output = """## Datadog Agent version [7.38.0](https://github.com/DataDog/datadog-agent/blob/master/CHANGELOG.rst#7380) + +### New Integrations +* datadog_checks_base [2.1.3](https://github.com/DataDog/integrations-core/blob/master/datadog_checks_base/CHANGELOG.md) +### Integration Updates +* foo [1.5.0](https://github.com/DataDog/integrations-core/blob/master/foo/CHANGELOG.md) +""" + assert result.output.rstrip('\n') == expected_output.strip('\n') + assert mock_fetch_tags.call_count == 1 + + @pytest.fixture def repo_with_fake_changelog(repo_with_history, config_file): config_file.model.repos['core'] = str(repo_with_history.path) diff --git a/ddev/tests/cli/release/agent/test_integrations.py b/ddev/tests/cli/release/agent/test_integrations.py index 10589324d25d3..7ef443b2a5fc9 100644 --- a/ddev/tests/cli/release/agent/test_integrations.py +++ b/ddev/tests/cli/release/agent/test_integrations.py @@ -2,9 +2,16 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) import re +from types import SimpleNamespace import pytest +from ddev.cli.release.agent.common import ( + UNRELEASED_INTEGRATIONS_CONFIG, + agent_version_in_range, + get_unreleased_integrations, +) + def test_integrations_without_arguments(fake_integrations, ddev): result = ddev('release', 'agent', 'integrations') @@ -53,6 +60,60 @@ def test_integrations_since_to(fake_integrations, ddev): assert result.output.rstrip('\n') == expected_output.strip('\n') +@pytest.mark.parametrize( + ('version', 'expected'), + [ + pytest.param('7.74.0', True, id='lower-bound-inclusive'), + pytest.param('7.78.0', True, id='upper-bound-inclusive'), + pytest.param('7.77.0', True, id='middle'), + pytest.param('7.73.0', False, id='below-range'), + pytest.param('7.79.0', False, id='above-range'), + ], +) +def test_agent_version_in_range_is_inclusive(version, expected): + assert agent_version_in_range(version, '7.74.0..7.78.0') is expected + + +def test_agent_version_in_range_raises_on_malformed_range(): + with pytest.raises(ValueError, match="Invalid version range '7.74.0'"): + agent_version_in_range('7.74.0', '7.74.0') + + +def test_get_unreleased_integrations_combines_both_keys(): + config_data = { + 'by-integration': {'datadog-bar': ['7.78.0']}, + 'by-agent-version-range': {'7.74.0..7.78.0': ['datadog-foo']}, + } + repo = SimpleNamespace( + config=SimpleNamespace( + get=lambda key, default=None: config_data if key == UNRELEASED_INTEGRATIONS_CONFIG else default, + ) + ) + + assert get_unreleased_integrations(repo, '7.78.0') == {'bar', 'foo'} + + +def test_integrations_skips_unreleased_integrations(repo_with_history, config_file, ddev, write_repo_config): + config_file.model.repos['core'] = str(repo_with_history.path) + config_file.save() + write_repo_config( + repo_with_history.path, + """ +[overrides.release.agent.unreleased-integrations.by-agent-version-range] +"7.38.0..7.39.0" = ["datadog-bar"] +""", + ) + + result = ddev('release', 'agent', 'integrations', '--since', '7.38.0', '--to', '7.38.0') + assert result.exit_code == 0 + + expected_output = """## Datadog Agent version 7.38.0 + +* foo: 1.5.0 +* datadog_checks_base: 2.1.3""" + assert result.output.rstrip('\n') == expected_output.strip('\n') + + @pytest.fixture def repo_with_fake_integrations(repo_with_history, config_file): config_file.model.repos['core'] = str(repo_with_history.path) diff --git a/ddev/tests/cli/validate/all/test_github.py b/ddev/tests/cli/validate/all/test_github.py index a32d2f6915dde..b6a1d6df621d2 100644 --- a/ddev/tests/cli/validate/all/test_github.py +++ b/ddev/tests/cli/validate/all/test_github.py @@ -17,7 +17,7 @@ from ddev.cli.validate.all.orchestrator import ValidationConfig, ValidationResult CONFIGS = { - "ci": ValidationConfig(description="Validate CI configuration and Codecov settings", repo_wide=True), + "ci": ValidationConfig(description="Validate CI configuration and code coverage settings", repo_wide=True), "config": ValidationConfig(description="Validate default configuration files against spec.yaml"), "metadata": ValidationConfig(description="Validate metadata.csv metric definitions"), } @@ -135,7 +135,7 @@ def test_format_pr_comment_all_passed(helpers): | Validation | Description | Status | |---|---|---| - | `ci` | Validate CI configuration and Codecov settings | βœ… | + | `ci` | Validate CI configuration and code coverage settings | βœ… | | `config` | Validate default configuration files against spec.yaml | βœ… | """) @@ -161,7 +161,7 @@ def test_format_pr_comment_one_failure_with_target(helpers): | Validation | Description | Status | |---|---|---| - | `ci` | Validate CI configuration and Codecov settings | βœ… | + | `ci` | Validate CI configuration and code coverage settings | βœ… | """) assert format_pr_comment(results, CONFIGS, "changed", list(results)) == expected @@ -267,7 +267,7 @@ def test_format_step_summary_all_passed(helpers): | Validation | Description | Status | |---|---|---| - | `ci` | Validate CI configuration and Codecov settings | βœ… | + | `ci` | Validate CI configuration and code coverage settings | βœ… | | `config` | Validate default configuration files against spec.yaml | βœ… |""") assert format_step_summary(results, CONFIGS, "changed", list(results)) == expected @@ -282,7 +282,7 @@ def test_format_step_summary_with_failures(helpers): | Validation | Description | Status | |---|---|---| - | `ci` | Validate CI configuration and Codecov settings | βœ… | + | `ci` | Validate CI configuration and code coverage settings | βœ… | | `config` | Validate default configuration files against spec.yaml | ❌ | Run `ddev validate all changed --fix` to attempt to auto-fix supported validations.""") diff --git a/ddev/tests/cli/validate/test_ci.py b/ddev/tests/cli/validate/test_ci.py index 4b20c114d538f..c033eb81e1307 100644 --- a/ddev/tests/cli/validate/test_ci.py +++ b/ddev/tests/cli/validate/test_ci.py @@ -1,48 +1,12 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +from pathlib import Path + import pytest import yaml -def test_exactly_one_flag(ddev, repository, helpers): - codecov_yaml = repository.path / '.codecov.yml' - - with codecov_yaml.open(encoding='utf-8') as file: - codecov_yaml_info = yaml.safe_load(file) - - codecov_yaml_info['coverage']['status']['project']['ActiveMQ_XML']['flags'].append('test') - - output = yaml.safe_dump(codecov_yaml_info, default_flow_style=False, sort_keys=False) - with codecov_yaml.open(mode='w', encoding='utf-8') as file: - file.write(output) - - result = ddev("validate", "ci") - - assert result.exit_code == 1, result.output - error = "Project `ActiveMQ_XML` must have exactly one flag" - assert error in helpers.remove_trailing_spaces(result.output) - - -def test_carryforward_flag(ddev, repository, helpers): - codecov_yaml = repository.path / '.codecov.yml' - - with codecov_yaml.open(encoding='utf-8') as file: - temp = yaml.safe_load(file) - - temp['flags']['active_directory']['carryforward'] = False - - output = yaml.safe_dump(temp, default_flow_style=False, sort_keys=False) - with codecov_yaml.open(mode='w', encoding='utf-8') as file: - file.write(output) - - result = ddev("validate", "ci") - - assert result.exit_code == 1, result.output - error = "Flag `active_directory` must have carryforward set to true" - assert error in helpers.remove_trailing_spaces(result.output) - - def test_missing_hatch_toml(ddev, repository, helpers): import os @@ -56,126 +20,127 @@ def test_missing_hatch_toml(ddev, repository, helpers): assert error in helpers.remove_trailing_spaces(result.output) -def test_incorrect_project_name(ddev, repository, helpers): - codecov_yaml = repository.path / '.codecov.yml' - with codecov_yaml.open(encoding='utf-8') as file: - codecov_yaml_info = yaml.safe_load(file) +def test_validate_ci_success(ddev, helpers): + result = ddev('validate', 'ci') + assert result.exit_code == 0, result.output + assert helpers.remove_trailing_spaces(result.output) == helpers.dedent( + """ + CI configuration validation - temp = codecov_yaml_info['coverage']['status']['project']['Active_Directory'] - codecov_yaml_info['coverage']['status']['project']['active directory'] = temp - codecov_yaml_info['coverage']['status']['project'].pop('Active_Directory') + Passed: 1 + """ + ) - output = yaml.safe_dump(codecov_yaml_info, default_flow_style=False, sort_keys=False) - with codecov_yaml.open(mode='w', encoding='utf-8') as file: - file.write(output) - result = ddev("validate", "ci") - assert result.exit_code == 1, result.output - error = "Project `active directory` should be called `Active_Directory`" - assert error in helpers.remove_trailing_spaces(result.output) +def _remove_service(config_path): + with config_path.open(encoding='utf-8') as f: + config = yaml.safe_load(f) + config['services'] = [s for s in config.get('services', []) if s.get('id') != 'apache'] -def test_check_in_multiple_projects(ddev, repository, helpers): - codecov_yaml = repository.path / '.codecov.yml' - with codecov_yaml.open(encoding='utf-8') as file: - codecov_yaml_info = yaml.safe_load(file) + with config_path.open(mode='w', encoding='utf-8') as f: + yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False) - codecov_yaml_info['coverage']['status']['project']['Airflow']['flags'] = ['active_directory'] - output = yaml.safe_dump(codecov_yaml_info, default_flow_style=False, sort_keys=False) - with codecov_yaml.open(mode='w', encoding='utf-8') as file: - file.write(output) +def _set_wrong_paths(config_path): + with config_path.open(encoding='utf-8') as f: + config = yaml.safe_load(f) - result = ddev("validate", "ci") - assert result.exit_code == 1, result.output - error = "Check `active_directory` is defined as a flag in more than one project" - assert error in helpers.remove_trailing_spaces(result.output) + for service in config.get('services', []): + if service.get('id') == 'active_directory': + service['paths'] = ['wrong/path/'] + break + with config_path.open(mode='w', encoding='utf-8') as f: + yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False) -def test_codecov_missing_projects(ddev, repository, helpers): - codecov_yaml = repository.path / '.codecov.yml' - with codecov_yaml.open(encoding='utf-8') as file: - codecov_yaml_info = yaml.safe_load(file) - codecov_yaml_info['coverage']['status']['project'].pop('Apache') +def _add_stale_service(config_path): + with config_path.open(encoding='utf-8') as f: + config = yaml.safe_load(f) - output = yaml.safe_dump(codecov_yaml_info, default_flow_style=False, sort_keys=False) - with codecov_yaml.open(mode='w', encoding='utf-8') as file: - file.write(output) + config.setdefault('services', []).append({'id': 'stale_service', 'paths': ['stale_service/tests/']}) - result = ddev("validate", "ci") - assert result.exit_code == 1, result.output - error = "Codecov config has 1 missing project" - assert error in helpers.remove_trailing_spaces(result.output) + with config_path.open(mode='w', encoding='utf-8') as f: + yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False) -def test_incorrect_coverage_source_path(ddev, repository, helpers): - codecov_yaml = repository.path / '.codecov.yml' - with codecov_yaml.open(encoding='utf-8') as file: - codecov_yaml_info = yaml.safe_load(file) +def _add_duplicate_service(config_path: Path) -> None: + with config_path.open(encoding='utf-8') as f: + config = yaml.safe_load(f) - codecov_yaml_info['flags']['active_directory']['paths'] = [ - 'active_directory/datadog_checks/test', - 'active_directory/tests', - ] + duplicate_service = next(service for service in config['services'] if service.get('id') == 'active_directory') + config['services'].append({'id': duplicate_service['id'], 'paths': list(duplicate_service['paths'])}) - output = yaml.safe_dump(codecov_yaml_info, default_flow_style=False, sort_keys=False) - with codecov_yaml.open(mode='w', encoding='utf-8') as file: - file.write(output) + with config_path.open(mode='w', encoding='utf-8') as f: + yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False) - result = ddev("validate", "ci") - assert result.exit_code == 1, result.output - error = "Flag `active_directory` has incorrect coverage source paths" - assert error in helpers.remove_trailing_spaces(result.output) +def _remove_gates(config_path: Path) -> None: + with config_path.open(encoding='utf-8') as f: + config = yaml.safe_load(f) -def test_codecov_missing_flag(ddev, repository, helpers): - codecov_yaml = repository.path / '.codecov.yml' - with codecov_yaml.open(encoding='utf-8') as file: - codecov_yaml_info = yaml.safe_load(file) + config.pop('gates', None) - codecov_yaml_info['flags'].pop('active_directory') + with config_path.open(mode='w', encoding='utf-8') as f: + yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False) - output = yaml.safe_dump(codecov_yaml_info, default_flow_style=False, sort_keys=False) - with codecov_yaml.open(mode='w', encoding='utf-8') as file: - file.write(output) - result = ddev("validate", "ci") - assert result.exit_code == 1, result.output - error = "Codecov config has 1 missing flag" - assert error in helpers.remove_trailing_spaces(result.output) +def _clear_gates(config_path: Path) -> None: + with config_path.open(encoding='utf-8') as f: + config = yaml.safe_load(f) + + config['gates'] = [] + + with config_path.open(mode='w', encoding='utf-8') as f: + yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False) -# TODO We do not have an off the shelf fixture to generate a marketplace repository @pytest.mark.parametrize( - 'repository_name, repository_flag, expected_exit_code, expected_output', + 'corrupt_config, expected_error', [ - pytest.param('core', '-c', 1, 'Unable to find the Codecov config file', id='integrations-core'), + pytest.param(_remove_service, "Code coverage config has 1 missing service", id='missing_services'), + pytest.param( + _set_wrong_paths, + "Service `active_directory` has incorrect coverage source paths", + id='incorrect_paths', + ), + pytest.param( + _add_stale_service, "Code coverage config has 1 stale service: stale_service", id='stale_services' + ), + pytest.param( + _add_duplicate_service, + "Code coverage config has 1 duplicate service ID: active_directory", + id='duplicate_services', + ), + pytest.param(_remove_gates, "Code coverage config has no coverage gates", id='missing_gates'), + pytest.param(_clear_gates, "Code coverage config has no coverage gates", id='empty_gates'), ], ) -def test_codecov_file_missing( - ddev, repository, helpers, config_file, repository_name, repository_flag, expected_exit_code, expected_output -): - config_file.model.repos[repository_name] = str(repository.path) - config_file.save() +def test_code_coverage_config(ddev, repository, helpers, corrupt_config, expected_error): + result = ddev("validate", "ci", "--sync") + assert result.exit_code == 0, result.output - (repository.path / '.codecov.yml').unlink() + config_path = repository.path / 'code-coverage.datadog.yml' + corrupt_config(config_path) - result = ddev(repository_flag, "validate", "ci") - assert result.exit_code == expected_exit_code, result.output - assert expected_output in helpers.remove_trailing_spaces(result.output) + result = ddev("validate", "ci") + assert result.exit_code == 1, f"Expected validation to detect corrupted config: {result.output}" + assert expected_error in helpers.remove_trailing_spaces(result.output) + result = ddev("validate", "ci", "--sync") + assert result.exit_code == 0, f"Expected --sync to fix corrupted config: {result.output}" -def test_validate_ci_success(ddev, helpers): - result = ddev('validate', 'ci') - assert result.exit_code == 0, result.output - assert helpers.remove_trailing_spaces(result.output) == helpers.dedent( - """ - CI configuration validation + result = ddev("validate", "ci") + assert result.exit_code == 0, f"Expected validation to pass after sync: {result.output}" - Passed: 1 - """ - ) + +def test_code_coverage_file_missing(ddev, repository, helpers): + (repository.path / 'code-coverage.datadog.yml').unlink() + + result = ddev("-c", "validate", "ci") + assert result.exit_code == 1, result.output + assert "Unable to find the code coverage config file" in helpers.remove_trailing_spaces(result.output) @pytest.mark.parametrize( diff --git a/ddev/tests/utils/test_github.py b/ddev/tests/utils/test_github.py index 59b67d6a97fa5..e1f103dadcb08 100644 --- a/ddev/tests/utils/test_github.py +++ b/ddev/tests/utils/test_github.py @@ -1,6 +1,8 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import json + import pytest from ddev.utils.github import PullRequest @@ -83,3 +85,46 @@ def test_create_label(self, network_replay, github_manager): assert label.json()['name'] == 'my_custom_label' assert label.json()['color'] == 'ff0000' + + +def test_dispatch_workflow_default_returns_none(github_manager, mocker): + """Default dispatch_workflow keeps the prior fire-and-forget behavior.""" + response = mocker.MagicMock() + api_post = mocker.patch('ddev.utils.github.GitHubManager._GitHubManager__api_post', return_value=response) + + result = github_manager.dispatch_workflow( + workflow_id='example.yaml', + ref='master', + inputs={'pr_number': '123', 'head_sha': 'deadbeef'}, + ) + + assert result is None + api_post.assert_called_once() + payload = json.loads(api_post.call_args.kwargs['content']) + assert payload == {'ref': 'master', 'inputs': {'pr_number': '123', 'head_sha': 'deadbeef'}} + assert 'return_run_details' not in payload + + +def test_dispatch_workflow_return_run_details_sends_flag_and_returns_json(github_manager, mocker): + """When return_run_details is true, the payload includes the flag and the parsed JSON is returned.""" + run_details = { + 'workflow_run_id': 42, + 'run_url': 'https://api.github.com/repos/o/r/actions/runs/42', + 'html_url': 'https://github.com/o/r/actions/runs/42', + } + response = mocker.MagicMock() + response.json.return_value = run_details + api_post = mocker.patch('ddev.utils.github.GitHubManager._GitHubManager__api_post', return_value=response) + + result = github_manager.dispatch_workflow( + workflow_id='example.yaml', + ref='master', + inputs={'pr_number': '123', 'head_sha': 'deadbeef'}, + return_run_details=True, + ) + + assert result == run_details + payload = json.loads(api_post.call_args.kwargs['content']) + assert payload['return_run_details'] is True + assert payload['ref'] == 'master' + assert payload['inputs'] == {'pr_number': '123', 'head_sha': 'deadbeef'} diff --git a/delinea_privilege_manager/assets/logs/delinea-privilege-manager_tests.yaml b/delinea_privilege_manager/assets/logs/delinea-privilege-manager_tests.yaml index 4cceff815c45d..1f5df7aac64fe 100644 --- a/delinea_privilege_manager/assets/logs/delinea-privilege-manager_tests.yaml +++ b/delinea_privilege_manager/assets/logs/delinea-privilege-manager_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: delinea-privilege-manager tests: - diff --git a/docs/developer/.snippets/links.txt b/docs/developer/.snippets/links.txt index 59a86366e995c..48502a9038fb6 100644 --- a/docs/developer/.snippets/links.txt +++ b/docs/developer/.snippets/links.txt @@ -13,7 +13,6 @@ [azp-templates-windows]: https://github.com/DataDog/integrations-core/blob/master/.azure-pipelines/templates/test-single-windows.yml [black-github]: https://github.com/psf/black [click-github]: https://github.com/pallets/click -[codecov-home]: https://codecov.io [config-spec-example-consumer]: https://github.com/DataDog/integrations-core/blob/master/datadog_checks_dev/datadog_checks/dev/tooling/configuration/consumers/example.py [config-spec-model-consumer]: https://github.com/DataDog/integrations-core/blob/master/datadog_checks_dev/datadog_checks/dev/tooling/configuration/consumers/model.py [config-spec-producer]: https://github.com/DataDog/integrations-core/blob/master/datadog_checks_dev/datadog_checks/dev/tooling/configuration/core.py diff --git a/docs/developer/index.md b/docs/developer/index.md index 1461f7774875e..161ac6ca58803 100644 --- a/docs/developer/index.md +++ b/docs/developer/index.md @@ -1,7 +1,6 @@ # Agent Integrations [![CI - Docs](https://github.com/DataDog/integrations-core/workflows/docs/badge.svg)](https://github.com/DataDog/integrations-core/actions?workflow=docs) -[![Coverage status](https://codecov.io/github/DataDog/integrations-core/coverage.svg?branch=master)](https://codecov.io/github/DataDog/integrations-core?branch=master) [![GitHub contributors](https://img.shields.io/github/contributors/DataDog/integrations-core)](https://github.com/DataDog/integrations-core) [![Downloads](https://pepy.tech/badge/datadog-checks-dev)](https://pepy.tech/project/datadog-checks-dev) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/datadog-checks-dev)](https://pypi.org/project/datadog-checks-dev) diff --git a/docs/developer/meta/ci/labels.md b/docs/developer/meta/ci/labels.md index c146c54a66180..ec3c451260956 100644 --- a/docs/developer/meta/ci/labels.md +++ b/docs/developer/meta/ci/labels.md @@ -10,7 +10,7 @@ The labeler is [configured](https://github.com/DataDog/integrations-core/blob/ma | --- | --- | | integration/<NAME> | any directory at the root that actually contains an integration | | documentation | any Markdown, [config specs](../config-specs.md), `manifest.json`, or anything in `/docs/` | -| dev/testing | [GitHub Actions](https://github.com/DataDog/integrations-core/tree/master/.github/workflows) or [Codecov](https://github.com/DataDog/integrations-core/blob/master/.codecov.yml) config | +| dev/testing | [GitHub Actions](https://github.com/DataDog/integrations-core/tree/master/.github/workflows) or [code coverage](https://github.com/DataDog/integrations-core/blob/master/code-coverage.datadog.yml) config | | dev/tooling | [GitLab](https://github.com/DataDog/integrations-core/tree/master/.gitlab) or [GitHub Actions](https://github.com/DataDog/integrations-core/tree/master/.github/workflows) config, or [ddev](../../ddev/about.md#cli) | | dependencies | any change in shipped dependencies | | release | any [base package](../../base/about.md), [dev package](../../ddev/about.md), or integration release | diff --git a/docs/developer/meta/ci/validation.md b/docs/developer/meta/ci/validation.md index 99bc19f770449..cf8ff1b5f5ef8 100644 --- a/docs/developer/meta/ci/validation.md +++ b/docs/developer/meta/ci/validation.md @@ -18,7 +18,7 @@ This validates that each integration version is in sync with the [`requirements- ddev validate ci ``` -This validates that all CI entries for integrations are valid. This includes checking if the integration has the correct [Codecov config](https://github.com/DataDog/integrations-core/blob/master/.codecov.yml), and has a valid [CI entry](testing.md#target-enumeration) if it is testable. +This validates that all CI entries for integrations are valid. This includes checking if the integration has the correct [Datadog Code Coverage config](https://github.com/DataDog/integrations-core/blob/master/code-coverage.datadog.yml), and has a valid [CI entry](testing.md#target-enumeration) if it is testable. !!! tip Run `ddev validate ci --sync` to resolve most errors. diff --git a/druid/assets/logs/druid_tests.yaml b/druid/assets/logs/druid_tests.yaml index b37d6fab11911..501d496eef1f6 100644 --- a/druid/assets/logs/druid_tests.yaml +++ b/druid/assets/logs/druid_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "druid" tests: diff --git a/gitlab/assets/logs/gitlab_tests.yaml b/gitlab/assets/logs/gitlab_tests.yaml index 76f9dc6b5288b..2fc43f6b89e99 100644 --- a/gitlab/assets/logs/gitlab_tests.yaml +++ b/gitlab/assets/logs/gitlab_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "gitlab" tests: diff --git a/gitlab_runner/assets/logs/gitlab-runner_tests.yaml b/gitlab_runner/assets/logs/gitlab-runner_tests.yaml index e6f4a2c02689c..1bb0d2c32953f 100644 --- a/gitlab_runner/assets/logs/gitlab-runner_tests.yaml +++ b/gitlab_runner/assets/logs/gitlab-runner_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "gitlab-runner" tests: diff --git a/gpu/README.md b/gpu/README.md index ca83e18b5aaa8..31650d2207bcb 100644 --- a/gpu/README.md +++ b/gpu/README.md @@ -33,21 +33,32 @@ The check also uses eBPF probes to assign GPU usage and performance metrics to p #### Host -The agent needs to be configured to enable GPU-related features. Add the following parameters to the `/etc/datadog-agent/datadog.yaml` configuration file and then restart the Agent: +GPU monitoring requires configuration in both `/etc/datadog-agent/datadog.yaml` and `/etc/datadog-agent/system-probe.yaml`. Configuring only one of these files results in incomplete metrics collection. + +1. Add the following parameters to `/etc/datadog-agent/datadog.yaml`: ```yaml +gpu: + enabled: true collect_gpu_tags: true enable_nvml_detection: true ``` -Enabling the `gpu` integration requires `system-probe` to have the configuration option enabled for collecting per-process metrics. Inside the `/etc/datadog-agent/system-probe.yaml` configuration file, the following parameters must be set: +2. Add the following parameter to `/etc/datadog-agent/system-probe.yaml`. This flag loads the eBPF module responsible for per-process GPU metrics and is required even for non-containerized hosts: ```yaml gpu_monitoring: enabled: true ``` -The check in the Agent configuration file is enabled by default whenever NVIDIA GPUs and their drivers are detected in the system, as long as the `enable_nvml_detection` parameter is set to `true`. However, it can also be configured manually following these steps: +3. Restart both the Agent and system-probe: + +```shell +sudo systemctl restart datadog-agent +sudo systemctl restart datadog-agent-sysprobe +``` + +The check in the Agent configuration file is enabled by default whenever NVIDIA GPUs and their drivers are detected in the system, as long as the `enable_nvml_detection` parameter is set to `true`. The check can also be configured manually following these steps: 1. Edit the `gpu.d/conf.yaml` file, in the `conf.d/` folder at the root of your Agent's configuration directory, to start collecting your GPU performance data. diff --git a/haproxy/assets/logs/haproxy_tests.yaml b/haproxy/assets/logs/haproxy_tests.yaml index 3dd1fb2e9a623..12e5d67e5ed40 100644 --- a/haproxy/assets/logs/haproxy_tests.yaml +++ b/haproxy/assets/logs/haproxy_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: haproxy tests: - sample: 10.0.1.2:33317 [06/Feb/2009:12:14:14.655] http-in static/srv1 10/0/30/69/109 200 2750 - - ---- 1/1/1/1/0 0/0 {1wt.eu} {} "GET /index.html HTTP/1.1" diff --git a/harbor/assets/logs/harbor_tests.yaml b/harbor/assets/logs/harbor_tests.yaml index c979bb1c452c2..5eddcfc77a643 100644 --- a/harbor/assets/logs/harbor_tests.yaml +++ b/harbor/assets/logs/harbor_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "harbor" tests: diff --git a/kafka/assets/logs/kafka_tests.yaml b/kafka/assets/logs/kafka_tests.yaml index 5a36845ba0c9e..00343fbbc39be 100644 --- a/kafka/assets/logs/kafka_tests.yaml +++ b/kafka/assets/logs/kafka_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "kafka" tests: diff --git a/kafka_consumer/changelog.d/23842.removed b/kafka_consumer/changelog.d/23842.removed new file mode 100644 index 0000000000000..9969ddeefc6a1 --- /dev/null +++ b/kafka_consumer/changelog.d/23842.removed @@ -0,0 +1 @@ +Remove the Data Streams live messages reading feature, which has moved to the kafka_actions integration. diff --git a/kafka_consumer/datadog_checks/kafka_consumer/client.py b/kafka_consumer/datadog_checks/kafka_consumer/client.py index 2beb6d8ef2b47..d26c81beed385 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/client.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/client.py @@ -270,22 +270,6 @@ def list_consumer_group_offsets(self, groups): offsets.append((response_offset_info.group_id, tpo)) return offsets - def start_collecting_messages(self, start_offsets, consumer_group): - self.open_consumer(consumer_group) - self._consumer.assign(start_offsets) - - def get_next_message(self): - return self._consumer.poll(timeout=1) - - def delete_consumer_group(self, consumer_group): - """Delete a consumer group using the AdminClient.""" - try: - future = self.kafka_client.delete_consumer_groups([consumer_group]) - future[consumer_group].result(timeout=self.config._request_timeout) - self.log.debug("Successfully deleted consumer group: %s", consumer_group) - except Exception as e: - self.log.warning("Failed to delete consumer group %s: %s", consumer_group, e) - def describe_consumer_group(self, consumer_group): desc = self.kafka_client.describe_consumer_groups([consumer_group])[consumer_group].result() return desc.state.name diff --git a/kafka_consumer/datadog_checks/kafka_consumer/config.py b/kafka_consumer/datadog_checks/kafka_consumer/config.py index 041d8be414f59..d296d30405850 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/config.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/config.py @@ -84,9 +84,6 @@ def __init__(self, init_config, instance, log) -> None: self._sasl_oauth_token_provider.get("tls_ca_cert") if self._sasl_oauth_token_provider else None ) - # Data Streams live messages - self.live_messages_configs = instance.get('live_messages_configs', []) - self._kafka_cluster_id_override = instance.get('kafka_cluster_id_override') self._auto_detected_cluster_id = "" @@ -216,70 +213,6 @@ def validate_config(self): ) self._validate_consumer_groups() - self._validate_live_messages_configs() - - def _validate_live_messages_configs(self): - live_messages_configs = [] - for config in self.live_messages_configs: - if 'id' not in config: - self.log.debug('Data Streams live messages configuration has no ID') - continue - kafka = config.get('kafka', None) - if not kafka: - self.log.debug('Data Streams live messages configuration has no kafka configuration') - continue - if not ( - 'cluster' in kafka - and 'topic' in kafka - and 'partition' in kafka - and 'start_offset' in kafka - and 'n_messages' in kafka - ): - self.log.debug('Data Streams live messages configuration missing required kafka parameters.', kafka) - continue - - # Validate value format - if kafka.get('value_format', '') == '': - kafka['value_format'] = 'json' - value_format = kafka['value_format'] - if value_format not in ['json', 'avro', 'protobuf']: - self.log.debug( - 'Unsupported value format for Data Streams live messages, got %s. ' - 'Supported formats: json, avro, protobuf', - value_format, - ) - continue - - # Validate key format - if kafka.get('key_format', '') == '': - kafka['key_format'] = 'json' - key_format = kafka['key_format'] - if key_format not in ['json', 'avro', 'protobuf']: - self.log.debug( - 'Unsupported key format for Data Streams live messages, got %s. ' - 'Supported formats: json, avro, protobuf', - key_format, - ) - continue - - # Validate schemas for non-JSON formats - if value_format in ['avro', 'protobuf']: - if 'value_schema' not in kafka or not kafka['value_schema']: - self.log.debug( - 'Value schema is required for %s format in Data Streams live messages configuration', - value_format, - ) - continue - - if key_format in ['avro', 'protobuf']: - if 'key_schema' not in kafka or not kafka['key_schema']: - self.log.debug( - 'Key schema is required for %s format in Data Streams live messages configuration', key_format - ) - continue - - live_messages_configs.append(config) - self.live_messages_configs = live_messages_configs def _compile_regex(self, consumer_groups_regex, consumer_groups): # Turn the dict of regex dicts into a single string and compile diff --git a/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py b/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py index 8f0fe0effbd0a..3fe6a81bab830 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py @@ -1,18 +1,10 @@ # (C) Datadog, Inc. 2019-present # All rights reserved # Licensed under Simplified BSD License (see LICENSE) -import base64 import json from collections import defaultdict -from io import BytesIO from time import time -from confluent_kafka import TopicPartition -from fastavro import schemaless_reader -from google.protobuf import descriptor_pb2, descriptor_pool, message_factory -from google.protobuf.json_format import MessageToJson -from google.protobuf.message import DecodeError, EncodeError - from datadog_checks.base import AgentCheck from datadog_checks.kafka_consumer.client import KafkaClient from datadog_checks.kafka_consumer.cluster_metadata import ClusterMetadataCollector @@ -24,8 +16,6 @@ ) MAX_TIMESTAMPS = 1000 -SCHEMA_REGISTRY_MAGIC_BYTE = 0x00 -DATA_STREAMS_MESSAGES_CACHE_KEY = 'get_messages_cache' class KafkaCheck(AgentCheck): @@ -126,7 +116,6 @@ def check(self, _): broker_timestamps, cluster_id, ) - self.data_streams_live_message(highwater_offsets or {}, cluster_id) # Collect cluster metadata if enabled if self.config._cluster_monitoring_enabled: @@ -233,30 +222,6 @@ def _load_broker_timestamps(self, persistent_cache_key): self.log.warning('Could not read broker timestamps from cache: %s', str(e)) return broker_timestamps - def _messages_have_been_retrieved(self, config_id): - """Check if messages have been retrieved for the given config ID.""" - try: - content = self.read_persistent_cache(DATA_STREAMS_MESSAGES_CACHE_KEY) - if content: - config_ids = set(content.split(",")) - return config_id in config_ids - except Exception as e: - self.log.warning('Could not read persistent cache: %s', str(e)) - return False - - def _mark_messages_retrieved(self, config_id): - """Mark that messages have been retrieved for the given config ID.""" - try: - content = self.read_persistent_cache(DATA_STREAMS_MESSAGES_CACHE_KEY) - if content: - config_ids = set(content.split(",")) - else: - config_ids = set() - config_ids.add(config_id) - self.write_persistent_cache(DATA_STREAMS_MESSAGES_CACHE_KEY, ",".join(config_ids)) - except Exception as e: - self.log.warning('Could not write to persistent cache: %s', str(e)) - def _add_broker_timestamps(self, broker_timestamps, highwater_offsets): for (topic, partition), highwater_offset in highwater_offsets.items(): timestamps = broker_timestamps["{}_{}".format(topic, partition)] @@ -458,128 +423,6 @@ def send_event(self, title, text, tags, event_type, aggregation_key, severity='i } self.event(event_dict) - def data_streams_live_message(self, highwater_offsets, cluster_id): - monitored_topics = None - for cfg in self.config.live_messages_configs: - monitored_topics = monitored_topics or {topic.lower() for (topic, _) in highwater_offsets.keys()} - kafka = cfg['kafka'] - topic = kafka["topic"] - partition = kafka["partition"] - start_offset = kafka["start_offset"] - n_messages = kafka["n_messages"] - cluster = kafka["cluster"] - config_id = cfg["id"] - value_format = kafka["value_format"] - value_schema_str = kafka.get("value_schema", "") - value_uses_schema_registry = kafka.get("value_uses_schema_registry", False) - key_format = kafka["key_format"] - key_schema_str = kafka.get("key_schema", "") - key_uses_schema_registry = kafka.get("key_uses_schema_registry", False) - if self._messages_have_been_retrieved(config_id): - continue - if not cluster or not cluster_id or cluster.lower() != cluster_id.lower(): - continue - if topic.lower() not in monitored_topics: - self.log.debug('Skipping live messages for topic %s because it is not monitored by this check', topic) - continue - start_offsets = resolve_start_offsets(highwater_offsets, topic, partition, start_offset, n_messages) - - if not start_offsets: - self.log.warning('Unable to get a list of partitions to read from for live messages') - self.send_log( - { - 'timestamp': int(time()), - 'config_id': config_id, - 'technology': 'kafka', - 'cluster': str(cluster), - 'topic': str(topic), - 'live_messages_error': 'Unable to list partitions to read from', - 'message': "Unable to list partitions to read from", - 'feature': 'data_streams_messages', - } - ) - continue - - try: - value_schema, key_schema = ( - build_schema(value_format, value_schema_str), - build_schema(key_format, key_schema_str), - ) - except ( - ValueError, - json.JSONDecodeError, - base64.binascii.Error, - IndexError, - KeyError, - TypeError, - DecodeError, - EncodeError, - ) as e: - self.log.error( - "Failed to build schemas for config_id: %s, topic: %s, partition: %s. Error: %s", - config_id, - topic, - partition, - e, - ) - continue - - consumer_group = f"datadog_messages_{config_id}" - self.client.start_collecting_messages(start_offsets, consumer_group) - try: - for _ in range(n_messages): - message = self.client.get_next_message() - if message is None: - self.log.debug('Live messages: no message to retrieve') - self.send_log( - { - 'timestamp': int(time()), - 'config_id': config_id, - 'technology': 'kafka', - 'cluster': str(cluster), - 'topic': str(topic), - 'live_messages_error': 'No more messages to retrieve', - 'message': "No more messages to retrieve", - 'feature': 'data_streams_messages', - } - ) - break - data = { - 'timestamp': int(time()), - 'technology': 'kafka', - 'cluster': str(cluster), - 'config_id': config_id, - 'topic': str(topic), - 'partition': str(message.partition()), - 'offset': str(message.offset()), - 'feature': 'data_streams_messages', - } - decoded_value, value_schema_id, decoded_key, key_schema_id = deserialize_message( - message, - value_format, - value_schema, - value_uses_schema_registry, - key_format, - key_schema, - key_uses_schema_registry, - ) - if decoded_value: - data['message_value'] = decoded_value - else: - data['message'] = "Message format not supported" - data['live_messages_error'] = 'Message format not supported' - if value_schema_id: - data['value_schema_id'] = str(value_schema_id) - if decoded_key: - data['message_key'] = decoded_key - if key_schema_id: - data['key_schema_id'] = str(key_schema_id) - self.send_log(data) - finally: - self.client.close_consumer() - self.client.delete_consumer_group(consumer_group) - self._mark_messages_retrieved(config_id) - def _get_interpolated_timestamp(timestamps, offset): if offset in timestamps: @@ -608,279 +451,3 @@ def _get_interpolated_timestamp(timestamps, offset): slope = (timestamp_after - timestamp_before) / float(offset_after - offset_before) timestamp = slope * (offset - offset_after) + timestamp_after return timestamp - - -def resolve_start_offsets(highwater_offsets, target_topic, target_partition, start_offset, n_messages): - if int(target_partition) == -1: - # in this case, we get n_messages, starting at offset latest - n_messages on each partition. - # this doesn't match exactly to the latest messages, but if we don't do that, we could run into - # edge cases when some partitions don't get any traffic. - start_offsets = [] - for topic, partition in highwater_offsets: - if topic == target_topic and highwater_offsets[(topic, partition)] >= 0: - start_offsets.append( - TopicPartition(topic, partition, max(0, highwater_offsets[(topic, partition)] - n_messages + 1)) - ) - if len(start_offsets) >= n_messages: - break - return start_offsets - if int(start_offset) == -1: - end_offset = highwater_offsets.get((target_topic, target_partition), -1) - return ( - [] - if end_offset < 0 - else [TopicPartition(target_topic, target_partition, max(0, end_offset - n_messages + 1))] - ) - return [TopicPartition(target_topic, target_partition, start_offset)] - - -def deserialize_message( - message, - value_format, - value_schema, - value_uses_schema_registry, - key_format, - key_schema, - key_uses_schema_registry, -): - try: - decoded_value, value_schema_id = _deserialize_bytes_maybe_schema_registry( - message.value(), value_format, value_schema, value_uses_schema_registry - ) - except (UnicodeDecodeError, json.JSONDecodeError, ValueError): - return None, None, None, None - try: - decoded_key, key_schema_id = _deserialize_bytes_maybe_schema_registry( - message.key(), key_format, key_schema, key_uses_schema_registry - ) - return decoded_value, value_schema_id, decoded_key, key_schema_id - except (UnicodeDecodeError, json.JSONDecodeError, ValueError): - return decoded_value, value_schema_id, None, None - - -def _read_varint(data): - shift = 0 - result = 0 - bytes_read = 0 - - for byte in data: - bytes_read += 1 - result |= (byte & 0x7F) << shift - if (byte & 0x80) == 0: - return result, bytes_read - shift += 7 - - raise ValueError("Incomplete varint") - - -def _read_protobuf_message_indices(payload): - """ - Read the Confluent Protobuf message indices array. - - The Confluent Protobuf wire format includes message indices after the schema ID: - [message_indices_length:varint][message_indices:varint...] - - The indices indicate which message type to use from the .proto schema. - For example, [0] = first message, [1] = second message, [0, 0] = nested message. - - Args: - payload: bytes after the schema ID - - Returns: - tuple: (message_indices list, remaining payload bytes) - """ - array_len, bytes_read = _read_varint(payload) - payload = payload[bytes_read:] - - indices = [] - for _ in range(array_len): - index, bytes_read = _read_varint(payload) - indices.append(index) - payload = payload[bytes_read:] - - return indices, payload - - -def _deserialize_bytes_maybe_schema_registry(message, message_format, schema, uses_schema_registry): - if not message: - return "", None - if uses_schema_registry: - return _deserialize_bytes(message, message_format, schema, True) - else: - # Fallback behavior: try without schema registry format first, then with it - try: - return _deserialize_bytes(message, message_format, schema, False) - except (UnicodeDecodeError, json.JSONDecodeError, ValueError): - return _deserialize_bytes(message, message_format, schema, True) - - -def _deserialize_bytes(message, message_format, schema, uses_schema_registry): - """Deserialize a message from Kafka. - Args: - message: Raw message bytes from Kafka - message_format: Format of the message (protobuf, avro, json, etc.) - schema: Schema object (type depends on message_format) - uses_schema_registry: Whether message uses schema registry format - Returns: - Tuple of (decoded_message, schema_id) where schema_id is None if not using schema registry - """ - if not message: - return "", None - - schema_id = None - if uses_schema_registry: - if len(message) < 5 or message[0] != SCHEMA_REGISTRY_MAGIC_BYTE: - msg_hex = message[:5].hex() if len(message) >= 5 else message.hex() - raise ValueError( - f"Expected schema registry format (magic byte 0x00 + 4-byte schema ID), " - f"but message is too short or has wrong magic byte: {msg_hex}" - ) - schema_id = int.from_bytes(message[1:5], 'big') - message = message[5:] - - if message_format == 'protobuf': - return _deserialize_protobuf(message, schema, uses_schema_registry), schema_id - elif message_format == 'avro': - return _deserialize_avro(message, schema), schema_id - else: - return _deserialize_json(message), schema_id - - -def _deserialize_json(message): - decoded = message.decode('utf-8') - json.loads(decoded) - return decoded - - -def _get_protobuf_message_class(schema_info, message_indices): - """Get the protobuf message class based on schema info and message indices. - - Args: - schema_info: Tuple of (descriptor_pool, file_descriptor_set) - message_indices: List of indices (e.g., [0], [1], [2, 0] for nested) - - Returns: - Message class for the specified type - """ - pool, descriptor_set = schema_info - - # First index is the message type in the file - file_descriptor = descriptor_set.file[0] - message_descriptor_proto = file_descriptor.message_type[message_indices[0]] - - package = file_descriptor.package - name_parts = [message_descriptor_proto.name] - - # Handle nested messages if there are more indices - current_proto = message_descriptor_proto - for idx in message_indices[1:]: - current_proto = current_proto.nested_type[idx] - name_parts.append(current_proto.name) - - if package: - full_name = f"{package}.{'.'.join(name_parts)}" - else: - full_name = '.'.join(name_parts) - - message_descriptor = pool.FindMessageTypeByName(full_name) - return message_factory.GetMessageClass(message_descriptor) - - -def _deserialize_protobuf(message, schema_info, uses_schema_registry): - """Deserialize a Protobuf message using google.protobuf with strict validation. - - Args: - message: Raw protobuf bytes - schema_info: Tuple of (descriptor_pool, file_descriptor_set) from build_protobuf_schema - uses_schema_registry: Whether to extract Confluent message indices from the message - """ - try: - if uses_schema_registry: - message_indices, message = _read_protobuf_message_indices(message) - # Empty indices array means use the first message type (index 0) - if not message_indices: - message_indices = [0] - else: - message_indices = [0] - - message_class = _get_protobuf_message_class(schema_info, message_indices) - schema_instance = message_class() - - bytes_consumed = schema_instance.ParseFromString(message) - - # Check if all bytes were consumed (strict validation) - if bytes_consumed != len(message): - raise ValueError( - f"Not all bytes were consumed during Protobuf decoding! " - f"Read {bytes_consumed} bytes, but message has {len(message)} bytes. " - ) - - return MessageToJson(schema_instance) - except Exception as e: - raise ValueError(f"Failed to deserialize Protobuf message: {e}") - - -def _deserialize_avro(message, schema): - """Deserialize an Avro message using fastavro with strict validation.""" - try: - bio = BytesIO(message) - initial_position = bio.tell() - data = schemaless_reader(bio, schema) - final_position = bio.tell() - - # Check if all bytes were consumed (strict validation) - bytes_read = final_position - initial_position - total_bytes = len(message) - - if bytes_read != total_bytes: - raise ValueError( - f"Not all bytes were consumed during Avro decoding! " - f"Read {bytes_read} bytes, but message has {total_bytes} bytes. " - ) - - return json.dumps(data) - except Exception as e: - raise ValueError(f"Failed to deserialize Avro message: {e}") - - -def build_schema(message_format, schema_str): - if message_format == 'protobuf': - return build_protobuf_schema(schema_str) - elif message_format == 'avro': - return build_avro_schema(schema_str) - return None - - -def build_avro_schema(schema_str): - """Build an Avro schema from a JSON string.""" - schema = json.loads(schema_str) - - if schema is None: - raise ValueError("Avro schema cannot be None") - - return schema - - -def build_protobuf_schema(schema_str): - """Build a Protobuf schema from a base64-encoded FileDescriptorSet. - - Returns a tuple of (descriptor_pool, file_descriptor_set) that can be used - to dynamically select and instantiate message types based on message indices. - - Args: - schema_str: Base64-encoded FileDescriptorSet - - Returns: - tuple: (DescriptorPool, FileDescriptorSet) - """ - # schema is encoded in base64, decode it before passing it to ParseFromString - schema_str = base64.b64decode(schema_str) - descriptor_set = descriptor_pb2.FileDescriptorSet() - descriptor_set.ParseFromString(schema_str) - - # Register all the file descriptors in a descriptor pool - pool = descriptor_pool.DescriptorPool() - for fd_proto in descriptor_set.file: - pool.Add(fd_proto) - - return (pool, descriptor_set) diff --git a/kafka_consumer/pyproject.toml b/kafka_consumer/pyproject.toml index f8c83b21fce9a..05f1b6dbb0f69 100644 --- a/kafka_consumer/pyproject.toml +++ b/kafka_consumer/pyproject.toml @@ -39,8 +39,6 @@ deps = [ "aws-msk-iam-sasl-signer-python==1.0.2", "boto3==1.42.72", "confluent-kafka==2.13.2", - "fastavro==1.12.1", - "protobuf==7.34.0", ] [project.urls] diff --git a/kafka_consumer/tests/test_integration.py b/kafka_consumer/tests/test_integration.py index 152eb61025e1b..7f4e3e45753a0 100644 --- a/kafka_consumer/tests/test_integration.py +++ b/kafka_consumer/tests/test_integration.py @@ -8,7 +8,6 @@ import mock import pytest -from confluent_kafka.admin import AdminClient from datadog_checks.dev.utils import get_metadata_metrics @@ -35,27 +34,6 @@ def mocked_time(): return 400 -def get_all_consumer_groups(kafka_instance): - """Get all consumer groups from Kafka cluster.""" - config = { - "bootstrap.servers": kafka_instance['kafka_connect_str'], - "socket.timeout.ms": 1000, - "topic.metadata.refresh.interval.ms": 2000, - } - config.update(common.get_authentication_configuration(kafka_instance)) - admin_client = AdminClient(config) - - final_groups = set() - try: - groups_result = admin_client.list_consumer_groups().result() - for valid_group in groups_result.valid: - final_groups.add(valid_group.group_id) - except Exception as e: - print(f"Error getting final consumer groups: {e}") - - return final_groups - - def test_check_kafka(aggregator, check, kafka_instance, dd_run_check): """ Testing Kafka_consumer check. @@ -488,58 +466,3 @@ def test_regex_consumer_groups( aggregator.assert_metric("kafka.estimated_consumer_lag", count=consumer_lag_seconds_count) assert expected_warning in caplog.text - - -@mock.patch('datadog_checks.kafka_consumer.kafka_consumer.time', mocked_time) -def test_data_streams_live_messages(dd_run_check, check, kafka_instance, datadog_agent): - cluster_id = common.get_cluster_id() - kafka_instance['live_messages_configs'] = [ - { - 'kafka': { - 'cluster': cluster_id, - 'topic': 'marvel', - 'partition': 0, - 'start_offset': 0, - 'n_messages': 2, - 'value_format': 'json', - }, - 'id': 'config_1_id', - } - ] - kafka_check = check(kafka_instance) - dd_run_check(kafka_check) - - # Verify that live messages is not leaving behind any new consumer groups - final_groups = get_all_consumer_groups(kafka_instance) - assert final_groups == {'my_consumer'} - - expected_logs = [ - { - 'timestamp': 400 * 1000, - 'technology': 'kafka', - 'cluster': str(cluster_id), - 'config_id': 'config_1_id', - 'topic': 'marvel', - 'partition': '0', - 'offset': '0', - 'feature': 'data_streams_messages', - 'message_value': '{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"}', - 'ddtags': 'optional:tag1', - }, - { - 'timestamp': 400 * 1000, - 'technology': 'kafka', - 'cluster': str(cluster_id), - 'config_id': 'config_1_id', - 'topic': 'marvel', - 'partition': '0', - 'offset': '1', - 'feature': 'data_streams_messages', - 'message_value': '{"name": "Bruce Banner", "age": 45,\ - "transaction_amount": 456, "currency": "dollar"}', - 'value_schema_id': '350', - 'message_key': '{"name": "Bruce Banner"}', - 'ddtags': 'optional:tag1', - }, - ] - datadog_agent.assert_logs(kafka_check.check_id, expected_logs) diff --git a/kafka_consumer/tests/test_unit.py b/kafka_consumer/tests/test_unit.py index bb1d7a20bd0d3..35bbcacaf1488 100644 --- a/kafka_consumer/tests/test_unit.py +++ b/kafka_consumer/tests/test_unit.py @@ -1,29 +1,15 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import base64 -import json import logging from contextlib import nullcontext as does_not_raise import mock import pytest -from confluent_kafka import TopicPartition -from google.protobuf import descriptor_pb2 -from google.protobuf.message import DecodeError from datadog_checks.kafka_consumer import KafkaCheck from datadog_checks.kafka_consumer.client import KafkaClient -from datadog_checks.kafka_consumer.kafka_consumer import ( - DATA_STREAMS_MESSAGES_CACHE_KEY, - _get_interpolated_timestamp, - _get_protobuf_message_class, - build_avro_schema, - build_protobuf_schema, - build_schema, - deserialize_message, - resolve_start_offsets, -) +from datadog_checks.kafka_consumer.kafka_consumer import _get_interpolated_timestamp pytestmark = [pytest.mark.unit] @@ -543,855 +529,6 @@ def test_add_broker_timestamps_evicts_by_oldest_timestamp(kafka_instance, check) assert 600 in timestamps -def test_resolve_start_offsets(): - highwater_offsets = { - ("topic1", 0): 100, - ("topic1", 1): 200, - ("topic2", 0): 150, - } - assert resolve_start_offsets(highwater_offsets, "topic1", 0, 80, 10) == [TopicPartition("topic1", 0, 80)] - assert resolve_start_offsets(highwater_offsets, "topic2", 0, -1, 10) == [TopicPartition("topic2", 0, 141)] - assert sorted(resolve_start_offsets(highwater_offsets, "topic1", -1, -1, 10)) == [ - TopicPartition("topic1", 0, 81), - TopicPartition("topic1", 1, 191), - ] - - -class MockedMessage: - def __init__(self, value, key=None, offset=0): - self.v = value - self.k = key - self.o = offset - - def value(self): - return self.v - - def key(self): - return self.k - - def partition(self): - return 0 - - def offset(self): - return self.o - - -def test_deserialize_message(): - message = b'{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"}' - # schema ID is 350, which is 0x015E in hex. - # A magic byte (0x00) is added and the schema ID (4-byte big-endian integer). - message_with_schema = ( - b'\x00\x00\x00\x01\x5e{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"}' - ) - key = b'{"name": "Peter Parker"}' - assert deserialize_message(MockedMessage(message, key), 'json', '', False, 'json', '', False) == ( - '{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"}', - None, - '{"name": "Peter Parker"}', - None, - ) - assert deserialize_message(MockedMessage(message_with_schema), 'json', '', False, 'json', '', False) == ( - '{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"}', - 350, - '', - None, - ) - invalid_json = b'{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"' - assert deserialize_message(MockedMessage(invalid_json, key), 'json', '', False, 'json', '', False) == ( - None, - None, - None, - None, - ) - - invalid_utf8 = b'{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"\xff' - assert deserialize_message(MockedMessage(invalid_utf8, key), 'json', '', False, 'json', '', False) == ( - None, - None, - None, - None, - ) - - # Test Avro deserialization - avro_schema = ( - '{"type": "record", "name": "Book", "namespace": "com.book", ' - '"fields": [{"name": "isbn", "type": "long"}, {"name": "title", "type": "string"}, ' - '{"name": "author", "type": "string"}]}' - ) - avro_message = b'\xd0\xf5\xe4\xd6\xa3\xb9\x046The Go Programming Language\x18Alan Donovan' - parsed_avro_schema = build_schema('avro', avro_schema) - assert deserialize_message( - MockedMessage(avro_message, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) == ( - '{"isbn": 9780134190440, "title": "The Go Programming Language", "author": "Alan Donovan"}', - None, - '{"name": "Peter Parker"}', - None, - ) - - # Test Protobuf deserialization - protobuf_schema = ( - 'CmoKDHNjaGVtYS5wcm90bxIIY29tLmJvb2siSAoEQm9vaxISCgRpc2JuGAEgASgDUgRpc2Ju' - 'EhQKBXRpdGxlGAIgASgJUgV0aXRsZRIWCgZhdXRob3IYAyABKAlSBmF1dGhvcmIGcHJvdG8z' - ) - protobuf_message = ( - b'\x08\xe8\xba\xb2\xeb\xd1\x9c\x02\x12\x1b\x54\x68\x65\x20\x47\x6f\x20\x50\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x4c\x61\x6e\x67\x75\x61\x67\x65' - b'\x1a\x0c\x41\x6c\x61\x6e\x20\x44\x6f\x6e\x6f\x76\x61\x6e' - ) - parsed_protobuf_schema = build_schema('protobuf', protobuf_schema) - assert deserialize_message( - MockedMessage(protobuf_message, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) == ( - '{\n "isbn": "9780134190440",\n "title": "The Go Programming Language",\n "author": "Alan Donovan"\n}', - None, - '{"name": "Peter Parker"}', - None, - ) - - # Test invalid Avro messages - # Empty message (returns empty string, not None) - assert deserialize_message(MockedMessage(b'', key), 'avro', parsed_avro_schema, False, 'json', '', False) == ( - '', - None, - '{"name": "Peter Parker"}', - None, - ) - - # Corrupted message (truncated) - corrupted_avro = b'\xd0\xf5\xe4\xd6\xa3\xb9\x046The Go Programming Language' # Missing author field - assert deserialize_message( - MockedMessage(corrupted_avro, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) == ( - None, - None, - None, - None, - ) - - # Wrong data type (string instead of long for isbn) - wrong_type_avro = b'\x02\x12\x1bThe Go Programming Language\x18Alan Donovan' # Wrong encoding for isbn - assert deserialize_message( - MockedMessage(wrong_type_avro, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) == ( - None, - None, - None, - None, - ) - - # Random bytes - random_avro = b'\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0' - assert deserialize_message( - MockedMessage(random_avro, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) == ( - None, - None, - None, - None, - ) - - # Completely invalid Avro message (random bytes) - invalid_avro = b'\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0' - assert deserialize_message( - MockedMessage(invalid_avro, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) == ( - None, - None, - None, - None, - ) - - # Avro message with wrong data types (string where long expected) - wrong_type_avro = b'\x02\x12\x1bThe Go Programming Language\x18Alan Donovan' # Wrong encoding for isbn - assert deserialize_message( - MockedMessage(wrong_type_avro, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) == ( - None, - None, - None, - None, - ) - - # Test invalid Protobuf messages - # Empty message (returns empty string, not None) - assert deserialize_message( - MockedMessage(b'', key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) == ( - '', - None, - '{"name": "Peter Parker"}', - None, - ) - - # Random bytes - random_protobuf = b'\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0' - assert deserialize_message( - MockedMessage(random_protobuf, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) == ( - None, - None, - None, - None, - ) - - # Completely invalid Protobuf message (random bytes) - invalid_protobuf = b'\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8\xf7\xf6\xf5\xf4\xf3\xf2\xf1\xf0' - assert deserialize_message( - MockedMessage(invalid_protobuf, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) == (None, None, None, None) - - # Protobuf message with wrong field number (field 99 instead of 1) - wrong_field_protobuf = ( - b'\x99\x01\xe8\xba\xb2\xeb\xd1\x9c\x02\x12\x1bThe Go Programming Language\x1a\x0cAlan Donovan' - ) - assert deserialize_message( - MockedMessage(wrong_field_protobuf, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) == (None, None, None, None) - - # Protobuf message with truncated varint - truncated_varint_protobuf = b'\x08\xff\xff\xff\xff\xff\xff\xff\xff\xff' # Incomplete varint - assert deserialize_message( - MockedMessage(truncated_varint_protobuf, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) == (None, None, None, None) - - -def test_strict_avro_validation(): - """Test that Avro deserialization fails when not all bytes are consumed.""" - key = b'{"name": "Peter Parker"}' - - # Test case 1: Simple primitive string schema with extra bytes - # A primitive string in Avro is encoded as: varint length + UTF-8 bytes - # An empty string is just: 0x00 (zero length) - # If we have 0x00 followed by extra bytes (e.g., magic byte + 4 bytes + stuff), - # the string decoder will read the empty string but leave bytes unconsumed - string_schema = '"string"' - parsed_string_schema = build_schema('avro', string_schema) - - # Message: 0x00 (empty string) + 0x00 (magic byte) + 4 bytes + some random data - # The Avro string decoder will only consume the first 0x00, leaving the rest - message_with_extra_bytes = b'\x00\x00\x00\x00\x01\x5e\x12\x34\x56\x78' - - # This should now fail because not all bytes are consumed - result = deserialize_message( - MockedMessage(message_with_extra_bytes, key), 'avro', parsed_string_schema, False, 'json', '', False - ) - assert result == (None, None, None, None), "Expected deserialization to fail due to unconsumed bytes" - - # Test case 2: Avro message with trailing garbage bytes after valid data - avro_schema = ( - '{"type": "record", "name": "Book", "namespace": "com.book", ' - '"fields": [{"name": "isbn", "type": "long"}, {"name": "title", "type": "string"}, ' - '{"name": "author", "type": "string"}]}' - ) - parsed_avro_schema = build_schema('avro', avro_schema) - - # Valid Avro message + trailing garbage - valid_avro_message = b'\xd0\xf5\xe4\xd6\xa3\xb9\x046The Go Programming Language\x18Alan Donovan' - message_with_trailing_bytes = valid_avro_message + b'\xff\xfe\xfd\xfc' - - # This should now fail because of the trailing bytes - result = deserialize_message( - MockedMessage(message_with_trailing_bytes, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) - assert result == (None, None, None, None), "Expected deserialization to fail due to trailing bytes" - - # Test case 3: Simple int schema with extra bytes - int_schema = '"int"' - parsed_int_schema = build_schema('avro', int_schema) - - # Message: 0x02 (int value 1) + extra bytes - message_int_with_extra = b'\x02\xde\xad\xbe\xef' - - result = deserialize_message( - MockedMessage(message_int_with_extra, key), 'avro', parsed_int_schema, False, 'json', '', False - ) - assert result == (None, None, None, None), "Expected deserialization to fail due to unconsumed bytes" - - # Test case 4: Verify that valid messages still work - valid_string_message = b'\x0aHello' # Length 5 (encoded as 0x0a = 10/2 = 5) + "Hello" - result = deserialize_message( - MockedMessage(valid_string_message, key), 'avro', parsed_string_schema, False, 'json', '', False - ) - assert result[0] == '"Hello"', "Expected valid string message to deserialize correctly" - assert result[1] is None - - valid_int_message = b'\x02' # int value 1 - result = deserialize_message( - MockedMessage(valid_int_message, key), 'avro', parsed_int_schema, False, 'json', '', False - ) - assert result[0] == '1', "Expected valid int message to deserialize correctly" - - -def test_strict_protobuf_validation(): - """Test that Protobuf deserialization fails when not all bytes are consumed.""" - key = b'{"name": "Peter Parker"}' - - # Build the same Book schema used in other tests - protobuf_schema = ( - 'CmoKDHNjaGVtYS5wcm90bxIIY29tLmJvb2siSAoEQm9vaxISCgRpc2JuGAEgASgDUgRpc2Ju' - 'EhQKBXRpdGxlGAIgASgJUgV0aXRsZRIWCgZhdXRob3IYAyABKAlSBmF1dGhvcmIGcHJvdG8z' - ) - parsed_protobuf_schema = build_schema('protobuf', protobuf_schema) - - # Test case 1: Valid Protobuf message with trailing garbage bytes - valid_protobuf_message = ( - b'\x08\xe8\xba\xb2\xeb\xd1\x9c\x02\x12\x1b\x54\x68\x65\x20\x47\x6f\x20\x50\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x4c\x61\x6e\x67\x75\x61\x67\x65' - b'\x1a\x0c\x41\x6c\x61\x6e\x20\x44\x6f\x6e\x6f\x76\x61\x6e' - ) - message_with_trailing_bytes = valid_protobuf_message + b'\xff\xfe\xfd\xfc' - - # This should now fail because of the trailing bytes - result = deserialize_message( - MockedMessage(message_with_trailing_bytes, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) - assert result == (None, None, None, None), "Expected deserialization to fail due to trailing bytes" - - # Test case 2: Message with extra fields that aren't in the schema - # Protobuf will parse this but leave bytes unconsumed if there are truly extra bytes beyond valid fields - # Adding a completely invalid trailing byte sequence - message_with_invalid_trailer = valid_protobuf_message + b'\x00\x00\x00\x01\x5e' - - result = deserialize_message( - MockedMessage(message_with_invalid_trailer, key), - 'protobuf', - parsed_protobuf_schema, - False, - 'json', - '', - False, - ) - assert result == (None, None, None, None), "Expected deserialization to fail due to unconsumed bytes" - - # Test case 3: Verify that valid messages still work - result = deserialize_message( - MockedMessage(valid_protobuf_message, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) - assert result[0] is not None, "Expected valid protobuf message to deserialize correctly" - assert 'The Go Programming Language' in result[0] - - -def test_schema_registry_explicit_configuration(): - """Test that explicit schema registry configuration is enforced.""" - key = b'{"name": "Peter Parker"}' - - # Test Avro with value_uses_schema_registry=True - avro_schema = ( - '{"type": "record", "name": "Book", "namespace": "com.book", ' - '"fields": [{"name": "isbn", "type": "long"}, {"name": "title", "type": "string"}, ' - '{"name": "author", "type": "string"}]}' - ) - parsed_avro_schema = build_schema('avro', avro_schema) - - # Valid Avro message WITHOUT schema registry format - avro_message_no_sr = b'\xd0\xf5\xe4\xd6\xa3\xb9\x046The Go Programming Language\x18Alan Donovan' - - # When uses_schema_registry=False, this should work - result = deserialize_message( - MockedMessage(avro_message_no_sr, key), 'avro', parsed_avro_schema, False, 'json', '', False - ) - assert result[0] is not None, "Should succeed when uses_schema_registry=False" - assert result[1] is None, "Should have no schema ID" - - # When uses_schema_registry=True, this should fail (missing magic byte and schema ID) - result = deserialize_message( - MockedMessage(avro_message_no_sr, key), 'avro', parsed_avro_schema, True, 'json', '', False - ) - assert result == (None, None, None, None), "Should fail when uses_schema_registry=True" - - # Valid Avro message WITH schema registry format (schema ID 350 = 0x015E) - avro_message_with_sr = ( - b'\x00\x00\x00\x01\x5e\xd0\xf5\xe4\xd6\xa3\xb9\x046The Go Programming Language\x18Alan Donovan' - ) - - # When uses_schema_registry=True, this should work - result = deserialize_message( - MockedMessage(avro_message_with_sr, key), 'avro', parsed_avro_schema, True, 'json', '', False - ) - assert result[0] is not None, "Should succeed when uses_schema_registry=True" - assert result[1] == 350, "Should extract schema ID 350" - assert 'The Go Programming Language' in result[0] - - # Test with wrong magic byte - wrong_magic_byte = b'\x01\x00\x00\x01\x5e\xd0\xf5\xe4\xd6\xa3\xb9\x046The Go Programming Language\x18Alan Donovan' - result = deserialize_message( - MockedMessage(wrong_magic_byte, key), 'avro', parsed_avro_schema, True, 'json', '', False - ) - assert result == (None, None, None, None), "Should fail with wrong magic byte" - - # Test with message too short (less than 5 bytes) - too_short = b'\x00\x00\x01' - result = deserialize_message(MockedMessage(too_short, key), 'avro', parsed_avro_schema, True, 'json', '', False) - assert result == (None, None, None, None), "Should fail when message too short for SR format" - - # Test Protobuf with value_uses_schema_registry=True - protobuf_schema = ( - 'CmoKDHNjaGVtYS5wcm90bxIIY29tLmJvb2siSAoEQm9vaxISCgRpc2JuGAEgASgDUgRpc2Ju' - 'EhQKBXRpdGxlGAIgASgJUgV0aXRsZRIWCgZhdXRob3IYAyABKAlSBmF1dGhvcmIGcHJvdG8z' - ) - parsed_protobuf_schema = build_schema('protobuf', protobuf_schema) - - # Valid Protobuf message WITHOUT schema registry format - protobuf_message_no_sr = ( - b'\x08\xe8\xba\xb2\xeb\xd1\x9c\x02\x12\x1b\x54\x68\x65\x20\x47\x6f\x20\x50\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x4c\x61\x6e\x67\x75\x61\x67\x65' - b'\x1a\x0c\x41\x6c\x61\x6e\x20\x44\x6f\x6e\x6f\x76\x61\x6e' - ) - - # When uses_schema_registry=False, this should work - result = deserialize_message( - MockedMessage(protobuf_message_no_sr, key), 'protobuf', parsed_protobuf_schema, False, 'json', '', False - ) - assert result[0] is not None, "Protobuf should succeed when uses_schema_registry=False" - assert result[1] is None, "Should have no schema ID" - - # When uses_schema_registry=True, this should fail - result = deserialize_message( - MockedMessage(protobuf_message_no_sr, key), 'protobuf', parsed_protobuf_schema, True, 'json', '', False - ) - assert result == (None, None, None, None), "Protobuf should fail when uses_schema_registry=True but no SR format" - - # Valid Protobuf message WITH schema registry format - # Confluent Protobuf wire format: - # [magic_byte][schema_id:4bytes][array_length:varint][index:varint][protobuf_payload] - protobuf_message_with_sr = ( - b'\x00\x00\x00\x01\x5e' # magic byte (0x00) + schema ID 350 (0x0000015e) - b'\x01' # message indices array length = 1 - b'\x00' # message index = 0 - b'\x08\xe8\xba\xb2\xeb\xd1\x9c\x02\x12\x1b\x54\x68\x65\x20\x47\x6f\x20\x50\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x4c\x61\x6e\x67\x75\x61\x67\x65' - b'\x1a\x0c\x41\x6c\x61\x6e\x20\x44\x6f\x6e\x6f\x76\x61\x6e' - ) - - # When uses_schema_registry=True, this should work - result = deserialize_message( - MockedMessage(protobuf_message_with_sr, key), - 'protobuf', - parsed_protobuf_schema, - True, - 'json', - '', - False, - ) - assert result[0] is not None, "Protobuf should succeed when uses_schema_registry=True with SR format" - assert result[1] == 350, "Should extract schema ID 350" - assert 'The Go Programming Language' in result[0] - - # Test key_uses_schema_registry=True - # When key has no schema registry format but key_uses_schema_registry=True, key decoding should fail - # but value should still succeed - result = deserialize_message( - MockedMessage(avro_message_no_sr, key), 'avro', parsed_avro_schema, False, 'json', '', True - ) - # Value should succeed, but key should fail (returning None for key fields) - assert result[0] is not None, "Value should succeed" - assert result[2] is None, "Key should fail when key_uses_schema_registry=True but no SR format" - assert result[3] is None, "Key schema ID should be None when key fails" - - -def test_protobuf_message_indices_with_schema_registry(): - """Test Confluent Protobuf wire format with different message indices.""" - key = b'{"test": "key"}' - - # Schema with multiple message types and nested type - # message Book { int64 isbn = 1; string title = 2; } - # message Author { string name = 1; int32 age = 2; } - # message Library { message Section { string name = 1; } string name = 1; } - protobuf_schema = ( - 'CpMBCgxzY2hlbWEucHJvdG8SC2NvbS5leGFtcGxlIh8KBEJvb2sSCgoEaXNibhgBKAMSCwoFdGl0bGUY' - 'AigJIh8KBkF1dGhvchIKCgRuYW1lGAEoCRIJCgNhZ2UYAigFIiwKB0xpYnJhcnkSCgoEbmFtZRgBKAka' - 'FQoHU2VjdGlvbhIKCgRuYW1lGAEoCWIGcHJvdG8z' - ) - parsed_schema = build_schema('protobuf', protobuf_schema) - - # Test index [0] - Book message - book_payload = bytes.fromhex('08e80712095465737420426f6f6b') - book_msg = b'\x00\x00\x00\x01\x5e\x01\x00' + book_payload - result = deserialize_message(MockedMessage(book_msg, key), 'protobuf', parsed_schema, True, 'json', '', False) - assert result[0] and 'Test Book' in result[0] - - # Test index [1] - Author message - author_payload = bytes.fromhex('0a0a4a616e6520536d697468101e') - author_msg = b'\x00\x00\x00\x01\x5e\x01\x01' + author_payload - result = deserialize_message(MockedMessage(author_msg, key), 'protobuf', parsed_schema, True, 'json', '', False) - assert result[0] and 'Jane Smith' in result[0] and '30' in result[0] - - # Test nested [2, 0] - Library.Section message - section_payload = bytes.fromhex('0a0746696374696f6e') - section_msg = b'\x00\x00\x00\x01\x5e\x02\x02\x00' + section_payload - result = deserialize_message(MockedMessage(section_msg, key), 'protobuf', parsed_schema, True, 'json', '', False) - assert result[0] and 'Fiction' in result[0] - - -def test_protobuf_empty_message_indices_with_schema_registry(): - """Test Confluent Protobuf wire format with empty message indices array. - - When message indices array is empty (encoded as varint 0x00), it should - default to using the first message type (index 0). - - This test uses real message bytes from a Kafka topic to ensure the - deserialization handles the Confluent wire format correctly. - """ - key = b'null' - - # Schema from real Kafka topic - Purchase message - # message Purchase { string order_id = 1; string customer_id = 2; int64 order_date = 3; - # string city = 6; string country = 7; } - protobuf_schema = ( - 'CrkDCgxzY2hlbWEucHJvdG8SCHB1cmNoYXNlIpMBCghQdXJjaGFzZRIZCghvcmRlcl9pZBgBIAEoCVIH' - 'b3JkZXJJZBIfCgtjdXN0b21lcl9pZBgCIAEoCVIKY3VzdG9tZXJJZBIdCgpvcmRlcl9kYXRlGAMgASgD' - 'UglvcmRlckRhdGUSEgoEY2l0eRgGIAEoCVIEY2l0eRIYCgdjb3VudHJ5GAcgASgJUgdjb3VudHJ5ItIB' - 'CgpQdXJjaGFzZVYyEiUKDnRyYW5zYWN0aW9uX2lkGAEgASgJUg10cmFuc2FjdGlvbklkEhcKB3VzZXJf' - 'aWQYAiABKAlSBnVzZXJJZBIcCgl0aW1lc3RhbXAYAyABKANSCXRpbWVzdGFtcBIaCghsb2NhdGlvbhgE' - 'IAEoCVIIbG9jYXRpb24SFgoGcmVnaW9uGAUgASgJUgZyZWdpb24SFgoGYW1vdW50GAYgASgBUgZhbW91' - 'bnQSGgoIY3VycmVuY3kYByABKAlSCGN1cnJlbmN5QiwKG2RhdGFkb2cua2Fma2EuZXhhbXBsZS5wcm90' - 'b0INUHVyY2hhc2VQcm90b2IGcHJvdG8z' - ) - parsed_schema = build_schema('protobuf', protobuf_schema) - - # Real message from Kafka topic "human-orders" - # Hex breakdown: - # 00 00 00 00 01 - Schema Registry header (magic byte + schema ID 1) - # 00 - Empty message indices array (varint 0 = 0 elements) - # 0a 05 31 32 33 34 35 ... - Protobuf payload (Purchase message) - message_hex = '0000000001000a0531323334351205363738393018f4eae0c4b8333a064d657869636f' - message_bytes = bytes.fromhex(message_hex) - - # Test with uses_schema_registry=True (explicit) - result = deserialize_message(MockedMessage(message_bytes, key), 'protobuf', parsed_schema, True, 'json', '', False) - assert result[0], "Deserialization should succeed" - assert '12345' in result[0], "Should contain order_id" - assert '67890' in result[0], "Should contain customer_id" - assert 'Mexico' in result[0], "Should contain country" - assert result[1] == 1, "Should detect schema ID 1" - - # Test with uses_schema_registry=False (fallback mode) - result_fallback = deserialize_message( - MockedMessage(message_bytes, key), 'protobuf', parsed_schema, False, 'json', '', False - ) - assert result_fallback[0], "Fallback mode should also succeed" - assert '12345' in result_fallback[0], "Fallback should contain order_id" - assert result_fallback[1] == 1, "Fallback should detect schema ID 1" - - -def mocked_time(): - return 400 - - -@mock.patch('datadog_checks.kafka_consumer.kafka_consumer.time', mocked_time) -@pytest.mark.parametrize( - 'messages, value_format, value_schema, persistent_cache_read_content, ' - 'expected_persistent_cache_writes, expected_logs', - [ - pytest.param( - [ - MockedMessage( - b'{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"}', - b'{"name": "Peter Parker"}', - 12, - ), - MockedMessage( - b'{"name": "Bruce Banner", "age": 45, "transaction_amount": 456, "currency": "dollar"}', - b'', - 13, - ), - None, - ], - 'json', - '', - "config_1_id,config_id_2", - [], - [], - id='Does not retrieve messages a second time', - ), - pytest.param( - [ - MockedMessage( - b'{"name": "Peter Parker", "age": 18, "transaction_amount": 123, "currency": "dollar"}', - b'{"name": "Peter Parker"}', - 12, - ), - MockedMessage( - b'{"name": "Bruce Banner", "age": 45, "transaction_amount": 456, "currency": "dollar"}', - b'', - 13, - ), - None, - ], - 'json', - '', - "", - ["config_1_id"], - [ - { - 'timestamp': 400, - 'technology': 'kafka', - 'cluster': 'cluster_id', - 'config_id': 'config_1_id', - 'topic': 'topic1', - 'partition': '0', - 'offset': '12', - 'feature': 'data_streams_messages', - 'message_value': '{"name": "Peter Parker", "age": 18, \ -"transaction_amount": 123, "currency": "dollar"}', - 'message_key': '{"name": "Peter Parker"}', - }, - { - 'timestamp': 400, - 'technology': 'kafka', - 'cluster': 'cluster_id', - 'config_id': 'config_1_id', - 'topic': 'topic1', - 'partition': '0', - 'offset': '13', - 'feature': 'data_streams_messages', - 'message_value': '{"name": "Bruce Banner", "age": 45, \ -"transaction_amount": 456, "currency": "dollar"}', - }, - { - 'timestamp': 400, - 'technology': 'kafka', - 'cluster': 'cluster_id', - 'config_id': 'config_1_id', - 'topic': 'topic1', - 'message': 'No more messages to retrieve', - 'live_messages_error': 'No more messages to retrieve', - 'feature': 'data_streams_messages', - }, - ], - id='Retrieves messages from Kafka', - ), - # This is the serialized Protobuf representing: - # syntax = "proto3"; - # package com.book; - # message Book { - # int64 isbn = 1; - # string title = 2; - # string author = 3; - # } - pytest.param( - [ - MockedMessage( - b'\x08\xe8\xba\xb2\xeb\xd1\x9c\x02\x12\x1b\x54\x68\x65\x20\x47\x6f\x20\x50\x72\x6f\x67\x72\x61\x6d\x6d\x69\x6e\x67\x20\x4c\x61\x6e\x67\x75\x61\x67\x65\x1a\x0c\x41\x6c\x61\x6e\x20\x44\x6f\x6e\x6f\x76\x61\x6e', - b'{"name": "Peter Parker"}', - 12, - ), - None, - ], - 'protobuf', - 'CmoKDHNjaGVtYS5wcm90bxIIY29tLmJvb2siSAoEQm9vaxISCgRpc2JuGAEgASgDUgRpc2JuEhQKBXRpdGxlGAIgASgJUgV0aXRsZRIWCgZhdXRob3IYAyABKAlSBmF1dGhvcmIGcHJvdG8z', - "", - ["config_1_id"], - [ - { - 'timestamp': 400, - 'technology': 'kafka', - 'cluster': 'cluster_id', - 'config_id': 'config_1_id', - 'topic': 'topic1', - 'partition': '0', - 'offset': '12', - 'feature': 'data_streams_messages', - 'message_value': ( - '{\n "isbn": "9780134190440",\n "title": "The Go Programming Language",\n ' - '"author": "Alan Donovan"\n}' - ), - 'message_key': '{"name": "Peter Parker"}', - }, - { - 'timestamp': 400, - 'technology': 'kafka', - 'cluster': 'cluster_id', - 'config_id': 'config_1_id', - 'topic': 'topic1', - 'message': 'No more messages to retrieve', - 'live_messages_error': 'No more messages to retrieve', - 'feature': 'data_streams_messages', - }, - ], - id='Retrieves Protobuf messages from Kafka', - ), - pytest.param( - [ - MockedMessage( - b'\xd0\xf5\xe4\xd6\xa3\xb9\x046The Go Programming Language\x18Alan Donovan', - b'{"name": "Peter Parker"}', - 12, - ), - None, - ], - 'avro', - ( - '{"type": "record", "name": "Book", "namespace": "com.book", ' - '"fields": [{"name": "isbn", "type": "long"}, {"name": "title", "type": "string"}, ' - '{"name": "author", "type": "string"}]}' - ), - "", - ["config_1_id"], - [ - { - 'timestamp': 400, - 'technology': 'kafka', - 'cluster': 'cluster_id', - 'config_id': 'config_1_id', - 'topic': 'topic1', - 'partition': '0', - 'offset': '12', - 'feature': 'data_streams_messages', - 'message_value': ( - '{"isbn": 9780134190440, "title": "The Go Programming Language", "author": "Alan Donovan"}' - ), - 'message_key': '{"name": "Peter Parker"}', - }, - { - 'timestamp': 400, - 'technology': 'kafka', - 'cluster': 'cluster_id', - 'config_id': 'config_1_id', - 'topic': 'topic1', - 'message': 'No more messages to retrieve', - 'live_messages_error': 'No more messages to retrieve', - 'feature': 'data_streams_messages', - }, - ], - id='Retrieves Avro messages from Kafka', - ), - ], -) -def test_data_streams_messages( - messages, - value_format, - value_schema, - persistent_cache_read_content, - expected_persistent_cache_writes, - expected_logs, - kafka_instance, - dd_run_check, - check, -): - ( - kafka_instance.update( - { - 'consumer_groups': {}, - 'monitor_unlisted_consumer_groups': True, - 'live_messages_configs': [ - { - 'kafka': { - 'cluster': 'cluster_id', - 'topic': 'topic1', - 'partition': 0, - 'start_offset': 0, - 'n_messages': 3, - 'value_format': value_format, - 'value_schema': value_schema, - 'key_format': 'json', - 'key_schema': '', - }, - 'id': 'config_1_id', - } - ], - } - ), - ) - mock_client = seed_mock_client(cluster_id="Cluster_id") - mock_client.get_next_message.side_effect = messages - check = check(kafka_instance) - check.client = mock_client - - def mocked_read_persistent_cache(key): - if key == DATA_STREAMS_MESSAGES_CACHE_KEY: - return persistent_cache_read_content - return "" - - check.read_persistent_cache = mock.Mock(side_effect=mocked_read_persistent_cache) - check.write_persistent_cache = mock.Mock() - check.send_log = mock.Mock() - - dd_run_check(check) - - for content in expected_persistent_cache_writes: - assert mock.call(DATA_STREAMS_MESSAGES_CACHE_KEY, content) in check.write_persistent_cache.mock_calls - assert [mock.call(log) for log in expected_logs] == check.send_log.mock_calls - - -def test_build_schema(): - """Test build_schema function with various valid and invalid schemas.""" - - # Test JSON format (should return None) - assert build_schema('json', '') is None - assert build_schema('json', '{"some": "json"}') is None - assert build_schema('json', None) is None - - # Test valid Avro schema - valid_avro_schema = ( - '{"type": "record", "name": "Book", "namespace": "com.book", ' - '"fields": [{"name": "isbn", "type": "long"}, {"name": "title", "type": "string"}, ' - '{"name": "author", "type": "string"}]}' - ) - avro_result = build_schema('avro', valid_avro_schema) - assert avro_result is not None - assert avro_result['type'] == 'record' - assert avro_result['name'] == 'Book' - assert avro_result['namespace'] == 'com.book' - - # Test valid Protobuf schema - valid_protobuf_schema = ( - 'CmoKDHNjaGVtYS5wcm90bxIIY29tLmJvb2siSAoEQm9vaxISCgRpc2JuGAEgASgDUgRpc2Ju' - 'EhQKBXRpdGxlGAIgASgJUgV0aXRsZRIWCgZhdXRob3IYAyABKAlSBmF1dGhvcmIGcHJvdG8z' - ) - protobuf_result = build_schema('protobuf', valid_protobuf_schema) - message_class = _get_protobuf_message_class(protobuf_result, [0]) - message_instance = message_class() - assert hasattr(message_instance, 'isbn') - assert hasattr(message_instance, 'title') - assert hasattr(message_instance, 'author') - - # Test unknown format - assert build_schema('unknown_format', 'some_schema') is None - - -def test_build_schema_error_cases(): - """Test build_schema with various error cases and edge cases.""" - - # Test Avro error cases - # Invalid JSON syntax - with pytest.raises(json.JSONDecodeError): - build_schema('avro', '{"invalid": json}') - - # Valid JSON but incomplete schema (fastavro is permissive) - result = build_schema('avro', '{"type": "record"}') # Missing name and fields - assert result is not None - - # Test Protobuf error cases - # Invalid base64 encoding - with pytest.raises(base64.binascii.Error): - build_schema('protobuf', 'invalid-base64!') - - # Valid base64 but invalid protobuf schema - # This is a valid base64 string that doesn't represent a valid FileDescriptorSet - with pytest.raises(DecodeError): # Will be a protobuf DecodeError - build_schema('protobuf', 'SGVsbG8gV29ybGQ=') # "Hello World" in base64 - - # Valid base64 but empty schema - should fail when trying to access message types - # Create a minimal but empty FileDescriptorSet - empty_descriptor = descriptor_pb2.FileDescriptorSet() - empty_descriptor_bytes = empty_descriptor.SerializeToString() - empty_descriptor_b64 = base64.b64encode(empty_descriptor_bytes).decode('utf-8') - - result = build_schema('protobuf', empty_descriptor_b64) - with pytest.raises(IndexError): # Should fail when trying to access file[0] - _get_protobuf_message_class(result, [0]) - - -def test_build_schema_none_handling(): - """Test that build_schema functions properly handle None values.""" - - # Test Avro schema with None - should raise TypeError - with pytest.raises(TypeError): - build_avro_schema(None) - - # Test Protobuf schema with None - should raise TypeError or base64.binascii.Error - with pytest.raises((TypeError, base64.binascii.Error)): - build_protobuf_schema(None) - - def test_count_consumer_contexts(check, kafka_instance): kafka_consumer_check = check(kafka_instance) consumer_offsets = { diff --git a/kubelet/metadata.csv b/kubelet/metadata.csv index 7b1460a0bcaf4..f82a55aee1c9f 100644 --- a/kubelet/metadata.csv +++ b/kubelet/metadata.csv @@ -15,6 +15,8 @@ kubernetes.cpu.cfs.throttled.seconds,gauge,,,,Total time duration the container kubernetes.cpu.capacity,gauge,,core,,The number of cores in this machine (available until kubernetes v1.18),0,kubernetes,k8s.cpu.capacity, kubernetes.cpu.usage.total,gauge,,nanocore,,The number of cores used,-1,kubernetes,k8s.cpu, kubernetes.cpu.limits,gauge,,core,,The limit of cpu cores set,0,kubernetes,k8s.cpu.limits, +kubernetes.pod.cpu.request,gauge,,core,,The pod-level requested CPU cores,0,kubernetes,k8s.pod.cpu.request, +kubernetes.pod.cpu.limit,gauge,,core,,The pod-level CPU core limit,0,kubernetes,k8s.pod.cpu.limit, kubernetes.cpu.requests,gauge,,core,,The requested cpu cores,0,kubernetes,k8s.cpu.requests, kubernetes.filesystem.usage,gauge,,byte,,The amount of disk used,-1,kubernetes,k8s.disk.usage, kubernetes.filesystem.usage_pct,gauge,,fraction,,The percentage of disk used,-1,kubernetes,k8s.disk.used_pct, @@ -24,6 +26,8 @@ kubernetes.memory.capacity,gauge,,byte,,The amount of memory (in bytes) in this kubernetes.memory.limits,gauge,,byte,,The limit of memory set,0,kubernetes,k8s.mem.limits, kubernetes.memory.sw_limit,gauge,,byte,,The limit of swap space set,0,kubernetes,k8s.mem.sw_limit, kubernetes.memory.requests,gauge,,byte,,The requested memory,0,kubernetes,k8s.mem.requests, +kubernetes.pod.memory.request,gauge,,byte,,The pod-level requested memory in bytes,0,kubernetes,k8s.pod.mem.request, +kubernetes.pod.memory.limit,gauge,,byte,,The pod-level memory limit in bytes,0,kubernetes,k8s.pod.mem.limit, kubernetes.memory.usage,gauge,,byte,,Current memory usage in bytes including all memory regardless of when it was accessed,-1,kubernetes,k8s.mem, kubernetes.memory.working_set,gauge,,byte,,Current working set in bytes - this is what the OOM killer is watching for,-1,kubernetes,k8s.mem.ws, kubernetes.memory.cache,gauge,,byte,,The amount of memory that is being used to cache data from disk (e.g. memory contents that can be associated precisely with a block on a block device),-1,kubernetes,k8s.mem.cache, diff --git a/kubernetes/assets/monitors/monitor_deployments_replicas.json b/kubernetes/assets/monitors/monitor_deployments_replicas.json index 39b6fb9816c5f..374a02aae8603 100644 --- a/kubernetes/assets/monitors/monitor_deployments_replicas.json +++ b/kubernetes/assets/monitors/monitor_deployments_replicas.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-07-28", - "last_updated_at": "2025-06-12", + "last_updated_at": "2026-04-09", "title": "Kubernetes Deployment Replicas are failing", "tags": [ "integration:kubernetes" ], "description": "Kubernetes replicas are clones that facilitate self-healing for pods. Each pod has a desired number of replica Pods that should be running at any given time. This monitor tracks the number of replicas that are failing per deployment.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nThere are at least 2 or more missing replicas for Deployment {{kube_namespace.name}}/{{kube_deployment.name}} over the last 15 minutes.\n\n{{/is_alert}}", + "message": "{{#is_alert}}\n\n## What's happening?\nThere are at least 2 or more missing replicas for Deployment {{kube_namespace.name}}/{{kube_deployment.name}} over the last 15 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+kube_deployment:{{kube_deployment.name}}+kube_namespace:{{kube_namespace.name}})\n- [Metrics Explorer (kubernetes_state.deployment.replicas_desired)](/metric/explorer?exp_metric=kubernetes_state.deployment.replicas_desired&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_deployment:{{kube_deployment.name}},kube_namespace:{{kube_namespace.name}}&exp_agg=avg&exp_type=line)\n- [Metrics Explorer (kubernetes_state.deployment.replicas_available)](/metric/explorer?exp_metric=kubernetes_state.deployment.replicas_available&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_deployment:{{kube_deployment.name}},kube_namespace:{{kube_namespace.name}}&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}", "name": "[Kubernetes] Monitor Kubernetes Deployments Replica Pods", "options": { "escalation_message": "", diff --git a/kubernetes/assets/monitors/monitor_node_unavailable.json b/kubernetes/assets/monitors/monitor_node_unavailable.json index 37ff9c574dcec..cc57835121156 100644 --- a/kubernetes/assets/monitors/monitor_node_unavailable.json +++ b/kubernetes/assets/monitors/monitor_node_unavailable.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-07-28", - "last_updated_at": "2025-06-12", + "last_updated_at": "2026-04-09", "title": "Nodes are unavailable", "tags": [ "integration:kubernetes" ], "description": "Kubernetes nodes can either be schedulable or unschedulable. When unschedulable, the node prevents the scheduler from placing new pods onto that node. This monitor tracks the percentage of schedulable nodes.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nThe percentage of schedulable nodes is below 80% for status:schedulable on ({{kube_cluster_name.name}} cluster over the last 15 minutes.\n\n{{/is_alert}}\n\n Keep in mind that this might be expected based on your infrastructure.", + "message": "{{#is_alert}}\n\n## What's happening?\nThe percentage of schedulable nodes is below 80% for status:schedulable on ({{kube_cluster_name.name}} cluster over the last 15 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+status:schedulable)\n- [Hosts](/infrastructure/hosts?scope=kube_cluster_name:{{kube_cluster_name.name}})\n- [Metrics Explorer (kubernetes_state.node.status)](/metric/explorer?exp_metric=kubernetes_state.node.status&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},status:schedulable&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}\n\n Keep in mind that this might be expected based on your infrastructure.", "name": "[Kubernetes] Monitor Unschedulable Kubernetes Nodes", "options": { "escalation_message": "", diff --git a/kubernetes/assets/monitors/monitor_pod_crashloopbackoff.json b/kubernetes/assets/monitors/monitor_pod_crashloopbackoff.json index 1b14f874c716a..317eec3fd0032 100644 --- a/kubernetes/assets/monitors/monitor_pod_crashloopbackoff.json +++ b/kubernetes/assets/monitors/monitor_pod_crashloopbackoff.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-07-28", - "last_updated_at": "2025-06-12", + "last_updated_at": "2026-04-09", "title": "Pod is in a CrashloopBackOff state", "tags": [ "integration:kubernetes" ], "description": "The status CrashloopBackOff means that a container in the Pod is started, crashes, and is restarted, over and over again. This monitor tracks when a pod is in a CrashloopBackOff state for your Kubernetes integration.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nAt least one container in pod {{pod_name.name}} on {{kube_namespace.name}} is in a waiting state due to reason crashloopbackoff in the last 10 minutes.\n\n{{/is_alert}}\n\n This alert could generate several alerts for a bad deployment. Adjust the thresholds of the query to suit your infrastructure.", + "message": "{{#is_alert}}\n\n## What's happening?\nAt least one container in pod {{pod_name.name}} on {{kube_namespace.name}} is in a waiting state due to reason crashloopbackoff in the last 10 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+kube_namespace:{{kube_namespace.name}}+pod_name:{{pod_name.name}}+reason:crashloopbackoff)\n- [Pod Explorer](/orchestration/explorer/pod?query={{pod_name.name}})\n- [Metrics Explorer (kubernetes_state.container.status_report.count.waiting)](/metric/explorer?exp_metric=kubernetes_state.container.status_report.count.waiting&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_namespace:{{kube_namespace.name}},pod_name:{{pod_name.name}},reason:crashloopbackoff&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}\n\n This alert could generate several alerts for a bad deployment. Adjust the thresholds of the query to suit your infrastructure.", "name": "[Kubernetes] Pod {{pod_name.name}} is CrashloopBackOff on namespace {{kube_namespace.name}}", "options": { "escalation_message": "", diff --git a/kubernetes/assets/monitors/monitor_pod_imagepullbackoff.json b/kubernetes/assets/monitors/monitor_pod_imagepullbackoff.json index 07f30a6eb7b44..a42c9be57e11d 100644 --- a/kubernetes/assets/monitors/monitor_pod_imagepullbackoff.json +++ b/kubernetes/assets/monitors/monitor_pod_imagepullbackoff.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-09-15", - "last_updated_at": "2025-06-12", + "last_updated_at": "2026-04-09", "title": "Pod is in an ImagePullBackOff state", "tags": [ "integration:kubernetes" ], "description": "The status ImagePullBackOff means that a container could not start because Kubernetes could not pull a container image. This monitor tracks when a pod is in an ImagePullBackOff state for your Kubernetes integration.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nAt least one container in pod {{pod_name.name}} on namespace {{kube_namespace.name}} is in a waiting state due to an ImagePullBackOff error in the last 10 minutes.\n\n{{/is_alert}}\n\n This could happen for several reasons, for example a bad image path or tag or if the credentials for pulling images are not configured properly.", + "message": "{{#is_alert}}\n\n## What's happening?\nAt least one container in pod {{pod_name.name}} on namespace {{kube_namespace.name}} is in a waiting state due to an ImagePullBackOff error in the last 10 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+kube_namespace:{{kube_namespace.name}}+pod_name:{{pod_name.name}}+reason:imagepullbackoff)\n- [Pod Explorer](/orchestration/explorer/pod?query={{pod_name.name}})\n- [Metrics Explorer (kubernetes_state.container.status_report.count.waiting)](/metric/explorer?exp_metric=kubernetes_state.container.status_report.count.waiting&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_namespace:{{kube_namespace.name}},pod_name:{{pod_name.name}},reason:imagepullbackoff&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}\n\n This could happen for several reasons, for example a bad image path or tag or if the credentials for pulling images are not configured properly.", "name": "[Kubernetes] Pod {{pod_name.name}} is ImagePullBackOff on namespace {{kube_namespace.name}}", "options": { "escalation_message": "", diff --git a/kubernetes/assets/monitors/monitor_pod_oomkilled.json b/kubernetes/assets/monitors/monitor_pod_oomkilled.json index 3eece4a5d9e41..e4f7ad7aa755c 100644 --- a/kubernetes/assets/monitors/monitor_pod_oomkilled.json +++ b/kubernetes/assets/monitors/monitor_pod_oomkilled.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2025-09-15", - "last_updated_at": "2025-09-15", + "last_updated_at": "2026-04-09", "title": "Pod is in an OOMKilled state", "tags": [ "integration:kubernetes" ], "description": "The status OOMKilled means that a container was killed because it exceeded memory limits or the node ran out of available memory. This monitor tracks when a pod is in an OOMKilled state for your Kubernetes integration.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nThere has been at least one container terminated in pod {{pod_name.name}} on namespace {{kube_namespace.name}} with reason oomkilled in the last 10 minutes.\n\n{{/is_alert}}\n\n This could happen for several reasons, for example insufficient memory limits, memory leaks in the application, or the node running out of available memory.", + "message": "{{#is_alert}}\n\n## What's happening?\nThere has been at least one container terminated in pod {{pod_name.name}} on namespace {{kube_namespace.name}} with reason oomkilled in the last 10 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+kube_namespace:{{kube_namespace.name}}+pod_name:{{pod_name.name}}+reason:oomkilled)\n- [Pod Explorer](/orchestration/explorer/pod?query={{pod_name.name}})\n- [Metrics Explorer (kubernetes.containers.state.terminated)](/metric/explorer?exp_metric=kubernetes.containers.state.terminated&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_namespace:{{kube_namespace.name}},pod_name:{{pod_name.name}},reason:oomkilled&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}\n\n This could happen for several reasons, for example insufficient memory limits, memory leaks in the application, or the node running out of available memory.", "name": "[Kubernetes] Pod {{pod_name.name}} is OOMKilled on namespace {{kube_namespace.name}}", "options": { "escalation_message": "", diff --git a/kubernetes/assets/monitors/monitor_pods_failed_state.json b/kubernetes/assets/monitors/monitor_pods_failed_state.json index 708a41da74ee4..33ee1b348348e 100644 --- a/kubernetes/assets/monitors/monitor_pods_failed_state.json +++ b/kubernetes/assets/monitors/monitor_pods_failed_state.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-07-28", - "last_updated_at": "2025-06-12", + "last_updated_at": "2026-04-09", "title": "Pods are failing", "tags": [ "integration:kubernetes" ], "description": "When a pod is failing it means the container either exited with non-zero status or was terminated by the system. This monitor tracks when more than 10 pods are failing for a given Kubernetes cluster.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nThe number of failed pods has increased by more than 10 in ({{kube_cluster_name.name}} cluster in the last 5 minutes.\n\n{{/is_alert}}\n\n The threshold of ten pods varies depending on your infrastructure. Change the threshold to suit your needs.", + "message": "{{#is_alert}}\n\n## What's happening?\nThe number of failed pods has increased by more than 10 in ({{kube_cluster_name.name}} cluster in the last 5 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+kube_namespace:{{kube_namespace.name}}+pod_phase:failed)\n- [Metrics Explorer (kubernetes_state.pod.status_phase)](/metric/explorer?exp_metric=kubernetes_state.pod.status_phase&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_namespace:{{kube_namespace.name}},pod_phase:failed&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}\n\n The threshold of ten pods varies depending on your infrastructure. Change the threshold to suit your needs.", "name": "[Kubernetes] Monitor Kubernetes Failed Pods in Namespaces", "options": { "escalation_message": "", diff --git a/kubernetes/assets/monitors/monitor_pods_restarting.json b/kubernetes/assets/monitors/monitor_pods_restarting.json index f35d90c629c09..c7cccced75755 100644 --- a/kubernetes/assets/monitors/monitor_pods_restarting.json +++ b/kubernetes/assets/monitors/monitor_pods_restarting.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-07-28", - "last_updated_at": "2025-06-12", + "last_updated_at": "2026-04-09", "title": "Pods are restarting", "tags": [ "integration:kubernetes" ], "description": "Kubernetes pods restart according to the restart policy. A restarting container can indicate problems with memory, CPU usage, or an application exiting prematurely. This monitor tracks when pods are restarting multiple times.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nThere has been an increase of more than 5 container restarts in the pod {{pod_name.name}} in the last 5 minutes.\n\n{{/is_alert}}", + "message": "{{#is_alert}}\n\n## What's happening?\nThere has been an increase of more than 5 container restarts in the pod {{pod_name.name}} in the last 5 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+pod_name:{{pod_name.name}})\n- [Pod Explorer](/orchestration/explorer/pod?query={{pod_name.name}})\n- [Metrics Explorer (kubernetes.containers.restarts)](/metric/explorer?exp_metric=kubernetes.containers.restarts&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},pod_name:{{pod_name.name}}&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}", "name": "[Kubernetes] Monitor Kubernetes Pods Restarting", "options": { "escalation_message": "", diff --git a/kubernetes/assets/monitors/monitor_statefulset_replicas.json b/kubernetes/assets/monitors/monitor_statefulset_replicas.json index b0954fe2785bb..ef5fbc979d832 100644 --- a/kubernetes/assets/monitors/monitor_statefulset_replicas.json +++ b/kubernetes/assets/monitors/monitor_statefulset_replicas.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-07-28", - "last_updated_at": "2025-06-12", + "last_updated_at": "2026-04-09", "title": "Kubernetes Statefulset Replicas are failing", "tags": [ "integration:kubernetes" ], "description": "Kubernetes replicas are clones that facilitate self-healing for pods. Each pod has a desired number of replica Pods that should be running at any given time. This monitor tracks when the number of replicas per statefulset is falling.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nThere are at least 2 desired replicas that are not ready for {{kube_namespace.name}}/{{kube_stateful_set.name}} StatefulSet over the last 15 minutes.\n\n{{/is_alert}}\n\n This might present an unsafe situation for any further manual operations, such as killing other pods.", + "message": "{{#is_alert}}\n\n## What's happening?\nThere are at least 2 desired replicas that are not ready for {{kube_namespace.name}}/{{kube_stateful_set.name}} StatefulSet over the last 15 minutes.\n\n## Related Links\n\n- [Logs](/logs?query=kube_cluster_name:{{kube_cluster_name.name}}+kube_namespace:{{kube_namespace.name}}+kube_stateful_set:{{kube_stateful_set.name}})\n- [Metrics Explorer (kubernetes_state.statefulset.replicas_desired)](/metric/explorer?exp_metric=kubernetes_state.statefulset.replicas_desired&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_namespace:{{kube_namespace.name}},kube_stateful_set:{{kube_stateful_set.name}}&exp_agg=avg&exp_type=line)\n- [Metrics Explorer (kubernetes_state.statefulset.replicas_ready)](/metric/explorer?exp_metric=kubernetes_state.statefulset.replicas_ready&exp_scope=kube_cluster_name:{{kube_cluster_name.name}},kube_namespace:{{kube_namespace.name}},kube_stateful_set:{{kube_stateful_set.name}}&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}\n\n This might present an unsafe situation for any further manual operations, such as killing other pods.", "name": "[Kubernetes] Monitor Kubernetes Statefulset Replicas", "options": { "escalation_message": "", diff --git a/kuma/assets/logs/kuma_tests.yaml b/kuma/assets/logs/kuma_tests.yaml index 876b406bd9f59..fa40b17145659 100644 --- a/kuma/assets/logs/kuma_tests.yaml +++ b/kuma/assets/logs/kuma_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: kuma tests: diff --git a/microsoft_copilot/assets/logs/microsoft-copilot_tests.yaml b/microsoft_copilot/assets/logs/microsoft-copilot_tests.yaml index 537acd9ad6686..1d1ded03dac5e 100644 --- a/microsoft_copilot/assets/logs/microsoft-copilot_tests.yaml +++ b/microsoft_copilot/assets/logs/microsoft-copilot_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: microsoft-copilot tests: - sample: >- diff --git a/mysql/assets/logs/mysql_tests.yaml b/mysql/assets/logs/mysql_tests.yaml index 144ed6c0edac5..005eccb18972d 100644 --- a/mysql/assets/logs/mysql_tests.yaml +++ b/mysql/assets/logs/mysql_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "mysql" tests: diff --git a/n8n/README.md b/n8n/README.md index 9e9d410984a93..18da75615dd84 100644 --- a/n8n/README.md +++ b/n8n/README.md @@ -4,18 +4,17 @@ This check monitors [n8n][1] through the Datadog Agent. -Collect n8n metrics including: +This integration collects n8n metrics including: - Cache metrics: hit, miss, and update counts. -- Workflow metrics: started, success, failed counters, audit workflow lifecycle counters; in n8n 2.x, an execution-duration histogram. -- Node metrics: per-node started and finished counters emitted by worker processes in queue mode. +- Workflow metrics: Started, success, and failed counters. Audit workflow life cycle counters. In n8n 2.x, an execution-duration histogram. +- Node metrics: per-node counters (started and finished) emitted by worker processes in queue mode. - Queue metrics: queue depth; enqueued, dequeued, completed, failed, and stalled counters; and scaling-mode worker gauges. - HTTP metrics: request duration histograms tagged with status code. - Process and Node.js runtime metrics. - ## Setup -Follow the instructions below to install and configure this check for an Agent running on a host. For containerized environments, see the [Autodiscovery Integration Templates][3] for guidance on applying these instructions. +Follow the instructions below to install and configure this check for an Agent running on a host. For containerized environments, see the [Autodiscovery integration templates][3] for guidance on applying these instructions. ### Installation @@ -26,7 +25,9 @@ No additional installation is needed on your server. #### Enable the n8n metrics endpoint -The `/metrics` endpoint is disabled by default and must be enabled in your n8n configuration. Note that the `/metrics` endpoint is only available for self-hosted instances and is not available on n8n Cloud. +The `/metrics` endpoint is disabled by default and must be enabled in your n8n configuration. + +**Note**: The `/metrics` endpoint is only available for self-hosted instances and is not available on n8n Cloud. Set the following environment variables to enable metrics: @@ -51,7 +52,7 @@ N8N_METRICS_PREFIX=n8n_ For more details, see the n8n documentation on [enabling Prometheus metrics][10]. -If you change `N8N_METRICS_PREFIX` from its default of `n8n_`, you **must** also set `raw_metric_prefix` in the integration's `conf.yaml` to the same value. Otherwise the check will not recognize the exposed metric names and will silently submit nothing: +If you change `N8N_METRICS_PREFIX` from its default of `n8n_`, you **must** also set `raw_metric_prefix` in the integration's `conf.yaml` to the same value. Otherwise the check does not recognize the exposed metric names and silently submits nothing: ```yaml instances: @@ -63,12 +64,12 @@ instances: Most n8n counters are registered dynamically the first time their underlying event fires. The integration ships mappings for around 70 of these event-bus counters, including: -- Workflow lifecycle: `n8n.workflow.started.count`, `n8n.workflow.success.count`, `n8n.workflow.failed.count`, `n8n.workflow.cancelled.count` +- Workflow life cycle: `n8n.workflow.started.count`, `n8n.workflow.success.count`, `n8n.workflow.failed.count`, `n8n.workflow.cancelled.count` - Audit (workflow, user, credentials, package, variable, execution data): `n8n.audit.workflow.executed.count`, `n8n.audit.user.login.success.count`, `n8n.audit.user.credentials.created.count`, and similar - AI nodes: `n8n.ai.tool.called.count`, `n8n.ai.llm.generated.count`, `n8n.ai.vector.store.searched.count`, and similar -- Runner, queue, and node lifecycle: `n8n.runner.task.requested.count`, `n8n.queue.job.completed.count`, `n8n.node.started.count`, `n8n.node.finished.count` +- Runner, queue, and node life cycle: `n8n.runner.task.requested.count`, `n8n.queue.job.completed.count`, `n8n.node.started.count`, `n8n.node.finished.count` -These counters do not appear on the `/metrics` endpoint until the corresponding event has occurred. A healthy idle deployment will not produce data points for them until that activity fires. The complete list is in [`metadata.csv`][7]. +These counters do not appear on the `/metrics` endpoint until the corresponding event has occurred. A healthy idle deployment does not produce datapoints for them until that activity fires. The complete list is in [`metadata.csv`][7]. If a future n8n release exposes a new event-driven counter that is not yet covered by this integration, add it to the `extra_metrics` option in your instance configuration: @@ -85,7 +86,9 @@ The left-hand side is the Prometheus counter name as n8n exposes it (keep the `_ In queue mode, n8n runs separate worker processes that execute jobs picked up from a Redis-backed queue. Each worker exposes its own `/metrics` endpoint and emits a different subset of metrics than the main process. Worker-observed metrics include `n8n.queue.job.dequeued.count`, `n8n.queue.job.stalled.count`, `n8n.node.started.count`, `n8n.node.finished.count`, and `n8n.runner.task.requested.count`. Main-only metrics include `n8n.instance.role.leader` and the `n8n.scaling.mode.queue.jobs.*` family. -To expose worker metrics, set `QUEUE_HEALTH_CHECK_ACTIVE=true` and `QUEUE_HEALTH_CHECK_PORT=` on each worker. **In n8n 2.x, port `5679` is reserved for the task runner broker, so pick a different port (for example `5680`).** +To expose worker metrics, set `QUEUE_HEALTH_CHECK_ACTIVE=true` and `QUEUE_HEALTH_CHECK_PORT=` on each worker. + +**Note**: In n8n 2.x, port `5679` is reserved for the task runner broker. Pick a different port (for example `5680`). For full coverage in queue deployments, configure one Datadog instance per n8n process exposing `/metrics`, including main and worker processes: @@ -107,11 +110,11 @@ Several metric families were introduced in n8n 2.x and are not emitted on n8n 1. - The `n8n.{production,manual,production.root}.executions`, `n8n.users.total`, `n8n.enabled.users`, `n8n.workflows.total`, and `n8n.credentials.total` family. Only emitted when `N8N_METRICS_INCLUDE_WORKFLOW_STATISTICS=true` is set. - The `n8n.expression.*` family (`evaluation.duration.seconds`, `code.cache.{hit,miss,eviction,size}`, `pool.{acquired,replenish.failed,scaled.up,scaled.to.zero}`). Only emitted when n8n is running the new VM-isolated expression engine *and* observability for it is on. Set `N8N_EXPRESSION_ENGINE=vm` and `N8N_EXPRESSION_ENGINE_OBSERVABILITY_ENABLED=true` on the n8n process; both default to off (the engine defaults to `legacy`). These metrics surface the per-expression evaluation latency, the compiled-expression LRU cache hit and miss rates, and the V8-isolate pool's idle scaling behavior. They are most useful for troubleshooting workflow latency that traces back to slow `{{ ... }}` evaluation. -Some metrics only emit samples after the corresponding runtime event occurs. For example, failures-only counters (`*.failures.count`) need an authentication failure, audit workflow counters need the matching workflow state transition, and the libuv `n8n.nodejs.active.requests` gauge needs an in-flight libuv request. A healthy idle deployment may not produce data points for these metrics until that activity occurs. +Some metrics only emit samples after the corresponding runtime event occurs. For example, failures-only counters (`*.failures.count`) need an authentication failure, audit workflow counters need the matching workflow state transition, and the libuv `n8n.nodejs.active.requests` gauge needs an in-flight libuv request. A healthy idle deployment may not produce datapoints for these metrics until that activity occurs. #### Tag cardinality -When `N8N_METRICS_INCLUDE_WORKFLOW_ID_LABEL=true`, http and workflow execution histograms are tagged with `workflow_id` (and similar labels for nodes). On deployments with many distinct workflows or nodes, this can produce high-cardinality metrics. Drop the label via `exclude_labels` or omit `N8N_METRICS_INCLUDE_WORKFLOW_ID_LABEL` to keep tag cardinality bounded. +When `N8N_METRICS_INCLUDE_WORKFLOW_ID_LABEL=true`, http and workflow execution histograms are tagged with `workflow_id` (and similar labels for nodes). On deployments with many distinct workflows or nodes, this can produce high-cardinality metrics. Drop the label through `exclude_labels` or omit `N8N_METRICS_INCLUDE_WORKFLOW_ID_LABEL` to keep tag cardinality bounded. #### Configure the Datadog Agent @@ -121,7 +124,7 @@ When `N8N_METRICS_INCLUDE_WORKFLOW_ID_LABEL=true`, http and workflow execution h ### Log collection -_Available for Agent versions >6.0_ +**Note**: Available for Agent versions 6.0 and later. #### Enable n8n logging diff --git a/nginx/assets/logs/nginx.yaml b/nginx/assets/logs/nginx.yaml index 21508a86b2e0a..e36c1fa983bbf 100755 --- a/nginx/assets/logs/nginx.yaml +++ b/nginx/assets/logs/nginx.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: nginx metric_id: nginx backend_only: false diff --git a/nginx/assets/logs/nginx_tests.yaml b/nginx/assets/logs/nginx_tests.yaml index b3392b1576202..cc9b568b5bc02 100755 --- a/nginx/assets/logs/nginx_tests.yaml +++ b/nginx/assets/logs/nginx_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "nginx" tests: - sample: '127.0.0.1 - frank [13/Jul/2016:10:55:36 +0000] "GET /apache_pb.gif HTTP/1.0" 200 2326' diff --git a/nginx/assets/monitors/4xx.json b/nginx/assets/monitors/4xx.json index 17fbd04888321..e38ee3436c7e3 100644 --- a/nginx/assets/monitors/4xx.json +++ b/nginx/assets/monitors/4xx.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-09-16", - "last_updated_at": "2026-03-09", + "last_updated_at": "2026-04-09", "title": "Upstream 4xx errors are high", "tags": [ "integration:nginx" ], "description": "NGINX sends requests to upstream peers that can fail eventually. This monitor tracks the count of 4xx HTTP responses to identify issues in the communication between NGINX and the backend servers.", "definition": { - "message": "{{#is_alert}}\n## 🚨 What's happening\n\nAn anomaly has been detected in the number of 4xx HTTP responses from NGINX upstream **{{upstream.name}}** (anomaly score: `{{value}}`, threshold: `{{threshold}}`). The 4xx response rate is significantly higher than normal, indicating that a notable portion of incoming requests are being rejected with client-side error codes.\n\nFirst triggered at **{{first_triggered_at}}**, active for **{{triggered_duration_sec}}** seconds.\n{{/is_alert}}{{#is_recovery}}\n## βœ… Recovered\n\nThe 4xx anomaly for upstream **{{upstream.name}}** has resolved. Current value: `{{value}}`.\n{{/is_recovery}}\n{{^is_recovery}}\n***\n\n## πŸ“ˆ Impact\n\nElevated 4xx error rates can result in failed requests for end users and may expose misconfigurations or broken routes. Services and clients relying on this NGINX upstream may experience partial or complete degradation of functionality.\n\n***\n\n## Runbook\n\n### Initial Troubleshooting Steps\n\n1. **Identify the affected upstream** from the alert (`{{upstream.name}}`).\n2. Open [**Metrics Explorer**](/metric/explorer) and inspect `nginx.upstream.peers.responses.4xx` broken down by `upstream`.\n3. Review NGINX access logs for specific endpoints and status codes:\n ```bash\n tail -f /var/log/nginx/access.log | grep \" 4[0-9][0-9] \"\n ```\n4. Correlate the spike with recent configuration changes, upstream deployments, or traffic shifts.\n\n### Cause and Resolution\n\n| Cause | Resolution |\n| ----- | ---------- |\n| Invalid or removed request paths (404) | Verify routes in NGINX configuration; update upstream routing rules to reflect the current backend state. |\n| Authentication or authorization failures (401/403) | Review auth configuration; check if credentials or access tokens have expired or been revoked. |\n| Malformed client requests (400) | Inspect incoming request headers and payloads; check client-side request construction. |\n| Rate limiting triggered (429) | Review rate limit thresholds; consider scaling upstream services or relaxing limits. |\n| Upstream endpoints renamed or removed | Update NGINX upstream configuration to reflect the current backend service endpoints. |\n\n### Related links\n\n* [Documentation](https://docs.datadoghq.com/integrations/nginx/)\n* [Metrics Explorer](/metric/explorer)\n* [Log Explorer](/logs?query=source%3Anginx)\n\n### Who should be notified?\n\nAssign the appropriate notification handle for this alert (e.g., `@slack-infra`, `@pagerduty-nginx`):\n`@your-team-handle`\n{{/is_recovery}}", + "message": "{{#is_alert}}\n## 🚨 What's happening\n\nAn anomaly has been detected in the number of 4xx HTTP responses from NGINX upstream **{{upstream.name}}** (anomaly score: `{{value}}`, threshold: `{{threshold}}`). The 4xx response rate is significantly higher than normal, indicating that a notable portion of incoming requests are being rejected with client-side error codes.\n\nFirst triggered at **{{first_triggered_at}}**, active for **{{triggered_duration_sec}}** seconds.\n{{/is_alert}}{{#is_recovery}}\n## βœ… Recovered\n\nThe 4xx anomaly for upstream **{{upstream.name}}** has resolved. Current value: `{{value}}`.\n{{/is_recovery}}\n{{^is_recovery}}\n***\n\n## πŸ“ˆ Impact\n\nElevated 4xx error rates can result in failed requests for end users and may expose misconfigurations or broken routes. Services and clients relying on this NGINX upstream may experience partial or complete degradation of functionality.\n\n***\n\n## Runbook\n\n### Initial Troubleshooting Steps\n\n1. **Identify the affected upstream** from the alert (`{{upstream.name}}`).\n2. Open [**Metrics Explorer**](/metric/explorer) and inspect `nginx.upstream.peers.responses.4xx` broken down by `upstream`.\n3. Review NGINX access logs for specific endpoints and status codes:\n ```bash\n tail -f /var/log/nginx/access.log | grep \" 4[0-9][0-9] \"\n ```\n4. Correlate the spike with recent configuration changes, upstream deployments, or traffic shifts.\n\n### Cause and Resolution\n\n| Cause | Resolution |\n| ----- | ---------- |\n| Invalid or removed request paths (404) | Verify routes in NGINX configuration; update upstream routing rules to reflect the current backend state. |\n| Authentication or authorization failures (401/403) | Review auth configuration; check if credentials or access tokens have expired or been revoked. |\n| Malformed client requests (400) | Inspect incoming request headers and payloads; check client-side request construction. |\n| Rate limiting triggered (429) | Review rate limit thresholds; consider scaling upstream services or relaxing limits. |\n| Upstream endpoints renamed or removed | Update NGINX upstream configuration to reflect the current backend service endpoints. |\n\n### Related links\n\n* [Documentation](https://docs.datadoghq.com/integrations/nginx/)\n* [Logs](/logs?query=upstream:{{upstream.name}})\n* [Metrics Explorer (nginx.upstream.peers.responses.4xx)](/metric/explorer?exp_metric=nginx.upstream.peers.responses.4xx&exp_scope=upstream:{{upstream.name}}&exp_agg=avg&exp_type=line)\n\n### Who should be notified?\n\nAssign the appropriate notification handle for this alert (e.g., `@slack-infra`, `@pagerduty-nginx`):\n`@your-team-handle`\n{{/is_recovery}}", "name": "[NGINX] 4xx Errors higher than usual", "options": { "escalation_message": "", diff --git a/nginx/assets/monitors/5xx.json b/nginx/assets/monitors/5xx.json index c7b9ef7201dbc..b98d0bf985336 100644 --- a/nginx/assets/monitors/5xx.json +++ b/nginx/assets/monitors/5xx.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-09-16", - "last_updated_at": "2026-03-09", + "last_updated_at": "2026-04-09", "title": "Upstream 5xx errors are high", "tags": [ "integration:nginx" ], "description": "β€œ5xx upstream request errors” are indicating server issues from backend servers. This monitor tracks the count of 5xx responses from NGINX's upstream peers to identify server-related issues in your web or application infrastructure.", "definition": { - "message": "{{#is_alert}}\n## 🚨 What's happening\n\nAn anomaly has been detected in the number of 5xx HTTP responses from NGINX upstream **{{upstream.name}}** (anomaly score: `{{value}}`, threshold: `{{threshold}}`). The 5xx error rate is significantly higher than normal, indicating that backend servers are failing to handle a notable portion of requests.\n\nFirst triggered at **{{first_triggered_at}}**, active for **{{triggered_duration_sec}}** seconds.\n{{/is_alert}}{{#is_recovery}}\n## βœ… Recovered\n\nThe 5xx anomaly for upstream **{{upstream.name}}** has resolved. Current value: `{{value}}`.\n{{/is_recovery}}\n{{^is_recovery}}\n***\n\n## πŸ“ˆ Impact\n\n5xx errors indicate server-side failures that cause direct service disruptions for users. Dependent services that rely on successful responses from this NGINX upstream may experience cascading failures or degraded functionality.\n\n***\n\n## Runbook\n\n### Initial Troubleshooting Steps\n\n1. **Identify the affected upstream** from the alert (`{{upstream.name}}`).\n2. Open [**Metrics Explorer**](/metric/explorer) and inspect `nginx.upstream.peers.responses.5xx` broken down by `upstream`.\n3. Review NGINX error logs for connection failures or backend errors:\n ```bash\n tail -f /var/log/nginx/error.log\n ```\n4. Check upstream backend service health and application logs.\n5. Correlate the spike with recent deployments or infrastructure changes.\n\n### Cause and Resolution\n\n| Cause | Resolution |\n| ----- | ---------- |\n| Backend server is down or crashed (502) | Verify the upstream service is running; restart the service if needed and check its logs. |\n| Gateway timeout due to slow upstream (504) | Check upstream response times; increase `proxy_read_timeout` if the upstream is legitimately slow. |\n| Application-level errors (500) | Inspect upstream application logs for unhandled exceptions or crashes; roll back recent deployments if correlated. |\n| Service unavailable due to overload (503) | Check upstream server resource utilization; scale out or enable load balancing across more peers. |\n| Resource exhaustion on upstream servers | Review CPU, memory, and connection pool usage on the backend; tune resource limits and autoscaling. |\n\n### Related links\n\n* [Documentation](https://docs.datadoghq.com/integrations/nginx/)\n* [Metrics Explorer](/metric/explorer)\n* [Log Explorer](/logs?query=source%3Anginx)\n\n### Who should be notified?\n\nAssign the appropriate notification handle for this alert (e.g., `@slack-infra`, `@pagerduty-nginx`):\n`@your-team-handle`\n{{/is_recovery}}", + "message": "{{#is_alert}}\n## 🚨 What's happening\n\nAn anomaly has been detected in the number of 5xx HTTP responses from NGINX upstream **{{upstream.name}}** (anomaly score: `{{value}}`, threshold: `{{threshold}}`). The 5xx error rate is significantly higher than normal, indicating that backend servers are failing to handle a notable portion of requests.\n\nFirst triggered at **{{first_triggered_at}}**, active for **{{triggered_duration_sec}}** seconds.\n{{/is_alert}}{{#is_recovery}}\n## βœ… Recovered\n\nThe 5xx anomaly for upstream **{{upstream.name}}** has resolved. Current value: `{{value}}`.\n{{/is_recovery}}\n{{^is_recovery}}\n***\n\n## πŸ“ˆ Impact\n\n5xx errors indicate server-side failures that cause direct service disruptions for users. Dependent services that rely on successful responses from this NGINX upstream may experience cascading failures or degraded functionality.\n\n***\n\n## Runbook\n\n### Initial Troubleshooting Steps\n\n1. **Identify the affected upstream** from the alert (`{{upstream.name}}`).\n2. Open [**Metrics Explorer**](/metric/explorer) and inspect `nginx.upstream.peers.responses.5xx` broken down by `upstream`.\n3. Review NGINX error logs for connection failures or backend errors:\n ```bash\n tail -f /var/log/nginx/error.log\n ```\n4. Check upstream backend service health and application logs.\n5. Correlate the spike with recent deployments or infrastructure changes.\n\n### Cause and Resolution\n\n| Cause | Resolution |\n| ----- | ---------- |\n| Backend server is down or crashed (502) | Verify the upstream service is running; restart the service if needed and check its logs. |\n| Gateway timeout due to slow upstream (504) | Check upstream response times; increase `proxy_read_timeout` if the upstream is legitimately slow. |\n| Application-level errors (500) | Inspect upstream application logs for unhandled exceptions or crashes; roll back recent deployments if correlated. |\n| Service unavailable due to overload (503) | Check upstream server resource utilization; scale out or enable load balancing across more peers. |\n| Resource exhaustion on upstream servers | Review CPU, memory, and connection pool usage on the backend; tune resource limits and autoscaling. |\n\n### Related links\n\n* [Documentation](https://docs.datadoghq.com/integrations/nginx/)\n* [Logs](/logs?query=upstream:{{upstream.name}})\n* [Metrics Explorer (nginx.upstream.peers.responses.5xx)](/metric/explorer?exp_metric=nginx.upstream.peers.responses.5xx&exp_scope=upstream:{{upstream.name}}&exp_agg=avg&exp_type=line)\n\n### Who should be notified?\n\nAssign the appropriate notification handle for this alert (e.g., `@slack-infra`, `@pagerduty-nginx`):\n`@your-team-handle`\n{{/is_recovery}}", "name": "[NGINX] 5xx Errors higher than usual", "options": { "escalation_message": "", diff --git a/nginx/assets/monitors/upstream_peer_fails.json b/nginx/assets/monitors/upstream_peer_fails.json index 08cef0b5647dd..d8f4b0fb003d1 100644 --- a/nginx/assets/monitors/upstream_peer_fails.json +++ b/nginx/assets/monitors/upstream_peer_fails.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2020-09-16", - "last_updated_at": "2026-03-09", + "last_updated_at": "2026-04-09", "title": "Upstream peers are failing", "tags": [ "integration:nginx" ], "description": "NGINX can be configured to distribute incoming client requests to multiple upstream peers (individual web servers, application servers, or other backend services). This monitor tracks anomalies in the number of failed upstream peers to identify issues.", "definition": { - "message": "{{#is_alert}}\n## 🚨 What's happening\n\nAn anomaly has been detected in the number of upstream peer communication failures for **{{upstream.name}}** (anomaly score: `{{value}}`, threshold: `{{threshold}}`). NGINX is experiencing an unusual number of unsuccessful attempts to connect to or communicate with one or more backend servers.\n\nFirst triggered at **{{first_triggered_at}}**, active for **{{triggered_duration_sec}}** seconds.\n{{/is_alert}}{{#is_recovery}}\n## βœ… Recovered\n\nUpstream peer failures for **{{upstream.name}}** have resolved. Current value: `{{value}}`.\n{{/is_recovery}}\n{{^is_recovery}}\n***\n\n## πŸ“ˆ Impact\n\nUpstream peer failures reduce the pool of available backend servers, increasing load on healthy peers. Users may experience intermittent errors or increased response times as NGINX retries or routes traffic around failed peers.\n\n***\n\n## Runbook\n\n### Initial Troubleshooting Steps\n\n1. **Identify the affected upstream** from the alert (`{{upstream.name}}`).\n2. Open [**Metrics Explorer**](/metric/explorer) and inspect `nginx.stream.upstream.peers.fails` broken down by `upstream` to identify which specific peers are failing.\n3. Review NGINX error logs for connection-level failures:\n ```bash\n tail -f /var/log/nginx/error.log | grep \"upstream\"\n ```\n4. Test connectivity from the NGINX host to the failing upstream servers:\n ```bash\n curl -v http://:/health\n ```\n5. Correlate with recent configuration changes or upstream service deployments.\n\n### Cause and Resolution\n\n| Cause | Resolution |\n| ----- | ---------- |\n| Upstream server is down or crashed | Verify the upstream service is running and listening on the expected port; restart if needed. |\n| Network connectivity issues | Test connectivity from the NGINX host to the upstream; check firewall rules and network routing. |\n| Upstream not responding within timeout | Review `proxy_connect_timeout` and `proxy_read_timeout` in NGINX config; increase if the upstream is legitimately slow. |\n| Misconfigured upstream address or port | Verify the upstream block in NGINX configuration has the correct server addresses and ports. |\n| Firewall or security group blocking traffic | Check security group rules and host-based firewall (iptables/nftables) on the upstream servers. |\n\n### Related links\n\n* [Documentation](https://docs.datadoghq.com/integrations/nginx/)\n* [Metrics Explorer](/metric/explorer)\n* [Log Explorer](/logs?query=source%3Anginx)\n\n### Who should be notified?\n\nAssign the appropriate notification handle for this alert (e.g., `@slack-infra`, `@pagerduty-nginx`):\n`@your-team-handle`\n{{/is_recovery}}", + "message": "{{#is_alert}}\n## 🚨 What's happening\n\nAn anomaly has been detected in the number of upstream peer communication failures for **{{upstream.name}}** (anomaly score: `{{value}}`, threshold: `{{threshold}}`). NGINX is experiencing an unusual number of unsuccessful attempts to connect to or communicate with one or more backend servers.\n\nFirst triggered at **{{first_triggered_at}}**, active for **{{triggered_duration_sec}}** seconds.\n{{/is_alert}}{{#is_recovery}}\n## βœ… Recovered\n\nUpstream peer failures for **{{upstream.name}}** have resolved. Current value: `{{value}}`.\n{{/is_recovery}}\n{{^is_recovery}}\n***\n\n## πŸ“ˆ Impact\n\nUpstream peer failures reduce the pool of available backend servers, increasing load on healthy peers. Users may experience intermittent errors or increased response times as NGINX retries or routes traffic around failed peers.\n\n***\n\n## Runbook\n\n### Initial Troubleshooting Steps\n\n1. **Identify the affected upstream** from the alert (`{{upstream.name}}`).\n2. Open [**Metrics Explorer**](/metric/explorer) and inspect `nginx.stream.upstream.peers.fails` broken down by `upstream` to identify which specific peers are failing.\n3. Review NGINX error logs for connection-level failures:\n ```bash\n tail -f /var/log/nginx/error.log | grep \"upstream\"\n ```\n4. Test connectivity from the NGINX host to the failing upstream servers:\n ```bash\n curl -v http://:/health\n ```\n5. Correlate with recent configuration changes or upstream service deployments.\n\n### Cause and Resolution\n\n| Cause | Resolution |\n| ----- | ---------- |\n| Upstream server is down or crashed | Verify the upstream service is running and listening on the expected port; restart if needed. |\n| Network connectivity issues | Test connectivity from the NGINX host to the upstream; check firewall rules and network routing. |\n| Upstream not responding within timeout | Review `proxy_connect_timeout` and `proxy_read_timeout` in NGINX config; increase if the upstream is legitimately slow. |\n| Misconfigured upstream address or port | Verify the upstream block in NGINX configuration has the correct server addresses and ports. |\n| Firewall or security group blocking traffic | Check security group rules and host-based firewall (iptables/nftables) on the upstream servers. |\n\n### Related links\n\n* [Documentation](https://docs.datadoghq.com/integrations/nginx/)\n* [Logs](/logs?query=upstream:{{upstream.name}})\n* [Metrics Explorer (nginx.stream.upstream.peers.fails)](/metric/explorer?exp_metric=nginx.stream.upstream.peers.fails&exp_scope=upstream:{{upstream.name}}&exp_agg=avg&exp_type=line)\n\n### Who should be notified?\n\nAssign the appropriate notification handle for this alert (e.g., `@slack-infra`, `@pagerduty-nginx`):\n`@your-team-handle`\n{{/is_recovery}}", "name": "[NGINX] Upstream peers fails", "options": { "escalation_message": "", diff --git a/pan_firewall/assets/logs/pan.firewall_tests.yaml b/pan_firewall/assets/logs/pan.firewall_tests.yaml index 6640136dc610e..620b0727552f6 100644 --- a/pan_firewall/assets/logs/pan.firewall_tests.yaml +++ b/pan_firewall/assets/logs/pan.firewall_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "pan.firewall" tests: - diff --git a/plaid/assets/logs/plaid_tests.yaml b/plaid/assets/logs/plaid_tests.yaml index d0e37ee789668..dfeaff3a76290 100644 --- a/plaid/assets/logs/plaid_tests.yaml +++ b/plaid/assets/logs/plaid_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "plaid" tests: - diff --git a/plivo/assets/logs/plivo_tests.yaml b/plivo/assets/logs/plivo_tests.yaml index f54636a02f8b7..0fc83b49751ad 100644 --- a/plivo/assets/logs/plivo_tests.yaml +++ b/plivo/assets/logs/plivo_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: plivo tests: - sample: >- diff --git a/postgres/assets/monitors/percent_usage_connections.json b/postgres/assets/monitors/percent_usage_connections.json index bfa477bbe4e8a..01608f00c12bb 100644 --- a/postgres/assets/monitors/percent_usage_connections.json +++ b/postgres/assets/monitors/percent_usage_connections.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2021-03-17", - "last_updated_at": "2023-07-24", + "last_updated_at": "2026-04-09", "title": "Connection pool is reaching saturation point", "tags": [ "integration:postgres" ], "description": "In PostgreSQL, there is a limit of concurrent connections that can be increased. When this limit is exceeded, new users cannot establish a connection with the database. This monitor tracks the total number of connections.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nPostgreSQL connection usage on host {{host.name}} has exceeded 90% of the maximum allowed connections over the last 15 minutes.\n\n{{/is_alert}}", + "message": "{{#is_alert}}\n\n## What's happening?\nPostgreSQL connection usage on host {{host.name}} has exceeded 90% of the maximum allowed connections over the last 15 minutes.\n\n## Related Links\n\n- [Metrics Explorer (postgresql.percent_usage_connections)](/metric/explorer?exp_metric=postgresql.percent_usage_connections&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}", "name": "[Postgres] Number of connections is approaching connection limit on {{host.name}}", "options": { "escalation_message": "", diff --git a/postgres/assets/monitors/replication_delay.json b/postgres/assets/monitors/replication_delay.json index 889700af13e39..6ec45e4efe1c5 100644 --- a/postgres/assets/monitors/replication_delay.json +++ b/postgres/assets/monitors/replication_delay.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2021-02-16", - "last_updated_at": "2021-03-17", + "last_updated_at": "2026-04-09", "title": "Replication delay is high", "tags": [ "integration:postgres" ], "description": "Replication lag is the delay between the time when data is written to the primary database and the time when it is replicated to the standby databases. This monitor tracks the replication lag of the postgres database.", "definition": { - "message": "{{#is_alert}}\n\n## What's happening?\nAnomalies in replication delay on host {{host.name}} for PostgreSQL have been detected above the expected range within the past 15 minutes, over the last hour.\n\n{{/is_alert}}", + "message": "{{#is_alert}}\n\n## What's happening?\nAnomalies in replication delay on host {{host.name}} for PostgreSQL have been detected above the expected range within the past 15 minutes, over the last hour.\n\n## Related Links\n\n- [Metrics Explorer (postgresql.replication_delay)](/metric/explorer?exp_metric=postgresql.replication_delay&exp_agg=avg&exp_type=line)\n\n{{/is_alert}}", "name": "[Postgres] Replication delay is abnormally high on {{host.name}}", "options": { "escalation_message": "", diff --git a/redisdb/assets/monitors/high_mem.json b/redisdb/assets/monitors/high_mem.json index b230aa497a55d..a360246d126a6 100644 --- a/redisdb/assets/monitors/high_mem.json +++ b/redisdb/assets/monitors/high_mem.json @@ -1,14 +1,14 @@ { "version": 2, "created_at": "2021-02-08", - "last_updated_at": "2021-02-08", + "last_updated_at": "2026-04-09", "title": "Memory consumption is high", "tags": [ "integration:redis" ], "description": "Redis servers use RAM to store data and memory is a critical resource for its performance. This monitor tracks the percentage of used memory to avoid the risk of running out of memory, which can lead to performance issues.", "definition": { - "message": "## What's happening?\n{{#is_alert}}\nRedis memory usage has exceeded 90% of its allocated limit in the last 5 minutes with current value of {{value}}.\n{{/is_alert}} \n\n{{#is_warning}}\nRedis memory usage has exceeded 70% of its allocated limit in the last 5 minutes with current value of {{value}}.\n{{/is_warning}}", + "message": "## What's happening?\n{{#is_alert}}\nRedis memory usage has exceeded 90% of its allocated limit in the last 5 minutes with current value of {{value}}.\n{{/is_alert}} \n\n{{#is_warning}}\nRedis memory usage has exceeded 70% of its allocated limit in the last 5 minutes with current value of {{value}}.\n{{/is_warning}}\n\n## Related Links\n\n- [Metrics Explorer (redis.mem.used)](/metric/explorer?exp_metric=redis.mem.used&exp_agg=avg&exp_type=line)\n- [Metrics Explorer (redis.mem.maxmemory)](/metric/explorer?exp_metric=redis.mem.maxmemory&exp_agg=avg&exp_type=line)", "name": "[Redis] High memory consumption", "options": { "escalation_message": "", diff --git a/riak/assets/logs/riak_tests.yaml b/riak/assets/logs/riak_tests.yaml index ff1a1f0f82cb3..c4fdc632805f1 100644 --- a/riak/assets/logs/riak_tests.yaml +++ b/riak/assets/logs/riak_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "riak" tests: - diff --git a/snmp/changelog.d/23791.fixed b/snmp/changelog.d/23791.changed similarity index 100% rename from snmp/changelog.d/23791.fixed rename to snmp/changelog.d/23791.changed diff --git a/sonicwall_firewall/assets/logs/sonicwall-firewall_tests.yaml b/sonicwall_firewall/assets/logs/sonicwall-firewall_tests.yaml index e71e99993bfab..27dfb21a379a7 100644 --- a/sonicwall_firewall/assets/logs/sonicwall-firewall_tests.yaml +++ b/sonicwall_firewall/assets/logs/sonicwall-firewall_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "sonicwall-firewall" tests: - diff --git a/sqlserver/changelog.d/23862.fixed b/sqlserver/changelog.d/23862.fixed new file mode 100644 index 0000000000000..816b836c62be4 --- /dev/null +++ b/sqlserver/changelog.d/23862.fixed @@ -0,0 +1 @@ +Restore agent hostname instrumentation for SQL Server named instance host configurations. \ No newline at end of file diff --git a/sqlserver/datadog_checks/sqlserver/sqlserver.py b/sqlserver/datadog_checks/sqlserver/sqlserver.py index 3953b8f1bd3fb..8d15b54580c5d 100644 --- a/sqlserver/datadog_checks/sqlserver/sqlserver.py +++ b/sqlserver/datadog_checks/sqlserver/sqlserver.py @@ -328,6 +328,9 @@ def port(self): return self.host_and_port[1] def resolve_db_host(self): + if "\\" in self.host: + # SQL Server instance names are not resolvable, this preserves original fallback behavior prior to v7.79.0 + return datadog_agent.get_hostname() return agent_host_resolver(self.host) @property diff --git a/sqlserver/tests/test_unit.py b/sqlserver/tests/test_unit.py index 21e43ecd686aa..99b2d7790826a 100644 --- a/sqlserver/tests/test_unit.py +++ b/sqlserver/tests/test_unit.py @@ -11,6 +11,7 @@ import mock import pytest +from datadog_checks.base.stubs.datadog_agent import datadog_agent from datadog_checks.dev import EnvVars from datadog_checks.sqlserver import SQLServer from datadog_checks.sqlserver.connection import split_sqlserver_host_port @@ -908,6 +909,71 @@ def test_split_sqlserver_host(instance_host, split_host, split_port): assert (s_host, s_port) == (split_host, split_port) +AGENT_HOSTNAME = 'sql-agent-host.example.com' + + +@pytest.fixture +def agent_hostname_for_resolve_db_host(): + datadog_agent.set_hostname(AGENT_HOSTNAME) + yield + datadog_agent.reset_hostname() + + +@pytest.mark.parametrize( + 'instance_host,host_part', + [ + (r'SQL-HOST01\INSTANCE01,1601', r'SQL-HOST01\INSTANCE01'), + (r'MY-SERVER\SQLEXPRESS,1433', r'MY-SERVER\SQLEXPRESS'), + (r'MY-SERVER\SQLEXPRESS', r'MY-SERVER\SQLEXPRESS'), + ], +) +def test_resolve_db_host_named_instance_returns_agent_hostname( + agent_hostname_for_resolve_db_host, instance_host, host_part +): + instance = { + 'host': instance_host, + 'username': 'datadog', + 'password': 'secret', + } + check = SQLServer(CHECK_NAME, {}, [instance]) + assert check.host == host_part + + # Agent 7.79+ base resolver returns the literal host string for unresolvable names. + with mock.patch( + 'datadog_checks.sqlserver.sqlserver.agent_host_resolver', + return_value=host_part, + ): + assert check.resolve_db_host() == AGENT_HOSTNAME + assert check.resolved_hostname == AGENT_HOSTNAME + assert check.database_hostname == AGENT_HOSTNAME + + +@pytest.mark.parametrize( + 'instance_host,host_part,base_resolver_return', + [ + ('db.example.com,1433', 'db.example.com', 'resolved-db.example.com'), + ('192.0.2.10,1433', '192.0.2.10', '192.0.2.10'), + ], +) +def test_resolve_db_host_plain_host_delegates_to_base_resolver( + agent_hostname_for_resolve_db_host, instance_host, host_part, base_resolver_return +): + instance = { + 'host': instance_host, + 'username': 'datadog', + 'password': 'secret', + } + check = SQLServer(CHECK_NAME, {}, [instance]) + assert check.host == host_part + + with mock.patch( + 'datadog_checks.sqlserver.sqlserver.agent_host_resolver', + return_value=base_resolver_return, + ) as mock_resolver: + assert check.resolve_db_host() == base_resolver_return + mock_resolver.assert_called_once_with(host_part) + + @pytest.mark.parametrize( "query,expected_comments,is_proc,expected_name", [ diff --git a/tenable/assets/logs/tenable_tests.yaml b/tenable/assets/logs/tenable_tests.yaml index e6716565e3055..6fd087762b73f 100644 --- a/tenable/assets/logs/tenable_tests.yaml +++ b/tenable/assets/logs/tenable_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "tenable" tests: - diff --git a/tomcat/assets/logs/tomcat_tests.yaml b/tomcat/assets/logs/tomcat_tests.yaml index 0d5f325f90ae5..d2314773a7726 100644 --- a/tomcat/assets/logs/tomcat_tests.yaml +++ b/tomcat/assets/logs/tomcat_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: tomcat tests: - sample: 2005-09-07 14:07:41,508 [main] INFO MyApp - Entering application. diff --git a/traefik_mesh/assets/dashboards/traefik_mesh_overview.json b/traefik_mesh/assets/dashboards/traefik_mesh_overview.json index f621603991fc8..55708c1c9afe1 100644 --- a/traefik_mesh/assets/dashboards/traefik_mesh_overview.json +++ b/traefik_mesh/assets/dashboards/traefik_mesh_overview.json @@ -584,17 +584,17 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:traefik_mesh.router.requests.count{$host,$service,$router,code:2*} by {code,router,service}.as_count()" + "query": "sum:traefik_mesh.router.requests.count{$host,$traefik_service,$router,code:2*} by {code,router,traefik_service}.as_count()" }, { "data_source": "metrics", "name": "query2", - "query": "sum:traefik_mesh.router.requests.count{$host AND $service AND $router AND code:4* OR code:5*} by {code,router,service}.as_count()" + "query": "sum:traefik_mesh.router.requests.count{$host AND $traefik_service AND $router AND code:4* OR code:5*} by {code,router,traefik_service}.as_count()" }, { "data_source": "metrics", "name": "query3", - "query": "sum:traefik_mesh.router.requests.count{$host,$service,$routercode:3*} by {code,router,service}.as_count()" + "query": "sum:traefik_mesh.router.requests.count{$host,$traefik_service,$router,code:3*} by {code,router,traefik_service}.as_count()" } ], "response_format": "timeseries", @@ -633,7 +633,7 @@ { "name": "query1", "data_source": "metrics", - "query": "sum:traefik_mesh.router.requests.count{$router, $endpoint} by {protocol,service}", + "query": "sum:traefik_mesh.router.requests.count{$router, $endpoint} by {protocol,traefik_service}", "aggregator": "sum" } ], @@ -694,7 +694,7 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:traefik_mesh.router.responses.bytes.count{$service, $router, $host} by {service,router,host,protocol}" + "query": "sum:traefik_mesh.router.responses.bytes.count{$traefik_service, $router, $host} by {traefik_service,router,host,protocol}" } ], "response_format": "timeseries", @@ -900,17 +900,17 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:traefik_mesh.service.requests.count{$host,$service,$endpoint,code:2*} by {protocol,code,method}.as_count()" + "query": "sum:traefik_mesh.service.requests.count{$host,$traefik_service,$endpoint,code:2*} by {protocol,code,method}.as_count()" }, { "data_source": "metrics", "name": "query2", - "query": "sum:traefik_mesh.service.requests.count{$host AND $service AND $endpoint AND code:4* OR code:5*} by {protocol,code,method}.as_count()" + "query": "sum:traefik_mesh.service.requests.count{$host AND $traefik_service AND $endpoint AND code:4* OR code:5*} by {protocol,code,method}.as_count()" }, { "data_source": "metrics", "name": "query3", - "query": "sum:traefik_mesh.service.requests.count{$host,$service ,$endpoint,code:3*} by {protocol,code,method}.as_count()" + "query": "sum:traefik_mesh.service.requests.count{$host,$traefik_service,$endpoint,code:3*} by {protocol,code,method}.as_count()" } ], "response_format": "timeseries", @@ -948,7 +948,7 @@ { "name": "query1", "data_source": "metrics", - "query": "avg:traefik_mesh.service.requests.count{$host} by {protocol,service,router}", + "query": "avg:traefik_mesh.service.requests.count{$host} by {protocol,traefik_service,router}", "aggregator": "sum" } ], @@ -1009,7 +1009,7 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:traefik_mesh.service.responses.bytes.count{$host, $service, $endpoint} by {host,service,endpoint}" + "query": "sum:traefik_mesh.service.responses.bytes.count{$host, $traefik_service, $endpoint} by {host,traefik_service,endpoint}" } ], "response_format": "timeseries", @@ -1058,7 +1058,7 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:traefik_mesh.service.server.up{$host, $service, $endpoint} by {service}" + "query": "sum:traefik_mesh.service.server.up{$host, $traefik_service, $endpoint} by {traefik_service}" } ], "response_format": "timeseries", @@ -1098,7 +1098,7 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:traefik_mesh.service.request.duration.seconds.sum{$host, $service, $endpoint} by {service,endpoint}" + "query": "sum:traefik_mesh.service.request.duration.seconds.sum{$host, $traefik_service, $endpoint} by {traefik_service,endpoint}" } ], "response_format": "timeseries", @@ -1452,8 +1452,8 @@ "default": "*" }, { - "name": "service", - "prefix": "service", + "name": "traefik_service", + "prefix": "traefik_service", "available_values": [], "default": "*" }, @@ -1485,4 +1485,4 @@ "layout_type": "ordered", "notify_list": [], "reflow_type": "fixed" -} \ No newline at end of file +} diff --git a/traefik_mesh/assets/logs/traefik_tests.yaml b/traefik_mesh/assets/logs/traefik_tests.yaml index 9e4e30e944404..2e90e7f32ffac 100644 --- a/traefik_mesh/assets/logs/traefik_tests.yaml +++ b/traefik_mesh/assets/logs/traefik_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "traefik" tests: - diff --git a/vault/assets/logs/vault_tests.yaml b/vault/assets/logs/vault_tests.yaml index 70f47c423a285..6295905cba662 100644 --- a/vault/assets/logs/vault_tests.yaml +++ b/vault/assets/logs/vault_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: "vault" tests: - diff --git a/vonage/assets/logs/vonage_tests.yaml b/vonage/assets/logs/vonage_tests.yaml index b65d23eebcb34..1040b50afb5a2 100644 --- a/vonage/assets/logs/vonage_tests.yaml +++ b/vonage/assets/logs/vonage_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks Expected sample output: id: "vonage" tests: diff --git a/zk/assets/logs/zookeeper_tests.yaml b/zk/assets/logs/zookeeper_tests.yaml index 0acc153a3ce49..443a6e6b6cced 100644 --- a/zk/assets/logs/zookeeper_tests.yaml +++ b/zk/assets/logs/zookeeper_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks # bypass-global-timestamp-format-in-sample-checks id: "zookeeper" tests: diff --git a/zscaler_private_access/assets/logs/zscaler-private-access_tests.yaml b/zscaler_private_access/assets/logs/zscaler-private-access_tests.yaml index fe46f6bcf8419..18f0de70a98a9 100644 --- a/zscaler_private_access/assets/logs/zscaler-private-access_tests.yaml +++ b/zscaler_private_access/assets/logs/zscaler-private-access_tests.yaml @@ -1,3 +1,4 @@ +# bypass-global-date-remapper-parse-failure-checks id: zscaler-private-access tests: -