From 78992672ceade93d6fa6f484294fa4cb9a13a892 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 22 May 2026 16:09:35 -0400 Subject: [PATCH 01/19] phase2 mvp: add HTTPXWrapper behind use_httpx flag Adds an httpx-backed HTTP client wrapper that implements the HTTPClientProtocol so a check can opt in via 'use_httpx: true' in instance config. RequestsWrapper remains the default. Opts kong and kuma into the new wrapper as a proof of concept. Extends mock_http_response so the existing test fixtures intercept both requests and httpx code paths. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../datadog_checks/base/checks/base.py | 14 +- .../datadog_checks/base/utils/http_httpx.py | 447 ++++++++++++++++++ datadog_checks_base/pyproject.toml | 1 + .../tests/base/utils/http_httpx/__init__.py | 3 + .../tests/base/utils/http_httpx/conftest.py | 104 ++++ .../base/utils/http_httpx/test_auth_basic.py | 40 ++ .../base/utils/http_httpx/test_config.py | 88 ++++ .../base/utils/http_httpx/test_exceptions.py | 78 +++ .../base/utils/http_httpx/test_lifecycle.py | 37 ++ .../base/utils/http_httpx/test_methods.py | 66 +++ .../base/utils/http_httpx/test_response.py | 135 ++++++ .../tests/base/utils/http_httpx/test_tls.py | 38 ++ .../datadog_checks/dev/plugin/pytest.py | 22 +- kong/tests/conftest.py | 3 +- kong/tests/test_unit.py | 4 + kuma/tests/conftest.py | 4 +- 16 files changed, 1076 insertions(+), 8 deletions(-) create mode 100644 datadog_checks_base/datadog_checks/base/utils/http_httpx.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/__init__.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/conftest.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/test_config.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/test_methods.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/test_response.py create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/test_tls.py diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index 13d107b5cea73..afe10394393f8 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -412,10 +412,18 @@ def http(self) -> HTTPClientProtocol: Only new checks or checks on Agent 6.13+ can and should use this for HTTP requests. """ if not hasattr(self, '_http'): - # See Performance Optimizations in this package's README.md. - from datadog_checks.base.utils.http import RequestsWrapper + instance = self.instance or {} + if is_affirmative(instance.get('use_httpx', False)): + # Per Phase 2 MVP D4: an ImportError surfaces at construction + # time if httpx is not installed. + from datadog_checks.base.utils.http_httpx import HTTPXWrapper + + self._http = HTTPXWrapper(instance, self.init_config, self.HTTP_CONFIG_REMAPPER, self.log) + else: + # See Performance Optimizations in this package's README.md. + from datadog_checks.base.utils.http import RequestsWrapper - self._http = RequestsWrapper(self.instance or {}, self.init_config, self.HTTP_CONFIG_REMAPPER, self.log) + self._http = RequestsWrapper(instance, self.init_config, self.HTTP_CONFIG_REMAPPER, self.log) return self._http diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py new file mode 100644 index 0000000000000..aa7143122174f --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py @@ -0,0 +1,447 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +""" +Minimum-viable httpx-backed HTTP client wrapper. + +Implements ``HTTPClientProtocol`` and ``HTTPResponseProtocol`` so a check that +opts in via ``use_httpx=true`` in instance config can transparently swap from +``RequestsWrapper``. Feature surface is intentionally narrow per the Phase 2 +MVP plan: basic auth, TLS verify/cert, headers, timeouts, common request +options, and the exception mapping into ``datadog_checks.base.utils.http_exceptions``. + +Auth tokens, proxies, Unix-socket transports, Kerberos / NTLM / AWS / Digest +auth, connection-pool tuning, HTTP/2 and multipart uploads are deferred to +Phase 3. +""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping +from typing import Any, Iterator + +import httpx + +from datadog_checks.base.config import is_affirmative + +from .headers import get_default_headers, update_headers +from .http_exceptions import ( + HTTPConnectionError, + HTTPError, + HTTPRequestError, + HTTPStatusError, + HTTPTimeoutError, +) + +LOGGER = logging.getLogger(__name__) + +DEFAULT_TIMEOUT = 10 + +STANDARD_FIELDS = { + 'allow_redirects': True, + 'connect_timeout': None, + 'extra_headers': None, + 'headers': None, + 'log_requests': False, + 'password': None, + 'read_timeout': None, + 'timeout': DEFAULT_TIMEOUT, + 'tls_ca_cert': None, + 'tls_cert': None, + 'tls_ignore_warning': False, + 'tls_private_key': None, + 'tls_verify': True, + 'use_legacy_auth_encoding': True, + 'username': None, +} + +DEFAULT_REMAPPED_FIELDS: dict[str, dict[str, Any]] = {} + + +def _build_basic_auth(config: dict[str, Any]) -> httpx.BasicAuth | None: + if config['username'] is not None and config['password'] is not None: + return httpx.BasicAuth(config['username'], config['password']) + return None + + +def _build_verify(config: dict[str, Any]) -> bool | str: + if isinstance(config['tls_ca_cert'], str): + return config['tls_ca_cert'] + if not is_affirmative(config['tls_verify']): + return False + return True + + +def _build_cert(config: dict[str, Any]) -> str | tuple[str, str] | None: + cert = config['tls_cert'] + if not isinstance(cert, str): + return None + private_key = config['tls_private_key'] + if isinstance(private_key, str): + return (cert, private_key) + return cert + + +def _build_timeout(config: dict[str, Any]) -> tuple[float, float]: + base = float(config['timeout']) + connect = float(config['connect_timeout']) if config['connect_timeout'] is not None else base + read = float(config['read_timeout']) if config['read_timeout'] is not None else base + return connect, read + + +def _map_httpx_exception(exc: BaseException) -> HTTPError: + """Translate an httpx exception into the library-agnostic equivalent. + + The mapping is symmetric with ``RequestsWrapper`` so that production + code which catches ``http_exceptions.HTTPTimeoutError`` (etc.) keeps + working when a check opts in to ``HTTPXWrapper``. + """ + if isinstance(exc, httpx.TimeoutException): + return HTTPTimeoutError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) + if isinstance(exc, httpx.ConnectError): + return HTTPConnectionError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) + if isinstance(exc, httpx.HTTPStatusError): + return HTTPStatusError( + str(exc) or exc.__class__.__name__, + request=getattr(exc, 'request', None), + response=getattr(exc, 'response', None), + ) + if isinstance(exc, httpx.RequestError): + return HTTPRequestError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) + return HTTPError(str(exc) or exc.__class__.__name__) + + +class HTTPXResponseAdapter: + """Wraps an ``httpx.Response`` to satisfy ``HTTPResponseProtocol``.""" + + __slots__ = ('_response',) + + def __init__(self, response: httpx.Response) -> None: + self._response = response + + @property + def status_code(self) -> int: + return self._response.status_code + + @property + def content(self) -> bytes: + return self._response.content + + @property + def text(self) -> str: + return self._response.text + + @property + def headers(self) -> Mapping[str, str]: + return self._response.headers + + @property + def ok(self) -> bool: + return self._response.status_code < 400 + + @property + def reason(self) -> str: + return self._response.reason_phrase + + @property + def encoding(self) -> str | None: + return self._response.encoding + + @encoding.setter + def encoding(self, value: str | None) -> None: + # OpenMetrics scrapers explicitly set ``encoding = 'utf-8'`` after the + # response is received; mirror that mutability. + self._response.encoding = value + + @property + def url(self) -> str: + return str(self._response.url) + + @property + def cookies(self) -> httpx.Cookies: + return self._response.cookies + + @property + def elapsed(self): + # httpx sets ``_elapsed`` via the bound stream's ``close()`` during the + # request lifecycle. Some transports (notably ``MockTransport`` used in + # tests) bypass that path because they construct a ``Response`` with + # buffered content from the start. Return a zero timedelta in that case + # so the attribute is always safe to read. + try: + return self._response.elapsed + except RuntimeError: + from datetime import timedelta + + return timedelta(0) + + def json(self, **kwargs: Any) -> Any: + return self._response.json(**kwargs) + + def raise_for_status(self) -> None: + # Mirror requests.Response.raise_for_status semantics (4xx/5xx only). + # httpx raises for any non-success including 3xx, but the migration + # target is requests behavior so existing checks keep working. + if self._response.status_code < 400: + return + try: + self._response.raise_for_status() + except httpx.HTTPStatusError as exc: + raise _map_httpx_exception(exc) from exc + + def close(self) -> None: + self._response.close() + + def iter_content(self, chunk_size: int | None = None, decode_unicode: bool = False) -> Iterator[bytes | str]: + # Always operate on the buffered ``.content`` so the behavior is identical + # regardless of whether the underlying object is a real ``httpx.Response`` + # or a fake response object produced by a test fixture. + content = self._response.content + if chunk_size is None: + yield content.decode('utf-8') if decode_unicode else content + return + for i in range(0, len(content), max(chunk_size, 1)): + chunk = content[i : i + chunk_size] + yield chunk.decode('utf-8') if decode_unicode else chunk + + def iter_lines( + self, + chunk_size: int | None = None, + decode_unicode: bool = False, + delimiter: bytes | str | None = None, + ) -> Iterator[bytes | str]: + if isinstance(delimiter, str): + delimiter = delimiter.encode('utf-8') + sep = delimiter or b'\n' + content = self._response.content + lines = content.split(sep) + if lines and not lines[-1]: + lines.pop() + for line in lines: + yield line.decode('utf-8') if decode_unicode else line + + def __enter__(self) -> 'HTTPXResponseAdapter': + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool | None: + self.close() + return None + + +class HTTPXWrapper: + """Implements ``HTTPClientProtocol`` using a single shared ``httpx.Client``. + + Per the Phase 2 MVP plan (D3), one ``httpx.Client`` is created at + construction and reused across all requests for the lifetime of the + wrapper. Closing the wrapper (or letting it fall out of scope) closes + the underlying client. + """ + + __slots__ = ( + '_client', + '_log_requests', + 'logger', + 'options', + ) + + def __init__( + self, + instance: dict[str, Any], + init_config: dict[str, Any] | None = None, + remapper: dict[str, dict[str, Any]] | None = None, + logger: logging.Logger | None = None, + transport: httpx.BaseTransport | None = None, + ) -> None: + self.logger = logger or LOGGER + init_config = init_config or {} + + config = self._resolve_config(instance, init_config, remapper) + + headers = get_default_headers() + if config['headers']: + headers.clear() + update_headers(headers, config['headers']) + if config['extra_headers']: + update_headers(headers, config['extra_headers']) + + auth = _build_basic_auth(config) + verify = _build_verify(config) + cert = _build_cert(config) + timeout = _build_timeout(config) + allow_redirects = is_affirmative(config['allow_redirects']) + + self.options: dict[str, Any] = { + 'auth': auth, + 'cert': cert, + 'headers': headers, + 'timeout': timeout, + 'verify': verify, + 'allow_redirects': allow_redirects, + } + + self._log_requests = is_affirmative(config['log_requests']) + self._client = self._build_client(transport) + + @staticmethod + def _resolve_config( + instance: dict[str, Any], + init_config: dict[str, Any], + remapper: dict[str, dict[str, Any]] | None, + ) -> dict[str, Any]: + default_fields = dict(STANDARD_FIELDS) + default_fields['log_requests'] = init_config.get('log_requests', default_fields['log_requests']) + default_fields['timeout'] = init_config.get('timeout', default_fields['timeout']) + default_fields['tls_ignore_warning'] = init_config.get( + 'tls_ignore_warning', default_fields['tls_ignore_warning'] + ) + + config = {field: instance.get(field, value) for field, value in default_fields.items()} + + remapper = dict(remapper) if remapper else {} + remapper.update(DEFAULT_REMAPPED_FIELDS) + + for remapped_field, data in remapper.items(): + field = data.get('name') + if field not in STANDARD_FIELDS: + continue + if field in instance: + continue + + default = default_fields[field] + if data.get('invert'): + default = not default + + value = instance.get(remapped_field, data.get('default', default)) + if data.get('invert'): + value = not is_affirmative(value) + + config[field] = value + return config + + def _build_client(self, transport: httpx.BaseTransport | None) -> httpx.Client: + kwargs: dict[str, Any] = { + 'headers': self.options['headers'], + 'timeout': httpx.Timeout( + connect=self.options['timeout'][0], read=self.options['timeout'][1], write=None, pool=None + ), + 'follow_redirects': self.options['allow_redirects'], + 'verify': self.options['verify'], + } + if self.options['cert'] is not None: + kwargs['cert'] = self.options['cert'] + if self.options['auth'] is not None: + kwargs['auth'] = self.options['auth'] + if transport is not None: + kwargs['transport'] = transport + return httpx.Client(**kwargs) + + def get_header(self, name: str, default: str | None = None) -> str | None: + for key, value in self.options['headers'].items(): + if key.lower() == name.lower(): + return value + return default + + def set_header(self, name: str, value: str) -> None: + for key in list(self.options['headers']): + if key.lower() == name.lower(): + self.options['headers'][key] = value + self._client.headers[key] = value + return + self.options['headers'][name] = value + self._client.headers[name] = value + + def get(self, url: str, **options: Any) -> HTTPXResponseAdapter: + return self._request('GET', url, options) + + def post(self, url: str, **options: Any) -> HTTPXResponseAdapter: + return self._request('POST', url, options) + + def put(self, url: str, **options: Any) -> HTTPXResponseAdapter: + return self._request('PUT', url, options) + + def delete(self, url: str, **options: Any) -> HTTPXResponseAdapter: + return self._request('DELETE', url, options) + + def head(self, url: str, **options: Any) -> HTTPXResponseAdapter: + return self._request('HEAD', url, options) + + def patch(self, url: str, **options: Any) -> HTTPXResponseAdapter: + return self._request('PATCH', url, options) + + def options_method(self, url: str, **options: Any) -> HTTPXResponseAdapter: + return self._request('OPTIONS', url, options) + + def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPXResponseAdapter: + if self._log_requests: + self.logger.debug('Sending %s request to %s', method, url) + + request_kwargs = self._build_request_kwargs(options) + try: + response = self._client.request(method, url, **request_kwargs) + except httpx.HTTPError as exc: + raise _map_httpx_exception(exc) from exc + return HTTPXResponseAdapter(response) + + def _build_request_kwargs(self, options: dict[str, Any]) -> dict[str, Any]: + """Translate the call-site options to httpx.Client.request kwargs. + + Honors per-request overrides for ``headers``, ``params``, ``json``, + ``data``, ``timeout``, and ``extra_headers``. ``allow_redirects`` and + ``verify`` / ``cert`` are client-level and not overridable per request + in the MVP. + """ + kwargs: dict[str, Any] = {} + passthrough = ('params', 'json', 'data', 'content', 'files', 'cookies') + for key in passthrough: + if key in options: + kwargs[key] = options[key] + + extra_headers = options.get('extra_headers') + headers = options.get('headers') + merged_headers: dict[str, str] | None = None + if headers is not None or extra_headers is not None: + merged_headers = {} + if headers is not None: + merged_headers.update(headers) + if extra_headers is not None: + merged_headers.update(extra_headers) + if merged_headers is not None: + kwargs['headers'] = merged_headers + + if 'timeout' in options: + timeout_value = options['timeout'] + if isinstance(timeout_value, (tuple, list)) and len(timeout_value) == 2: + kwargs['timeout'] = httpx.Timeout( + connect=float(timeout_value[0]), + read=float(timeout_value[1]), + write=None, + pool=None, + ) + else: + kwargs['timeout'] = float(timeout_value) # type: ignore[arg-type] + + if 'follow_redirects' in options: + kwargs['follow_redirects'] = bool(options['follow_redirects']) + elif 'allow_redirects' in options: + kwargs['follow_redirects'] = bool(options['allow_redirects']) + + return kwargs + + def close(self) -> None: + client = getattr(self, '_client', None) + if client is not None: + client.close() + + def __enter__(self) -> 'HTTPXWrapper': + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool | None: + self.close() + return None + + def __del__(self) -> None: + try: + self.close() + except Exception: + pass diff --git a/datadog_checks_base/pyproject.toml b/datadog_checks_base/pyproject.toml index d52b26f0a731a..1bd0d55c569c0 100644 --- a/datadog_checks_base/pyproject.toml +++ b/datadog_checks_base/pyproject.toml @@ -38,6 +38,7 @@ deps = [ "cachetools==7.0.5", "cryptography==46.0.7", "ddtrace==3.19.5", + "httpx==0.28.1", "jellyfish==1.2.1", "lazy-loader==0.5", "prometheus-client==0.24.1", diff --git a/datadog_checks_base/tests/base/utils/http_httpx/__init__.py b/datadog_checks_base/tests/base/utils/http_httpx/__init__.py new file mode 100644 index 0000000000000..75c6647cb9233 --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/conftest.py b/datadog_checks_base/tests/base/utils/http_httpx/conftest.py new file mode 100644 index 0000000000000..7b507436d09c6 --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/conftest.py @@ -0,0 +1,104 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Callable + +import httpx +import pytest + + +@pytest.fixture +def make_transport() -> Callable[[Callable[[httpx.Request], httpx.Response]], httpx.MockTransport]: + """Returns a factory for building httpx.MockTransport from a handler callable.""" + + def _factory(handler): + return httpx.MockTransport(handler) + + return _factory + + +@pytest.fixture +def echo_transport() -> httpx.MockTransport: + """Mock transport that echoes the request method, URL, headers, and body as JSON.""" + + def handler(request: httpx.Request) -> httpx.Response: + body = b'' + try: + body = request.content + except httpx.RequestNotRead: + pass + payload = { + 'method': request.method, + 'url': str(request.url), + 'headers': dict(request.headers), + 'body': body.decode('utf-8') if body else '', + } + return httpx.Response(200, json=payload) + + return httpx.MockTransport(handler) + + +@pytest.fixture +def status_transport_factory() -> Callable[[int, bytes | str], httpx.MockTransport]: + """Builds a transport that returns a fixed status code and body.""" + + def _factory(status_code: int, body: bytes | str = b''): + def handler(_request: httpx.Request) -> httpx.Response: + if isinstance(body, str): + return httpx.Response(status_code, text=body) + return httpx.Response(status_code, content=body) + + return httpx.MockTransport(handler) + + return _factory + + +@pytest.fixture +def json_transport_factory() -> Callable[[dict, int], httpx.MockTransport]: + def _factory(payload: dict, status_code: int = 200): + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(status_code, json=payload) + + return httpx.MockTransport(handler) + + return _factory + + +@pytest.fixture +def raising_transport_factory() -> Callable[[Exception], httpx.MockTransport]: + """Builds a transport that raises a given exception when invoked.""" + + def _factory(exc: Exception): + def handler(_request: httpx.Request) -> httpx.Response: + raise exc + + return httpx.MockTransport(handler) + + return _factory + + +@pytest.fixture +def captured_requests() -> list[httpx.Request]: + return [] + + +@pytest.fixture +def capturing_transport(captured_requests: list[httpx.Request]) -> httpx.MockTransport: + def handler(request: httpx.Request) -> httpx.Response: + # Read the body so tests can inspect it. + _ = request.content + captured_requests.append(request) + return httpx.Response(200, json={'ok': True}) + + return httpx.MockTransport(handler) + + +def parse_basic_auth(header_value: str) -> tuple[str, str]: + """Decode a Basic auth header value into (user, pass).""" + import base64 + + scheme, _, b64 = header_value.partition(' ') + assert scheme.lower() == 'basic' + user_pass = base64.b64decode(b64).decode('utf-8') + user, _, password = user_pass.partition(':') + return user, password diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py b/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py new file mode 100644 index 0000000000000..2548ec8c04c24 --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py @@ -0,0 +1,40 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from datadog_checks.base.utils.http_httpx import HTTPXWrapper + +from .conftest import parse_basic_auth + + +def test_basic_auth_sent_on_request(capturing_transport, captured_requests): + http = HTTPXWrapper( + {'username': 'alice', 'password': 'secret'}, + {}, + transport=capturing_transport, + ) + http.get('http://example.test/') + assert len(captured_requests) == 1 + user, password = parse_basic_auth(captured_requests[0].headers['authorization']) + assert user == 'alice' + assert password == 'secret' + + +def test_no_auth_without_credentials(capturing_transport, captured_requests): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + http.get('http://example.test/') + assert 'authorization' not in captured_requests[0].headers + + +def test_basic_auth_only_when_both_user_and_password_set(capturing_transport, captured_requests): + http = HTTPXWrapper({'username': 'alice'}, {}, transport=capturing_transport) + http.get('http://example.test/') + assert 'authorization' not in captured_requests[0].headers + + +def test_basic_auth_options_exposes_auth(capturing_transport): + http = HTTPXWrapper( + {'username': 'alice', 'password': 'secret'}, + {}, + transport=capturing_transport, + ) + assert http.options['auth'] is not None diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_config.py b/datadog_checks_base/tests/base/utils/http_httpx/test_config.py new file mode 100644 index 0000000000000..922d97d3c91ef --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_config.py @@ -0,0 +1,88 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from datadog_checks.base.utils.http_httpx import HTTPXWrapper + + +def test_options_dict_shape(capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + options = http.options + assert 'auth' in options + assert 'cert' in options + assert 'headers' in options + assert 'timeout' in options + assert 'verify' in options + assert 'allow_redirects' in options + + +def test_default_headers_include_user_agent(capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + assert any(key.lower() == 'user-agent' for key in http.options['headers']) + + +def test_extra_headers_merge(capturing_transport, captured_requests): + http = HTTPXWrapper({'extra_headers': {'X-Extra': 'value'}}, {}, transport=capturing_transport) + http.get('http://example.test/') + assert captured_requests[0].headers['x-extra'] == 'value' + + +def test_headers_override_defaults(capturing_transport, captured_requests): + http = HTTPXWrapper({'headers': {'User-Agent': 'custom-agent/1.0'}}, {}, transport=capturing_transport) + http.get('http://example.test/') + assert captured_requests[0].headers['user-agent'] == 'custom-agent/1.0' + + +def test_per_request_headers_merge_into_request(capturing_transport, captured_requests): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + http.get('http://example.test/', headers={'X-Per-Request': 'yes'}) + assert captured_requests[0].headers['x-per-request'] == 'yes' + + +def test_timeout_default(capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + timeout = http.options['timeout'] + assert isinstance(timeout, tuple) and len(timeout) == 2 + + +def test_timeout_from_instance(capturing_transport): + http = HTTPXWrapper({'timeout': 25}, {}, transport=capturing_transport) + connect, read = http.options['timeout'] + assert connect == 25.0 + assert read == 25.0 + + +def test_connect_and_read_timeout_split(capturing_transport): + http = HTTPXWrapper({'connect_timeout': 5, 'read_timeout': 30}, {}, transport=capturing_transport) + connect, read = http.options['timeout'] + assert connect == 5.0 + assert read == 30.0 + + +def test_verify_defaults_to_true(capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + assert http.options['verify'] is True + + +def test_verify_false_when_tls_verify_off(capturing_transport): + http = HTTPXWrapper({'tls_verify': False}, {}, transport=capturing_transport) + assert http.options['verify'] is False + + +def test_get_header_case_insensitive(capturing_transport): + http = HTTPXWrapper({'extra_headers': {'X-Foo': 'bar'}}, {}, transport=capturing_transport) + assert http.get_header('x-foo') == 'bar' + assert http.get_header('X-FOO') == 'bar' + assert http.get_header('missing') is None + assert http.get_header('missing', default='fallback') == 'fallback' + + +def test_set_header_overrides_existing(capturing_transport): + http = HTTPXWrapper({'extra_headers': {'X-Foo': 'bar'}}, {}, transport=capturing_transport) + http.set_header('X-FOO', 'new') + assert http.get_header('x-foo') == 'new' + + +def test_remapper_renames_field(capturing_transport): + remapper = {'ssl_validation': {'name': 'tls_verify'}} + http = HTTPXWrapper({'ssl_validation': False}, {}, remapper=remapper, transport=capturing_transport) + assert http.options['verify'] is False diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py b/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py new file mode 100644 index 0000000000000..f6367e5ebfba5 --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py @@ -0,0 +1,78 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import httpx +import pytest + +from datadog_checks.base.utils.http_exceptions import ( + HTTPConnectionError, + HTTPError, + HTTPStatusError, + HTTPTimeoutError, +) +from datadog_checks.base.utils.http_httpx import HTTPXWrapper + + +def test_connect_timeout_maps_to_timeout_error(raising_transport_factory): + transport = raising_transport_factory(httpx.ConnectTimeout('boom')) + http = HTTPXWrapper({}, {}, transport=transport) + with pytest.raises(HTTPTimeoutError): + http.get('http://example.test/') + + +def test_read_timeout_maps_to_timeout_error(raising_transport_factory): + transport = raising_transport_factory(httpx.ReadTimeout('slow')) + http = HTTPXWrapper({}, {}, transport=transport) + with pytest.raises(HTTPTimeoutError): + http.get('http://example.test/') + + +def test_pool_timeout_maps_to_timeout_error(raising_transport_factory): + transport = raising_transport_factory(httpx.PoolTimeout('pool')) + http = HTTPXWrapper({}, {}, transport=transport) + with pytest.raises(HTTPTimeoutError): + http.get('http://example.test/') + + +def test_connect_error_maps_to_connection_error(raising_transport_factory): + transport = raising_transport_factory(httpx.ConnectError('refused')) + http = HTTPXWrapper({}, {}, transport=transport) + with pytest.raises(HTTPConnectionError): + http.get('http://example.test/') + + +def test_protocol_error_maps_to_http_error(raising_transport_factory): + transport = raising_transport_factory(httpx.LocalProtocolError('bad')) + http = HTTPXWrapper({}, {}, transport=transport) + with pytest.raises(HTTPError): + http.get('http://example.test/') + + +def test_request_error_maps_to_http_error(raising_transport_factory): + transport = raising_transport_factory(httpx.RequestError('generic')) + http = HTTPXWrapper({}, {}, transport=transport) + with pytest.raises(HTTPError): + http.get('http://example.test/') + + +def test_raise_for_status_4xx_maps_to_status_error(status_transport_factory): + transport = status_transport_factory(404, b'not found') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + with pytest.raises(HTTPStatusError): + response.raise_for_status() + + +def test_raise_for_status_5xx_maps_to_status_error(status_transport_factory): + transport = status_transport_factory(502, b'bad gateway') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + with pytest.raises(HTTPStatusError): + response.raise_for_status() + + +def test_raise_for_status_3xx_does_not_raise(status_transport_factory): + transport = status_transport_factory(301, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + response.raise_for_status() diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py new file mode 100644 index 0000000000000..d24d3f2c53c5c --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py @@ -0,0 +1,37 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import sys + +import pytest + +from datadog_checks.base.utils.http_httpx import HTTPXWrapper + + +def test_close_is_idempotent(capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + http.close() + http.close() + + +def test_context_manager(capturing_transport): + with HTTPXWrapper({}, {}, transport=capturing_transport) as http: + response = http.get('http://example.test/') + assert response.status_code == 200 + + +def test_single_client_reused_across_requests(capturing_transport, captured_requests): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + http.get('http://example.test/a') + http.get('http://example.test/b') + http.post('http://example.test/c', json={'x': 1}) + assert len(captured_requests) == 3 + + +def test_module_import_fails_without_httpx(monkeypatch): + """Per D4: importing http_httpx without httpx installed raises a clean ImportError.""" + # Force a re-import with httpx missing from sys.modules + monkeypatch.setitem(sys.modules, 'httpx', None) + monkeypatch.delitem(sys.modules, 'datadog_checks.base.utils.http_httpx', raising=False) + with pytest.raises(ImportError): + import datadog_checks.base.utils.http_httpx # noqa: F401 diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py b/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py new file mode 100644 index 0000000000000..90326944c68d6 --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py @@ -0,0 +1,66 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from datadog_checks.base.utils.http_exceptions import HTTPStatusError +from datadog_checks.base.utils.http_httpx import HTTPXWrapper + +METHODS = ['get', 'post', 'put', 'delete', 'head', 'patch', 'options_method'] +HTTP_VERBS = { + 'get': 'GET', + 'post': 'POST', + 'put': 'PUT', + 'delete': 'DELETE', + 'head': 'HEAD', + 'patch': 'PATCH', + 'options_method': 'OPTIONS', +} + + +@pytest.mark.parametrize('method', METHODS) +def test_method_happy_path(method, captured_requests, capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + fn = getattr(http, method) + response = fn('http://example.test/path', headers={'X-Test': '1'}) + + assert response.status_code == 200 + assert len(captured_requests) == 1 + assert captured_requests[0].method == HTTP_VERBS[method] + assert str(captured_requests[0].url) == 'http://example.test/path' + + +@pytest.mark.parametrize('method', METHODS) +def test_method_5xx_does_not_raise_unless_asked(method, status_transport_factory): + transport = status_transport_factory(500, b'oops') + http = HTTPXWrapper({}, {}, transport=transport) + fn = getattr(http, method) + response = fn('http://example.test/path') + assert response.status_code == 500 + + +@pytest.mark.parametrize('method', METHODS) +def test_method_raise_for_status_propagates(method, status_transport_factory): + transport = status_transport_factory(503, b'server unavailable') + http = HTTPXWrapper({}, {}, transport=transport) + fn = getattr(http, method) + response = fn('http://example.test/path') + with pytest.raises(HTTPStatusError): + response.raise_for_status() + + +def test_post_json_body_is_serialized(capturing_transport, captured_requests): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + http.post('http://example.test/path', json={'a': 1, 'b': 'two'}) + assert len(captured_requests) == 1 + req = captured_requests[0] + assert req.headers['content-type'] == 'application/json' + assert b'"a":' in req.content + assert b'"b":' in req.content + + +def test_get_query_params_forwarded(capturing_transport, captured_requests): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + http.get('http://example.test/path', params={'foo': 'bar', 'baz': '1'}) + assert captured_requests[0].url.params['foo'] == 'bar' + assert captured_requests[0].url.params['baz'] == '1' diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py new file mode 100644 index 0000000000000..bdd0f691954cf --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py @@ -0,0 +1,135 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import httpx +import pytest + +from datadog_checks.base.utils.http_exceptions import HTTPStatusError +from datadog_checks.base.utils.http_httpx import HTTPXWrapper + + +def test_response_content_bytes(status_transport_factory): + transport = status_transport_factory(200, b'hello world') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.content == b'hello world' + + +def test_response_text(status_transport_factory): + transport = status_transport_factory(200, 'hello') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.text == 'hello' + + +def test_response_status_code(status_transport_factory): + transport = status_transport_factory(204, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.status_code == 204 + + +def test_response_json(json_transport_factory): + transport = json_transport_factory({'a': 1, 'b': [1, 2, 3]}) + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.json() == {'a': 1, 'b': [1, 2, 3]} + + +def test_response_headers_case_insensitive(): + def handler(_request): + return httpx.Response(200, headers={'X-Custom': 'foo', 'Content-Type': 'text/plain'}) + + transport = httpx.MockTransport(handler) + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.headers['x-custom'] == 'foo' + assert response.headers['X-CUSTOM'] == 'foo' + assert response.headers['content-type'] == 'text/plain' + + +def test_response_raise_for_status_4xx(status_transport_factory): + transport = status_transport_factory(404, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + with pytest.raises(HTTPStatusError): + response.raise_for_status() + + +def test_response_raise_for_status_5xx(status_transport_factory): + transport = status_transport_factory(500, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + with pytest.raises(HTTPStatusError): + response.raise_for_status() + + +def test_response_iter_content(status_transport_factory): + transport = status_transport_factory(200, b'abcdef') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + chunks = list(response.iter_content(chunk_size=2)) + assert b''.join(chunks) == b'abcdef' + + +def test_response_iter_lines_bytes_default(status_transport_factory): + transport = status_transport_factory(200, b'a\nb\nc') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + lines = list(response.iter_lines()) + assert lines == [b'a', b'b', b'c'] + + +def test_response_iter_lines_decode_unicode(status_transport_factory): + transport = status_transport_factory(200, b'a\nb\nc') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + lines = list(response.iter_lines(decode_unicode=True)) + assert lines == ['a', 'b', 'c'] + + +def test_response_encoding(status_transport_factory): + transport = status_transport_factory(200, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.encoding is not None + + +def test_response_url(status_transport_factory): + transport = status_transport_factory(200, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/path?x=1') + assert 'example.test' in str(response.url) + + +def test_response_cookies_empty_by_default(status_transport_factory): + transport = status_transport_factory(200, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.cookies is not None + + +def test_response_elapsed(status_transport_factory): + transport = status_transport_factory(200, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.elapsed is not None + + +def test_response_close(status_transport_factory): + transport = status_transport_factory(200, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + response.close() + + +def test_response_ok_property(status_transport_factory): + transport = status_transport_factory(200, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.ok is True + + transport = status_transport_factory(500, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.ok is False diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_tls.py b/datadog_checks_base/tests/base/utils/http_httpx/test_tls.py new file mode 100644 index 0000000000000..6f57a6ded7ad1 --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_tls.py @@ -0,0 +1,38 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from datadog_checks.base.utils.http_httpx import HTTPXWrapper + + +def test_tls_verify_default_true(capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + assert http.options['verify'] is True + + +def test_tls_verify_false(capturing_transport): + http = HTTPXWrapper({'tls_verify': False}, {}, transport=capturing_transport) + assert http.options['verify'] is False + + +def test_tls_ca_cert_uses_path(capturing_transport): + http = HTTPXWrapper({'tls_ca_cert': '/etc/ssl/ca.pem'}, {}, transport=capturing_transport) + assert http.options['verify'] == '/etc/ssl/ca.pem' + + +def test_tls_client_cert_string(capturing_transport): + http = HTTPXWrapper({'tls_cert': '/etc/ssl/client.pem'}, {}, transport=capturing_transport) + assert http.options['cert'] == '/etc/ssl/client.pem' + + +def test_tls_client_cert_with_key(capturing_transport): + http = HTTPXWrapper( + {'tls_cert': '/etc/ssl/client.pem', 'tls_private_key': '/etc/ssl/client.key'}, + {}, + transport=capturing_transport, + ) + assert http.options['cert'] == ('/etc/ssl/client.pem', '/etc/ssl/client.key') + + +def test_tls_no_cert_when_not_configured(capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + assert http.options['cert'] is None diff --git a/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py b/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py index 9e474ba6a2ef0..1adb26e907d91 100644 --- a/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py +++ b/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py @@ -297,9 +297,25 @@ def mock_response(): @pytest.fixture def mock_http_response(mocker, mock_response): - yield lambda *args, **kwargs: mocker.patch( - kwargs.pop('method', _DEFAULT_MOCK_METHOD), return_value=mock_response(*args, **kwargs) - ) + """Patch the default HTTP entry-point to return a ``MockHTTPResponse``. + + Patches ``requests.Session.get`` by default. When ``method`` is the default, + we also patch ``httpx.Client.send`` so that checks opting into the Phase 2 + HTTPXWrapper via ``use_httpx=true`` are intercepted by the same fixture. + """ + + def _patch(*args, **kwargs): + method = kwargs.pop('method', _DEFAULT_MOCK_METHOD) + response = mock_response(*args, **kwargs) + primary = mocker.patch(method, return_value=response) + if method == _DEFAULT_MOCK_METHOD: + try: + mocker.patch('httpx.Client.request', return_value=response) + except (ImportError, AttributeError, ModuleNotFoundError): + pass + return primary + + yield _patch @pytest.fixture diff --git a/kong/tests/conftest.py b/kong/tests/conftest.py index 762a3bf82e8ab..8cf8152cd8e93 100644 --- a/kong/tests/conftest.py +++ b/kong/tests/conftest.py @@ -26,4 +26,5 @@ def dd_environment(): @pytest.fixture def instance_openmetrics_v2(): - return common.openmetrics_instance + # Phase 2 MVP POC opt-in (see RFC 2026-02-11): exercise HTTPXWrapper end-to-end. + return {**common.openmetrics_instance, 'use_httpx': True} diff --git a/kong/tests/test_unit.py b/kong/tests/test_unit.py index 41dfb02a77171..f467fce5c94f4 100644 --- a/kong/tests/test_unit.py +++ b/kong/tests/test_unit.py @@ -57,6 +57,8 @@ def test_check_v3(aggregator, dd_run_check, mock_http_response): instance = { 'openmetrics_endpoint': METRICS_URL, 'extra_metrics': [{'kong_memory_workers_lua_vms_bytes': 'memory.workers.lua.vms.bytes'}], + # Phase 2 MVP POC opt-in (see RFC 2026-02-11): exercise HTTPXWrapper end-to-end. + 'use_httpx': True, } check = Kong('kong', {}, [instance]) @@ -79,6 +81,8 @@ def test_check(aggregator, dd_run_check, mock_http_response): instance = { 'openmetrics_endpoint': METRICS_URL, 'extra_metrics': [{'kong_memory_workers_lua_vms_bytes': 'memory.workers.lua.vms.bytes'}], + # Phase 2 MVP POC opt-in (see RFC 2026-02-11): exercise HTTPXWrapper end-to-end. + 'use_httpx': True, } check = Kong('kong', {}, [instance]) dd_run_check(check) diff --git a/kuma/tests/conftest.py b/kuma/tests/conftest.py index e7f54ba7284fc..767a3f22a4755 100644 --- a/kuma/tests/conftest.py +++ b/kuma/tests/conftest.py @@ -82,9 +82,11 @@ def dd_environment(dd_save_state): @pytest.fixture(scope='session') def instance(dd_get_state): - return dd_get_state( + # Phase 2 MVP POC opt-in (see RFC 2026-02-11): exercise HTTPXWrapper end-to-end. + base_instance = dd_get_state( 'kuma_instance', default={ 'openmetrics_endpoint': 'http://localhost:5680/metrics', }, ) + return {**base_instance, 'use_httpx': True} From 72d4bc3faf24a94046c22be07a71c81f01e35569 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 22 May 2026 17:13:43 -0400 Subject: [PATCH 02/19] phase2 mvp: add changelog fragment Co-Authored-By: Claude Opus 4.7 (1M context) --- datadog_checks_base/changelog.d/23822.added | 1 + 1 file changed, 1 insertion(+) create mode 100644 datadog_checks_base/changelog.d/23822.added diff --git a/datadog_checks_base/changelog.d/23822.added b/datadog_checks_base/changelog.d/23822.added new file mode 100644 index 0000000000000..115c23bd94631 --- /dev/null +++ b/datadog_checks_base/changelog.d/23822.added @@ -0,0 +1 @@ +Add HTTPXWrapper, an httpx-backed HTTP client wrapper opt-in via the ``use_httpx`` instance config flag. The default remains the existing requests-based client. From 108567dc4a714627ed5030a0a025faa262f23746 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 22 May 2026 17:39:33 -0400 Subject: [PATCH 03/19] phase2 mvp: address agint:review iteration 1 Fix docstring drift on mock_http_response (patches Client.request, not send). Add options['proxies'] = None for shape-parity with RequestsWrapper. Document the silent kwarg-drop policy in _build_request_kwargs. Add tests for password-only auth branch, iter_content branches, elapsed RuntimeError fallback, and AgentCheck.http httpx dispatch. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../datadog_checks/base/utils/http_httpx.py | 13 +++++++ .../base/utils/http_httpx/test_auth_basic.py | 6 +++ .../base/utils/http_httpx/test_lifecycle.py | 12 ++++++ .../base/utils/http_httpx/test_response.py | 37 +++++++++++++++++++ .../datadog_checks/dev/plugin/pytest.py | 2 +- 5 files changed, 69 insertions(+), 1 deletion(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py index aa7143122174f..8678d62523e4c 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py @@ -271,10 +271,15 @@ def __init__( timeout = _build_timeout(config) allow_redirects = is_affirmative(config['allow_redirects']) + # ``proxies`` is included as ``None`` for shape-parity with + # ``RequestsWrapper.options`` so existing reads of + # ``check.http.options['proxies']`` do not KeyError on a check that + # opts into HTTPXWrapper. Proxy wiring itself is Phase 3. self.options: dict[str, Any] = { 'auth': auth, 'cert': cert, 'headers': headers, + 'proxies': None, 'timeout': timeout, 'verify': verify, 'allow_redirects': allow_redirects, @@ -390,6 +395,14 @@ def _build_request_kwargs(self, options: dict[str, Any]) -> dict[str, Any]: ``data``, ``timeout``, and ``extra_headers``. ``allow_redirects`` and ``verify`` / ``cert`` are client-level and not overridable per request in the MVP. + + Any kwarg not in the passthrough list below is silently dropped. This + is intentional — ``RequestsWrapper`` accepts a broader set of options + than the MVP supports, and silently dropping unknown kwargs lets + existing call sites (notably the OM v2 scraper, which passes + ``stream=True``) work without lib-specific branches at the call site. + Unsupported kwargs that materially affect behavior should be added to + the passthrough list in Phase 3. """ kwargs: dict[str, Any] = {} passthrough = ('params', 'json', 'data', 'content', 'files', 'cookies') diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py b/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py index 2548ec8c04c24..73c397de67464 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py @@ -31,6 +31,12 @@ def test_basic_auth_only_when_both_user_and_password_set(capturing_transport, ca assert 'authorization' not in captured_requests[0].headers +def test_basic_auth_skipped_when_only_password_set(capturing_transport, captured_requests): + http = HTTPXWrapper({'password': 'secret'}, {}, transport=capturing_transport) + http.get('http://example.test/') + assert 'authorization' not in captured_requests[0].headers + + def test_basic_auth_options_exposes_auth(capturing_transport): http = HTTPXWrapper( {'username': 'alice', 'password': 'secret'}, diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py index d24d3f2c53c5c..8dd32d2b4edb6 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py @@ -35,3 +35,15 @@ def test_module_import_fails_without_httpx(monkeypatch): monkeypatch.delitem(sys.modules, 'datadog_checks.base.utils.http_httpx', raising=False) with pytest.raises(ImportError): import datadog_checks.base.utils.http_httpx # noqa: F401 + + +def test_agentcheck_http_dispatch_returns_httpx_wrapper(): + """Mirror of test_http.py::test_activate for the use_httpx=True path. + + The default-path counterpart (``use_httpx`` absent → ``RequestsWrapper``) is + already covered by ``tests/base/utils/http/test_http.py::TestAttribute::test_activate``. + """ + from datadog_checks.base import AgentCheck + + check = AgentCheck('test', {}, [{'use_httpx': True}]) + assert isinstance(check.http, HTTPXWrapper) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py index bdd0f691954cf..7bd226be7ae06 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py @@ -72,6 +72,23 @@ def test_response_iter_content(status_transport_factory): assert b''.join(chunks) == b'abcdef' +def test_response_iter_content_chunk_size_none(status_transport_factory): + transport = status_transport_factory(200, b'hello') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + chunks = list(response.iter_content()) + assert chunks == [b'hello'] + + +def test_response_iter_content_decode_unicode(status_transport_factory): + transport = status_transport_factory(200, b'hello world') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + chunks = list(response.iter_content(chunk_size=3, decode_unicode=True)) + assert all(isinstance(c, str) for c in chunks) + assert ''.join(chunks) == 'hello world' + + def test_response_iter_lines_bytes_default(status_transport_factory): transport = status_transport_factory(200, b'a\nb\nc') http = HTTPXWrapper({}, {}, transport=transport) @@ -116,6 +133,26 @@ def test_response_elapsed(status_transport_factory): assert response.elapsed is not None +def test_response_elapsed_returns_zero_on_runtime_error(status_transport_factory): + """Cover the RuntimeError fallback in HTTPXResponseAdapter.elapsed. + + httpx 0.28 raises ``RuntimeError`` from ``.elapsed`` until the bound stream's + ``close()`` has finalized the timer. When MockTransport bypasses that path + by serving buffered content, the adapter should return ``timedelta(0)`` so + callers never see the RuntimeError. + """ + from datetime import timedelta + + transport = status_transport_factory(200, b'hello') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + # Forge the MockTransport quirk explicitly: if ``_elapsed`` was set, drop it + # so the property has to take the except branch. + if hasattr(response._response, '_elapsed'): + delattr(response._response, '_elapsed') + assert response.elapsed == timedelta(0) + + def test_response_close(status_transport_factory): transport = status_transport_factory(200, b'') http = HTTPXWrapper({}, {}, transport=transport) diff --git a/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py b/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py index 1adb26e907d91..65e9bc030407c 100644 --- a/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py +++ b/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py @@ -300,7 +300,7 @@ def mock_http_response(mocker, mock_response): """Patch the default HTTP entry-point to return a ``MockHTTPResponse``. Patches ``requests.Session.get`` by default. When ``method`` is the default, - we also patch ``httpx.Client.send`` so that checks opting into the Phase 2 + we also patch ``httpx.Client.request`` so that checks opting into the Phase 2 HTTPXWrapper via ``use_httpx=true`` are intercepted by the same fixture. """ From 5b5b02982537a3253aad4fb1ded9820259b44124 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Fri, 22 May 2026 17:58:53 -0400 Subject: [PATCH 04/19] phase2 mvp: address agint:review iteration 2 - Map HTTPConnectionError alongside requests ConnectionError in OM v2 scraper so ignore_connection_errors keeps working for use_httpx opt-ins - Bridge reason vs reason_phrase: prefer httpx's reason_phrase, fall back to the protocol-standard reason exposed by MockHTTPResponse - Narrow HTTPXWrapper.__del__ to AttributeError, matching RequestsWrapper - Drop unimplemented STANDARD_FIELDS keys (use_legacy_auth_encoding, tls_ignore_warning) so the config surface only lists what is honored - Strengthen response-surface tests with concrete assertions - Pin the single-shared-client invariant in test_lifecycle.py - Move use_httpx into kuma's dd_environment yield so e2e exercises the wrapper the same way kong does Co-Authored-By: Claude Opus 4.7 (1M context) --- .../openmetrics/v2/scraper/base_scraper.py | 6 ++- .../datadog_checks/base/utils/http_httpx.py | 20 ++++--- .../base/utils/http_httpx/test_lifecycle.py | 3 ++ .../base/utils/http_httpx/test_response.py | 52 +++++++++++++++---- .../datadog_checks/dev/plugin/pytest.py | 18 +++++-- kuma/tests/conftest.py | 11 ++-- 6 files changed, 83 insertions(+), 27 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py index 24719c329efef..773f57a2e2148 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py @@ -23,6 +23,7 @@ from datadog_checks.base.constants import ServiceCheck from datadog_checks.base.errors import ConfigurationError from datadog_checks.base.utils.functions import no_op, return_true +from datadog_checks.base.utils.http_exceptions import HTTPConnectionError class OpenMetricsScraper: @@ -405,7 +406,10 @@ def stream_connection_lines(self): self._content_type = connection.headers.get('Content-Type', '') for line in connection.iter_lines(decode_unicode=True): yield line - except ConnectionError as e: + except (ConnectionError, HTTPConnectionError) as e: + # ``HTTPConnectionError`` is the library-agnostic equivalent surfaced + # by ``HTTPXWrapper``; ``requests.exceptions.ConnectionError`` is the + # default RequestsWrapper path. if self.ignore_connection_errors: self.log.warning("OpenMetrics endpoint %s is not accessible", self.endpoint) else: diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py index 8678d62523e4c..b61abce736c91 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py @@ -49,10 +49,8 @@ 'timeout': DEFAULT_TIMEOUT, 'tls_ca_cert': None, 'tls_cert': None, - 'tls_ignore_warning': False, 'tls_private_key': None, 'tls_verify': True, - 'use_legacy_auth_encoding': True, 'username': None, } @@ -142,7 +140,14 @@ def ok(self) -> bool: @property def reason(self) -> str: - return self._response.reason_phrase + # The protocol field is ``reason``; ``httpx.Response`` calls the same + # field ``reason_phrase``, while ``MockHTTPResponse`` (used by the + # ``mock_http_response`` test fixture) exposes ``reason`` directly. + # Prefer the httpx name, fall back to the agnostic one. + reason = getattr(self._response, 'reason_phrase', None) + if reason is not None: + return reason + return getattr(self._response, 'reason', '') or '' @property def encoding(self) -> str | None: @@ -297,9 +302,6 @@ def _resolve_config( default_fields = dict(STANDARD_FIELDS) default_fields['log_requests'] = init_config.get('log_requests', default_fields['log_requests']) default_fields['timeout'] = init_config.get('timeout', default_fields['timeout']) - default_fields['tls_ignore_warning'] = init_config.get( - 'tls_ignore_warning', default_fields['tls_ignore_warning'] - ) config = {field: instance.get(field, value) for field, value in default_fields.items()} @@ -454,7 +456,11 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool | None: return None def __del__(self) -> None: + # Match ``RequestsWrapper.__del__`` — narrow to ``AttributeError`` so + # genuine httpx-close failures still surface during teardown. + # ``AttributeError`` fires when ``__init__`` raised before ``_client`` + # was assigned and Python still calls ``__del__``. try: self.close() - except Exception: + except AttributeError: pass diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py index 8dd32d2b4edb6..188de48939d91 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py @@ -21,10 +21,13 @@ def test_context_manager(capturing_transport): def test_single_client_reused_across_requests(capturing_transport, captured_requests): + """Phase 2 MVP D3: the wrapper holds one ``httpx.Client`` across all requests.""" http = HTTPXWrapper({}, {}, transport=capturing_transport) + client_before_requests = http._client http.get('http://example.test/a') http.get('http://example.test/b') http.post('http://example.test/c', json={'x': 1}) + assert http._client is client_before_requests assert len(captured_requests) == 3 diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py index 7bd226be7ae06..c57c1afd6c8aa 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py @@ -105,32 +105,46 @@ def test_response_iter_lines_decode_unicode(status_transport_factory): assert lines == ['a', 'b', 'c'] -def test_response_encoding(status_transport_factory): +def test_response_encoding_default_is_utf8(status_transport_factory): transport = status_transport_factory(200, b'') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/') - assert response.encoding is not None + # httpx defaults to utf-8 when no charset is signalled. Pin the exact value + # so a future httpx change that returns ``None`` here surfaces immediately. + assert response.encoding == 'utf-8' + + +def test_response_encoding_setter_propagates_to_inner_response(status_transport_factory): + """OM v2 scraper does ``response.encoding = 'utf-8'`` after the request.""" + transport = status_transport_factory(200, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + response.encoding = 'latin-1' + assert response.encoding == 'latin-1' + assert response._response.encoding == 'latin-1' def test_response_url(status_transport_factory): transport = status_transport_factory(200, b'') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/path?x=1') - assert 'example.test' in str(response.url) + assert str(response.url) == 'http://example.test/path?x=1' def test_response_cookies_empty_by_default(status_transport_factory): transport = status_transport_factory(200, b'') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/') - assert response.cookies is not None + assert len(response.cookies) == 0 def test_response_elapsed(status_transport_factory): + from datetime import timedelta + transport = status_transport_factory(200, b'') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/') - assert response.elapsed is not None + assert isinstance(response.elapsed, timedelta) def test_response_elapsed_returns_zero_on_runtime_error(status_transport_factory): @@ -153,20 +167,36 @@ def test_response_elapsed_returns_zero_on_runtime_error(status_transport_factory assert response.elapsed == timedelta(0) -def test_response_close(status_transport_factory): +def test_response_close_marks_inner_response_closed(status_transport_factory): transport = status_transport_factory(200, b'') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/') response.close() + assert response._response.is_closed is True -def test_response_ok_property(status_transport_factory): - transport = status_transport_factory(200, b'') +@pytest.mark.parametrize('status_code,expected_ok', [(200, True), (204, True), (301, True), (400, False), (500, False)]) +def test_response_ok_property(status_transport_factory, status_code, expected_ok): + transport = status_transport_factory(status_code, b'') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/') - assert response.ok is True + assert response.ok is expected_ok - transport = status_transport_factory(500, b'') + +def test_response_reason_from_httpx_response(status_transport_factory): + """``reason`` reads from the underlying ``httpx.Response.reason_phrase``.""" + transport = status_transport_factory(200, b'') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/') - assert response.ok is False + assert response.reason == 'OK' + + +def test_response_reason_falls_back_when_reason_phrase_missing(): + """Mock fixtures expose ``.reason``, not ``.reason_phrase`` — the adapter handles that.""" + from datadog_checks.base.utils.http_httpx import HTTPXResponseAdapter + + class _FakeResponseExposingReason: + reason = 'Not Found' + + adapter = HTTPXResponseAdapter(_FakeResponseExposingReason()) # type: ignore[arg-type] + assert adapter.reason == 'Not Found' diff --git a/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py b/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py index 65e9bc030407c..6fda7a5deff30 100644 --- a/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py +++ b/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py @@ -299,9 +299,15 @@ def mock_response(): def mock_http_response(mocker, mock_response): """Patch the default HTTP entry-point to return a ``MockHTTPResponse``. - Patches ``requests.Session.get`` by default. When ``method`` is the default, - we also patch ``httpx.Client.request`` so that checks opting into the Phase 2 - HTTPXWrapper via ``use_httpx=true`` are intercepted by the same fixture. + Patches ``requests.Session.get`` by default. When the caller does not + override ``method``, also patches ``httpx.Client.request`` so that checks + opting into the Phase 2 HTTPXWrapper via ``use_httpx=true`` are intercepted + by the same fixture. + + Note: passing an explicit ``method=`` skips the httpx companion patch. This + is intentional — a custom ``method`` already names the exact target. If a + check under ``use_httpx=true`` needs both, patch the httpx target with its + own ``method=`` call after. """ def _patch(*args, **kwargs): @@ -309,9 +315,13 @@ def _patch(*args, **kwargs): response = mock_response(*args, **kwargs) primary = mocker.patch(method, return_value=response) if method == _DEFAULT_MOCK_METHOD: + # ``ImportError`` covers the case where httpx is not installed in + # the test env (default RequestsWrapper-only consumers). Anything + # narrower (like a missing ``Client.request``) is a genuine bug we + # want to surface, not swallow. try: mocker.patch('httpx.Client.request', return_value=response) - except (ImportError, AttributeError, ModuleNotFoundError): + except ImportError: pass return primary diff --git a/kuma/tests/conftest.py b/kuma/tests/conftest.py index 767a3f22a4755..a3a5e29ccaa0f 100644 --- a/kuma/tests/conftest.py +++ b/kuma/tests/conftest.py @@ -73,7 +73,11 @@ def dd_environment(dd_save_state): metrics_endpoint = f'http://{kuma_metrics_url}:{kuma_metrics_port}/metrics' - env_instance = {'openmetrics_endpoint': metrics_endpoint} + # Phase 2 MVP POC opt-in: kuma is one of two integrations exercising + # HTTPXWrapper end-to-end. Setting use_httpx here (rather than only + # in the `instance` fixture) makes the flag flow into the Agent + # process during e2e runs that read from dd_environment. + env_instance = {'openmetrics_endpoint': metrics_endpoint, 'use_httpx': True} dd_save_state("kuma_instance", env_instance) @@ -82,11 +86,10 @@ def dd_environment(dd_save_state): @pytest.fixture(scope='session') def instance(dd_get_state): - # Phase 2 MVP POC opt-in (see RFC 2026-02-11): exercise HTTPXWrapper end-to-end. - base_instance = dd_get_state( + return dd_get_state( 'kuma_instance', default={ 'openmetrics_endpoint': 'http://localhost:5680/metrics', + 'use_httpx': True, }, ) - return {**base_instance, 'use_httpx': True} From 3962a15e19a470843d9e659000b475ccd168c46e Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Tue, 26 May 2026 14:51:53 -0400 Subject: [PATCH 05/19] phase2 mvp: condense docstrings to teammate style --- .../datadog_checks/base/checks/base.py | 2 - .../openmetrics/v2/scraper/base_scraper.py | 3 - .../datadog_checks/base/utils/http_httpx.py | 80 +++---------------- .../tests/base/utils/http_httpx/conftest.py | 10 --- .../base/utils/http_httpx/test_lifecycle.py | 8 -- .../base/utils/http_httpx/test_methods.py | 11 ++- .../base/utils/http_httpx/test_response.py | 14 ---- .../datadog_checks/dev/plugin/pytest.py | 17 +--- kuma/tests/conftest.py | 5 +- 9 files changed, 16 insertions(+), 134 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index afe10394393f8..9ec690ab2432b 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -414,8 +414,6 @@ def http(self) -> HTTPClientProtocol: if not hasattr(self, '_http'): instance = self.instance or {} if is_affirmative(instance.get('use_httpx', False)): - # Per Phase 2 MVP D4: an ImportError surfaces at construction - # time if httpx is not installed. from datadog_checks.base.utils.http_httpx import HTTPXWrapper self._http = HTTPXWrapper(instance, self.init_config, self.HTTP_CONFIG_REMAPPER, self.log) diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py index 773f57a2e2148..3f12c8e9f9c8a 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py @@ -407,9 +407,6 @@ def stream_connection_lines(self): for line in connection.iter_lines(decode_unicode=True): yield line except (ConnectionError, HTTPConnectionError) as e: - # ``HTTPConnectionError`` is the library-agnostic equivalent surfaced - # by ``HTTPXWrapper``; ``requests.exceptions.ConnectionError`` is the - # default RequestsWrapper path. if self.ignore_connection_errors: self.log.warning("OpenMetrics endpoint %s is not accessible", self.endpoint) else: diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py index b61abce736c91..2f412f360a0ce 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py @@ -1,24 +1,11 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -""" -Minimum-viable httpx-backed HTTP client wrapper. - -Implements ``HTTPClientProtocol`` and ``HTTPResponseProtocol`` so a check that -opts in via ``use_httpx=true`` in instance config can transparently swap from -``RequestsWrapper``. Feature surface is intentionally narrow per the Phase 2 -MVP plan: basic auth, TLS verify/cert, headers, timeouts, common request -options, and the exception mapping into ``datadog_checks.base.utils.http_exceptions``. - -Auth tokens, proxies, Unix-socket transports, Kerberos / NTLM / AWS / Digest -auth, connection-pool tuning, HTTP/2 and multipart uploads are deferred to -Phase 3. -""" - from __future__ import annotations import logging from collections.abc import Mapping +from datetime import timedelta from typing import Any, Iterator import httpx @@ -89,12 +76,7 @@ def _build_timeout(config: dict[str, Any]) -> tuple[float, float]: def _map_httpx_exception(exc: BaseException) -> HTTPError: - """Translate an httpx exception into the library-agnostic equivalent. - - The mapping is symmetric with ``RequestsWrapper`` so that production - code which catches ``http_exceptions.HTTPTimeoutError`` (etc.) keeps - working when a check opts in to ``HTTPXWrapper``. - """ + """Translate an httpx exception into the library-agnostic equivalent.""" if isinstance(exc, httpx.TimeoutException): return HTTPTimeoutError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) if isinstance(exc, httpx.ConnectError): @@ -111,7 +93,7 @@ def _map_httpx_exception(exc: BaseException) -> HTTPError: class HTTPXResponseAdapter: - """Wraps an ``httpx.Response`` to satisfy ``HTTPResponseProtocol``.""" + """Wraps an httpx.Response to satisfy HTTPResponseProtocol.""" __slots__ = ('_response',) @@ -140,10 +122,6 @@ def ok(self) -> bool: @property def reason(self) -> str: - # The protocol field is ``reason``; ``httpx.Response`` calls the same - # field ``reason_phrase``, while ``MockHTTPResponse`` (used by the - # ``mock_http_response`` test fixture) exposes ``reason`` directly. - # Prefer the httpx name, fall back to the agnostic one. reason = getattr(self._response, 'reason_phrase', None) if reason is not None: return reason @@ -155,8 +133,6 @@ def encoding(self) -> str | None: @encoding.setter def encoding(self, value: str | None) -> None: - # OpenMetrics scrapers explicitly set ``encoding = 'utf-8'`` after the - # response is received; mirror that mutability. self._response.encoding = value @property @@ -168,26 +144,17 @@ def cookies(self) -> httpx.Cookies: return self._response.cookies @property - def elapsed(self): - # httpx sets ``_elapsed`` via the bound stream's ``close()`` during the - # request lifecycle. Some transports (notably ``MockTransport`` used in - # tests) bypass that path because they construct a ``Response`` with - # buffered content from the start. Return a zero timedelta in that case - # so the attribute is always safe to read. + def elapsed(self) -> timedelta: try: return self._response.elapsed except RuntimeError: - from datetime import timedelta - return timedelta(0) def json(self, **kwargs: Any) -> Any: return self._response.json(**kwargs) def raise_for_status(self) -> None: - # Mirror requests.Response.raise_for_status semantics (4xx/5xx only). - # httpx raises for any non-success including 3xx, but the migration - # target is requests behavior so existing checks keep working. + # Mirror requests semantics (4xx/5xx only); httpx also raises on 3xx. if self._response.status_code < 400: return try: @@ -199,9 +166,6 @@ def close(self) -> None: self._response.close() def iter_content(self, chunk_size: int | None = None, decode_unicode: bool = False) -> Iterator[bytes | str]: - # Always operate on the buffered ``.content`` so the behavior is identical - # regardless of whether the underlying object is a real ``httpx.Response`` - # or a fake response object produced by a test fixture. content = self._response.content if chunk_size is None: yield content.decode('utf-8') if decode_unicode else content @@ -235,13 +199,7 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool | None: class HTTPXWrapper: - """Implements ``HTTPClientProtocol`` using a single shared ``httpx.Client``. - - Per the Phase 2 MVP plan (D3), one ``httpx.Client`` is created at - construction and reused across all requests for the lifetime of the - wrapper. Closing the wrapper (or letting it fall out of scope) closes - the underlying client. - """ + """Implements HTTPClientProtocol using a single shared httpx.Client per wrapper.""" __slots__ = ( '_client', @@ -276,10 +234,7 @@ def __init__( timeout = _build_timeout(config) allow_redirects = is_affirmative(config['allow_redirects']) - # ``proxies`` is included as ``None`` for shape-parity with - # ``RequestsWrapper.options`` so existing reads of - # ``check.http.options['proxies']`` do not KeyError on a check that - # opts into HTTPXWrapper. Proxy wiring itself is Phase 3. + # `proxies` is None for shape-parity with RequestsWrapper.options; proxy wiring is Phase 3. self.options: dict[str, Any] = { 'auth': auth, 'cert': cert, @@ -391,21 +346,7 @@ def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPXRespo return HTTPXResponseAdapter(response) def _build_request_kwargs(self, options: dict[str, Any]) -> dict[str, Any]: - """Translate the call-site options to httpx.Client.request kwargs. - - Honors per-request overrides for ``headers``, ``params``, ``json``, - ``data``, ``timeout``, and ``extra_headers``. ``allow_redirects`` and - ``verify`` / ``cert`` are client-level and not overridable per request - in the MVP. - - Any kwarg not in the passthrough list below is silently dropped. This - is intentional — ``RequestsWrapper`` accepts a broader set of options - than the MVP supports, and silently dropping unknown kwargs lets - existing call sites (notably the OM v2 scraper, which passes - ``stream=True``) work without lib-specific branches at the call site. - Unsupported kwargs that materially affect behavior should be added to - the passthrough list in Phase 3. - """ + """Translate call-site options to httpx.Client.request kwargs; unknown kwargs are dropped.""" kwargs: dict[str, Any] = {} passthrough = ('params', 'json', 'data', 'content', 'files', 'cookies') for key in passthrough: @@ -456,10 +397,7 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool | None: return None def __del__(self) -> None: - # Match ``RequestsWrapper.__del__`` — narrow to ``AttributeError`` so - # genuine httpx-close failures still surface during teardown. - # ``AttributeError`` fires when ``__init__`` raised before ``_client`` - # was assigned and Python still calls ``__del__``. + # AttributeError fires when __init__ raised before _client was assigned. try: self.close() except AttributeError: diff --git a/datadog_checks_base/tests/base/utils/http_httpx/conftest.py b/datadog_checks_base/tests/base/utils/http_httpx/conftest.py index 7b507436d09c6..00e57ff53d146 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/conftest.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/conftest.py @@ -9,8 +9,6 @@ @pytest.fixture def make_transport() -> Callable[[Callable[[httpx.Request], httpx.Response]], httpx.MockTransport]: - """Returns a factory for building httpx.MockTransport from a handler callable.""" - def _factory(handler): return httpx.MockTransport(handler) @@ -19,8 +17,6 @@ def _factory(handler): @pytest.fixture def echo_transport() -> httpx.MockTransport: - """Mock transport that echoes the request method, URL, headers, and body as JSON.""" - def handler(request: httpx.Request) -> httpx.Response: body = b'' try: @@ -40,8 +36,6 @@ def handler(request: httpx.Request) -> httpx.Response: @pytest.fixture def status_transport_factory() -> Callable[[int, bytes | str], httpx.MockTransport]: - """Builds a transport that returns a fixed status code and body.""" - def _factory(status_code: int, body: bytes | str = b''): def handler(_request: httpx.Request) -> httpx.Response: if isinstance(body, str): @@ -66,8 +60,6 @@ def handler(_request: httpx.Request) -> httpx.Response: @pytest.fixture def raising_transport_factory() -> Callable[[Exception], httpx.MockTransport]: - """Builds a transport that raises a given exception when invoked.""" - def _factory(exc: Exception): def handler(_request: httpx.Request) -> httpx.Response: raise exc @@ -85,7 +77,6 @@ def captured_requests() -> list[httpx.Request]: @pytest.fixture def capturing_transport(captured_requests: list[httpx.Request]) -> httpx.MockTransport: def handler(request: httpx.Request) -> httpx.Response: - # Read the body so tests can inspect it. _ = request.content captured_requests.append(request) return httpx.Response(200, json={'ok': True}) @@ -94,7 +85,6 @@ def handler(request: httpx.Request) -> httpx.Response: def parse_basic_auth(header_value: str) -> tuple[str, str]: - """Decode a Basic auth header value into (user, pass).""" import base64 scheme, _, b64 = header_value.partition(' ') diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py index 188de48939d91..0ee8c24d74772 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py @@ -21,7 +21,6 @@ def test_context_manager(capturing_transport): def test_single_client_reused_across_requests(capturing_transport, captured_requests): - """Phase 2 MVP D3: the wrapper holds one ``httpx.Client`` across all requests.""" http = HTTPXWrapper({}, {}, transport=capturing_transport) client_before_requests = http._client http.get('http://example.test/a') @@ -32,8 +31,6 @@ def test_single_client_reused_across_requests(capturing_transport, captured_requ def test_module_import_fails_without_httpx(monkeypatch): - """Per D4: importing http_httpx without httpx installed raises a clean ImportError.""" - # Force a re-import with httpx missing from sys.modules monkeypatch.setitem(sys.modules, 'httpx', None) monkeypatch.delitem(sys.modules, 'datadog_checks.base.utils.http_httpx', raising=False) with pytest.raises(ImportError): @@ -41,11 +38,6 @@ def test_module_import_fails_without_httpx(monkeypatch): def test_agentcheck_http_dispatch_returns_httpx_wrapper(): - """Mirror of test_http.py::test_activate for the use_httpx=True path. - - The default-path counterpart (``use_httpx`` absent → ``RequestsWrapper``) is - already covered by ``tests/base/utils/http/test_http.py::TestAttribute::test_activate``. - """ from datadog_checks.base import AgentCheck check = AgentCheck('test', {}, [{'use_httpx': True}]) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py b/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py index 90326944c68d6..713eaab7edf96 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py @@ -6,7 +6,6 @@ from datadog_checks.base.utils.http_exceptions import HTTPStatusError from datadog_checks.base.utils.http_httpx import HTTPXWrapper -METHODS = ['get', 'post', 'put', 'delete', 'head', 'patch', 'options_method'] HTTP_VERBS = { 'get': 'GET', 'post': 'POST', @@ -18,19 +17,19 @@ } -@pytest.mark.parametrize('method', METHODS) -def test_method_happy_path(method, captured_requests, capturing_transport): +@pytest.mark.parametrize('method,verb', HTTP_VERBS.items()) +def test_method_happy_path(method, verb, captured_requests, capturing_transport): http = HTTPXWrapper({}, {}, transport=capturing_transport) fn = getattr(http, method) response = fn('http://example.test/path', headers={'X-Test': '1'}) assert response.status_code == 200 assert len(captured_requests) == 1 - assert captured_requests[0].method == HTTP_VERBS[method] + assert captured_requests[0].method == verb assert str(captured_requests[0].url) == 'http://example.test/path' -@pytest.mark.parametrize('method', METHODS) +@pytest.mark.parametrize('method', HTTP_VERBS) def test_method_5xx_does_not_raise_unless_asked(method, status_transport_factory): transport = status_transport_factory(500, b'oops') http = HTTPXWrapper({}, {}, transport=transport) @@ -39,7 +38,7 @@ def test_method_5xx_does_not_raise_unless_asked(method, status_transport_factory assert response.status_code == 500 -@pytest.mark.parametrize('method', METHODS) +@pytest.mark.parametrize('method', HTTP_VERBS) def test_method_raise_for_status_propagates(method, status_transport_factory): transport = status_transport_factory(503, b'server unavailable') http = HTTPXWrapper({}, {}, transport=transport) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py index c57c1afd6c8aa..d57b428405221 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py @@ -109,13 +109,10 @@ def test_response_encoding_default_is_utf8(status_transport_factory): transport = status_transport_factory(200, b'') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/') - # httpx defaults to utf-8 when no charset is signalled. Pin the exact value - # so a future httpx change that returns ``None`` here surfaces immediately. assert response.encoding == 'utf-8' def test_response_encoding_setter_propagates_to_inner_response(status_transport_factory): - """OM v2 scraper does ``response.encoding = 'utf-8'`` after the request.""" transport = status_transport_factory(200, b'') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/') @@ -148,20 +145,11 @@ def test_response_elapsed(status_transport_factory): def test_response_elapsed_returns_zero_on_runtime_error(status_transport_factory): - """Cover the RuntimeError fallback in HTTPXResponseAdapter.elapsed. - - httpx 0.28 raises ``RuntimeError`` from ``.elapsed`` until the bound stream's - ``close()`` has finalized the timer. When MockTransport bypasses that path - by serving buffered content, the adapter should return ``timedelta(0)`` so - callers never see the RuntimeError. - """ from datetime import timedelta transport = status_transport_factory(200, b'hello') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/') - # Forge the MockTransport quirk explicitly: if ``_elapsed`` was set, drop it - # so the property has to take the except branch. if hasattr(response._response, '_elapsed'): delattr(response._response, '_elapsed') assert response.elapsed == timedelta(0) @@ -184,7 +172,6 @@ def test_response_ok_property(status_transport_factory, status_code, expected_ok def test_response_reason_from_httpx_response(status_transport_factory): - """``reason`` reads from the underlying ``httpx.Response.reason_phrase``.""" transport = status_transport_factory(200, b'') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/') @@ -192,7 +179,6 @@ def test_response_reason_from_httpx_response(status_transport_factory): def test_response_reason_falls_back_when_reason_phrase_missing(): - """Mock fixtures expose ``.reason``, not ``.reason_phrase`` — the adapter handles that.""" from datadog_checks.base.utils.http_httpx import HTTPXResponseAdapter class _FakeResponseExposingReason: diff --git a/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py b/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py index 6fda7a5deff30..03f6aeacedf29 100644 --- a/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py +++ b/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py @@ -297,28 +297,13 @@ def mock_response(): @pytest.fixture def mock_http_response(mocker, mock_response): - """Patch the default HTTP entry-point to return a ``MockHTTPResponse``. - - Patches ``requests.Session.get`` by default. When the caller does not - override ``method``, also patches ``httpx.Client.request`` so that checks - opting into the Phase 2 HTTPXWrapper via ``use_httpx=true`` are intercepted - by the same fixture. - - Note: passing an explicit ``method=`` skips the httpx companion patch. This - is intentional — a custom ``method`` already names the exact target. If a - check under ``use_httpx=true`` needs both, patch the httpx target with its - own ``method=`` call after. - """ + """Patch the default HTTP entry-point (and httpx.Client.request) to return a MockHTTPResponse.""" def _patch(*args, **kwargs): method = kwargs.pop('method', _DEFAULT_MOCK_METHOD) response = mock_response(*args, **kwargs) primary = mocker.patch(method, return_value=response) if method == _DEFAULT_MOCK_METHOD: - # ``ImportError`` covers the case where httpx is not installed in - # the test env (default RequestsWrapper-only consumers). Anything - # narrower (like a missing ``Client.request``) is a genuine bug we - # want to surface, not swallow. try: mocker.patch('httpx.Client.request', return_value=response) except ImportError: diff --git a/kuma/tests/conftest.py b/kuma/tests/conftest.py index a3a5e29ccaa0f..63ffe21cb61d1 100644 --- a/kuma/tests/conftest.py +++ b/kuma/tests/conftest.py @@ -73,10 +73,7 @@ def dd_environment(dd_save_state): metrics_endpoint = f'http://{kuma_metrics_url}:{kuma_metrics_port}/metrics' - # Phase 2 MVP POC opt-in: kuma is one of two integrations exercising - # HTTPXWrapper end-to-end. Setting use_httpx here (rather than only - # in the `instance` fixture) makes the flag flow into the Agent - # process during e2e runs that read from dd_environment. + # Phase 2 MVP POC opt-in: exercise HTTPXWrapper end-to-end. env_instance = {'openmetrics_endpoint': metrics_endpoint, 'use_httpx': True} dd_save_state("kuma_instance", env_instance) From 0d65e5acce4f7456cec62096bd505ec1f56092ec Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Tue, 26 May 2026 14:57:24 -0400 Subject: [PATCH 06/19] phase2 mvp: rephrase kong/kuma httpx opt-in comments --- kong/tests/conftest.py | 2 +- kong/tests/test_unit.py | 4 ++-- kuma/tests/conftest.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kong/tests/conftest.py b/kong/tests/conftest.py index 8cf8152cd8e93..4c00a120bc60e 100644 --- a/kong/tests/conftest.py +++ b/kong/tests/conftest.py @@ -26,5 +26,5 @@ def dd_environment(): @pytest.fixture def instance_openmetrics_v2(): - # Phase 2 MVP POC opt-in (see RFC 2026-02-11): exercise HTTPXWrapper end-to-end. + # kong is one of the first integrations migrated to the httpx-backed HTTP client. return {**common.openmetrics_instance, 'use_httpx': True} diff --git a/kong/tests/test_unit.py b/kong/tests/test_unit.py index f467fce5c94f4..91c925c8939ef 100644 --- a/kong/tests/test_unit.py +++ b/kong/tests/test_unit.py @@ -57,7 +57,7 @@ def test_check_v3(aggregator, dd_run_check, mock_http_response): instance = { 'openmetrics_endpoint': METRICS_URL, 'extra_metrics': [{'kong_memory_workers_lua_vms_bytes': 'memory.workers.lua.vms.bytes'}], - # Phase 2 MVP POC opt-in (see RFC 2026-02-11): exercise HTTPXWrapper end-to-end. + # kong is one of the first integrations migrated to the httpx-backed HTTP client. 'use_httpx': True, } @@ -81,7 +81,7 @@ def test_check(aggregator, dd_run_check, mock_http_response): instance = { 'openmetrics_endpoint': METRICS_URL, 'extra_metrics': [{'kong_memory_workers_lua_vms_bytes': 'memory.workers.lua.vms.bytes'}], - # Phase 2 MVP POC opt-in (see RFC 2026-02-11): exercise HTTPXWrapper end-to-end. + # kong is one of the first integrations migrated to the httpx-backed HTTP client. 'use_httpx': True, } check = Kong('kong', {}, [instance]) diff --git a/kuma/tests/conftest.py b/kuma/tests/conftest.py index 63ffe21cb61d1..b484746723c9a 100644 --- a/kuma/tests/conftest.py +++ b/kuma/tests/conftest.py @@ -73,7 +73,7 @@ def dd_environment(dd_save_state): metrics_endpoint = f'http://{kuma_metrics_url}:{kuma_metrics_port}/metrics' - # Phase 2 MVP POC opt-in: exercise HTTPXWrapper end-to-end. + # kuma is one of the first integrations migrated to the httpx-backed HTTP client. env_instance = {'openmetrics_endpoint': metrics_endpoint, 'use_httpx': True} dd_save_state("kuma_instance", env_instance) From 5dc83267e19d61b6d5c940f5e1318c4804219329 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Tue, 26 May 2026 15:44:18 -0400 Subject: [PATCH 07/19] phase2 mvp: align httpx tests with teammate norms Cut stdlib-passthrough tests, parametrize duplicates, remove private-attr asserts, fold test_tls.py into test_config.py, move parse_basic_auth to common.py. 87 to 52 tests, same wrapper coverage. --- .../tests/base/utils/http_httpx/common.py | 12 ++ .../tests/base/utils/http_httpx/conftest.py | 48 ------ .../base/utils/http_httpx/test_auth_basic.py | 12 +- .../base/utils/http_httpx/test_config.py | 41 +++-- .../base/utils/http_httpx/test_exceptions.py | 82 ++-------- .../base/utils/http_httpx/test_lifecycle.py | 10 -- .../base/utils/http_httpx/test_methods.py | 32 +--- .../base/utils/http_httpx/test_response.py | 151 +++--------------- .../tests/base/utils/http_httpx/test_tls.py | 38 ----- 9 files changed, 79 insertions(+), 347 deletions(-) create mode 100644 datadog_checks_base/tests/base/utils/http_httpx/common.py delete mode 100644 datadog_checks_base/tests/base/utils/http_httpx/test_tls.py diff --git a/datadog_checks_base/tests/base/utils/http_httpx/common.py b/datadog_checks_base/tests/base/utils/http_httpx/common.py new file mode 100644 index 0000000000000..d63c9b58cb84a --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http_httpx/common.py @@ -0,0 +1,12 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import base64 + + +def parse_basic_auth(header_value: str) -> tuple[str, str]: + scheme, _, b64 = header_value.partition(' ') + assert scheme.lower() == 'basic' + user_pass = base64.b64decode(b64).decode('utf-8') + user, _, password = user_pass.partition(':') + return user, password diff --git a/datadog_checks_base/tests/base/utils/http_httpx/conftest.py b/datadog_checks_base/tests/base/utils/http_httpx/conftest.py index 00e57ff53d146..7d501050b5c69 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/conftest.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/conftest.py @@ -7,33 +7,6 @@ import pytest -@pytest.fixture -def make_transport() -> Callable[[Callable[[httpx.Request], httpx.Response]], httpx.MockTransport]: - def _factory(handler): - return httpx.MockTransport(handler) - - return _factory - - -@pytest.fixture -def echo_transport() -> httpx.MockTransport: - def handler(request: httpx.Request) -> httpx.Response: - body = b'' - try: - body = request.content - except httpx.RequestNotRead: - pass - payload = { - 'method': request.method, - 'url': str(request.url), - 'headers': dict(request.headers), - 'body': body.decode('utf-8') if body else '', - } - return httpx.Response(200, json=payload) - - return httpx.MockTransport(handler) - - @pytest.fixture def status_transport_factory() -> Callable[[int, bytes | str], httpx.MockTransport]: def _factory(status_code: int, body: bytes | str = b''): @@ -47,17 +20,6 @@ def handler(_request: httpx.Request) -> httpx.Response: return _factory -@pytest.fixture -def json_transport_factory() -> Callable[[dict, int], httpx.MockTransport]: - def _factory(payload: dict, status_code: int = 200): - def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response(status_code, json=payload) - - return httpx.MockTransport(handler) - - return _factory - - @pytest.fixture def raising_transport_factory() -> Callable[[Exception], httpx.MockTransport]: def _factory(exc: Exception): @@ -82,13 +44,3 @@ def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(200, json={'ok': True}) return httpx.MockTransport(handler) - - -def parse_basic_auth(header_value: str) -> tuple[str, str]: - import base64 - - scheme, _, b64 = header_value.partition(' ') - assert scheme.lower() == 'basic' - user_pass = base64.b64decode(b64).decode('utf-8') - user, _, password = user_pass.partition(':') - return user, password diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py b/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py index 73c397de67464..8c776c51b4d50 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py @@ -3,7 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from datadog_checks.base.utils.http_httpx import HTTPXWrapper -from .conftest import parse_basic_auth +from .common import parse_basic_auth def test_basic_auth_sent_on_request(capturing_transport, captured_requests): @@ -13,7 +13,6 @@ def test_basic_auth_sent_on_request(capturing_transport, captured_requests): transport=capturing_transport, ) http.get('http://example.test/') - assert len(captured_requests) == 1 user, password = parse_basic_auth(captured_requests[0].headers['authorization']) assert user == 'alice' assert password == 'secret' @@ -35,12 +34,3 @@ def test_basic_auth_skipped_when_only_password_set(capturing_transport, captured http = HTTPXWrapper({'password': 'secret'}, {}, transport=capturing_transport) http.get('http://example.test/') assert 'authorization' not in captured_requests[0].headers - - -def test_basic_auth_options_exposes_auth(capturing_transport): - http = HTTPXWrapper( - {'username': 'alice', 'password': 'secret'}, - {}, - transport=capturing_transport, - ) - assert http.options['auth'] is not None diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_config.py b/datadog_checks_base/tests/base/utils/http_httpx/test_config.py index 922d97d3c91ef..92757d83a4003 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_config.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_config.py @@ -4,17 +4,6 @@ from datadog_checks.base.utils.http_httpx import HTTPXWrapper -def test_options_dict_shape(capturing_transport): - http = HTTPXWrapper({}, {}, transport=capturing_transport) - options = http.options - assert 'auth' in options - assert 'cert' in options - assert 'headers' in options - assert 'timeout' in options - assert 'verify' in options - assert 'allow_redirects' in options - - def test_default_headers_include_user_agent(capturing_transport): http = HTTPXWrapper({}, {}, transport=capturing_transport) assert any(key.lower() == 'user-agent' for key in http.options['headers']) @@ -38,12 +27,6 @@ def test_per_request_headers_merge_into_request(capturing_transport, captured_re assert captured_requests[0].headers['x-per-request'] == 'yes' -def test_timeout_default(capturing_transport): - http = HTTPXWrapper({}, {}, transport=capturing_transport) - timeout = http.options['timeout'] - assert isinstance(timeout, tuple) and len(timeout) == 2 - - def test_timeout_from_instance(capturing_transport): http = HTTPXWrapper({'timeout': 25}, {}, transport=capturing_transport) connect, read = http.options['timeout'] @@ -68,6 +51,30 @@ def test_verify_false_when_tls_verify_off(capturing_transport): assert http.options['verify'] is False +def test_tls_ca_cert_uses_path(capturing_transport): + http = HTTPXWrapper({'tls_ca_cert': '/etc/ssl/ca.pem'}, {}, transport=capturing_transport) + assert http.options['verify'] == '/etc/ssl/ca.pem' + + +def test_tls_client_cert_string(capturing_transport): + http = HTTPXWrapper({'tls_cert': '/etc/ssl/client.pem'}, {}, transport=capturing_transport) + assert http.options['cert'] == '/etc/ssl/client.pem' + + +def test_tls_client_cert_with_key(capturing_transport): + http = HTTPXWrapper( + {'tls_cert': '/etc/ssl/client.pem', 'tls_private_key': '/etc/ssl/client.key'}, + {}, + transport=capturing_transport, + ) + assert http.options['cert'] == ('/etc/ssl/client.pem', '/etc/ssl/client.key') + + +def test_tls_no_cert_when_not_configured(capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + assert http.options['cert'] is None + + def test_get_header_case_insensitive(capturing_transport): http = HTTPXWrapper({'extra_headers': {'X-Foo': 'bar'}}, {}, transport=capturing_transport) assert http.get_header('x-foo') == 'bar' diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py b/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py index f6367e5ebfba5..53a1afdcd3e36 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py @@ -4,75 +4,23 @@ import httpx import pytest -from datadog_checks.base.utils.http_exceptions import ( - HTTPConnectionError, - HTTPError, - HTTPStatusError, - HTTPTimeoutError, -) +from datadog_checks.base.utils.http_exceptions import HTTPConnectionError, HTTPError, HTTPTimeoutError from datadog_checks.base.utils.http_httpx import HTTPXWrapper -def test_connect_timeout_maps_to_timeout_error(raising_transport_factory): - transport = raising_transport_factory(httpx.ConnectTimeout('boom')) - http = HTTPXWrapper({}, {}, transport=transport) - with pytest.raises(HTTPTimeoutError): - http.get('http://example.test/') - - -def test_read_timeout_maps_to_timeout_error(raising_transport_factory): - transport = raising_transport_factory(httpx.ReadTimeout('slow')) - http = HTTPXWrapper({}, {}, transport=transport) - with pytest.raises(HTTPTimeoutError): - http.get('http://example.test/') - - -def test_pool_timeout_maps_to_timeout_error(raising_transport_factory): - transport = raising_transport_factory(httpx.PoolTimeout('pool')) - http = HTTPXWrapper({}, {}, transport=transport) - with pytest.raises(HTTPTimeoutError): - http.get('http://example.test/') - - -def test_connect_error_maps_to_connection_error(raising_transport_factory): - transport = raising_transport_factory(httpx.ConnectError('refused')) - http = HTTPXWrapper({}, {}, transport=transport) - with pytest.raises(HTTPConnectionError): - http.get('http://example.test/') - - -def test_protocol_error_maps_to_http_error(raising_transport_factory): - transport = raising_transport_factory(httpx.LocalProtocolError('bad')) - http = HTTPXWrapper({}, {}, transport=transport) - with pytest.raises(HTTPError): - http.get('http://example.test/') - - -def test_request_error_maps_to_http_error(raising_transport_factory): - transport = raising_transport_factory(httpx.RequestError('generic')) +@pytest.mark.parametrize( + 'raised,expected', + [ + pytest.param(httpx.ConnectTimeout('boom'), HTTPTimeoutError, id='connect-timeout'), + pytest.param(httpx.ReadTimeout('slow'), HTTPTimeoutError, id='read-timeout'), + pytest.param(httpx.PoolTimeout('pool'), HTTPTimeoutError, id='pool-timeout'), + pytest.param(httpx.ConnectError('refused'), HTTPConnectionError, id='connect-error'), + pytest.param(httpx.LocalProtocolError('bad'), HTTPError, id='local-protocol-error'), + pytest.param(httpx.RequestError('generic'), HTTPError, id='request-error'), + ], +) +def test_request_exception_mapping(raising_transport_factory, raised, expected): + transport = raising_transport_factory(raised) http = HTTPXWrapper({}, {}, transport=transport) - with pytest.raises(HTTPError): + with pytest.raises(expected): http.get('http://example.test/') - - -def test_raise_for_status_4xx_maps_to_status_error(status_transport_factory): - transport = status_transport_factory(404, b'not found') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - with pytest.raises(HTTPStatusError): - response.raise_for_status() - - -def test_raise_for_status_5xx_maps_to_status_error(status_transport_factory): - transport = status_transport_factory(502, b'bad gateway') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - with pytest.raises(HTTPStatusError): - response.raise_for_status() - - -def test_raise_for_status_3xx_does_not_raise(status_transport_factory): - transport = status_transport_factory(301, b'') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - response.raise_for_status() diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py index 0ee8c24d74772..d14e68a6c97cf 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py @@ -20,16 +20,6 @@ def test_context_manager(capturing_transport): assert response.status_code == 200 -def test_single_client_reused_across_requests(capturing_transport, captured_requests): - http = HTTPXWrapper({}, {}, transport=capturing_transport) - client_before_requests = http._client - http.get('http://example.test/a') - http.get('http://example.test/b') - http.post('http://example.test/c', json={'x': 1}) - assert http._client is client_before_requests - assert len(captured_requests) == 3 - - def test_module_import_fails_without_httpx(monkeypatch): monkeypatch.setitem(sys.modules, 'httpx', None) monkeypatch.delitem(sys.modules, 'datadog_checks.base.utils.http_httpx', raising=False) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py b/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py index 713eaab7edf96..115f5711b71f6 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py @@ -3,7 +3,6 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import pytest -from datadog_checks.base.utils.http_exceptions import HTTPStatusError from datadog_checks.base.utils.http_httpx import HTTPXWrapper HTTP_VERBS = { @@ -18,48 +17,19 @@ @pytest.mark.parametrize('method,verb', HTTP_VERBS.items()) -def test_method_happy_path(method, verb, captured_requests, capturing_transport): +def test_method_dispatches_with_correct_verb(method, verb, captured_requests, capturing_transport): http = HTTPXWrapper({}, {}, transport=capturing_transport) fn = getattr(http, method) response = fn('http://example.test/path', headers={'X-Test': '1'}) assert response.status_code == 200 - assert len(captured_requests) == 1 assert captured_requests[0].method == verb - assert str(captured_requests[0].url) == 'http://example.test/path' - - -@pytest.mark.parametrize('method', HTTP_VERBS) -def test_method_5xx_does_not_raise_unless_asked(method, status_transport_factory): - transport = status_transport_factory(500, b'oops') - http = HTTPXWrapper({}, {}, transport=transport) - fn = getattr(http, method) - response = fn('http://example.test/path') - assert response.status_code == 500 - - -@pytest.mark.parametrize('method', HTTP_VERBS) -def test_method_raise_for_status_propagates(method, status_transport_factory): - transport = status_transport_factory(503, b'server unavailable') - http = HTTPXWrapper({}, {}, transport=transport) - fn = getattr(http, method) - response = fn('http://example.test/path') - with pytest.raises(HTTPStatusError): - response.raise_for_status() def test_post_json_body_is_serialized(capturing_transport, captured_requests): http = HTTPXWrapper({}, {}, transport=capturing_transport) http.post('http://example.test/path', json={'a': 1, 'b': 'two'}) - assert len(captured_requests) == 1 req = captured_requests[0] assert req.headers['content-type'] == 'application/json' assert b'"a":' in req.content assert b'"b":' in req.content - - -def test_get_query_params_forwarded(capturing_transport, captured_requests): - http = HTTPXWrapper({}, {}, transport=capturing_transport) - http.get('http://example.test/path', params={'foo': 'bar', 'baz': '1'}) - assert captured_requests[0].url.params['foo'] == 'bar' - assert captured_requests[0].url.params['baz'] == '1' diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py index d57b428405221..45c0385a76a74 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py @@ -1,70 +1,24 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import httpx +from datetime import timedelta + import pytest from datadog_checks.base.utils.http_exceptions import HTTPStatusError -from datadog_checks.base.utils.http_httpx import HTTPXWrapper - - -def test_response_content_bytes(status_transport_factory): - transport = status_transport_factory(200, b'hello world') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - assert response.content == b'hello world' - - -def test_response_text(status_transport_factory): - transport = status_transport_factory(200, 'hello') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - assert response.text == 'hello' - - -def test_response_status_code(status_transport_factory): - transport = status_transport_factory(204, b'') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - assert response.status_code == 204 - - -def test_response_json(json_transport_factory): - transport = json_transport_factory({'a': 1, 'b': [1, 2, 3]}) - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - assert response.json() == {'a': 1, 'b': [1, 2, 3]} +from datadog_checks.base.utils.http_httpx import HTTPXResponseAdapter, HTTPXWrapper -def test_response_headers_case_insensitive(): - def handler(_request): - return httpx.Response(200, headers={'X-Custom': 'foo', 'Content-Type': 'text/plain'}) - - transport = httpx.MockTransport(handler) - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - assert response.headers['x-custom'] == 'foo' - assert response.headers['X-CUSTOM'] == 'foo' - assert response.headers['content-type'] == 'text/plain' - - -def test_response_raise_for_status_4xx(status_transport_factory): - transport = status_transport_factory(404, b'') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - with pytest.raises(HTTPStatusError): - response.raise_for_status() - - -def test_response_raise_for_status_5xx(status_transport_factory): - transport = status_transport_factory(500, b'') +@pytest.mark.parametrize('status_code', [404, 500]) +def test_response_raise_for_status_raises_on_error_codes(status_transport_factory, status_code): + transport = status_transport_factory(status_code, b'') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/') with pytest.raises(HTTPStatusError): response.raise_for_status() -def test_response_iter_content(status_transport_factory): +def test_response_iter_content_bytes(status_transport_factory): transport = status_transport_factory(200, b'abcdef') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/') @@ -72,95 +26,44 @@ def test_response_iter_content(status_transport_factory): assert b''.join(chunks) == b'abcdef' -def test_response_iter_content_chunk_size_none(status_transport_factory): - transport = status_transport_factory(200, b'hello') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - chunks = list(response.iter_content()) - assert chunks == [b'hello'] - - def test_response_iter_content_decode_unicode(status_transport_factory): - transport = status_transport_factory(200, b'hello world') + transport = status_transport_factory(200, b'abcdef') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/') chunks = list(response.iter_content(chunk_size=3, decode_unicode=True)) - assert all(isinstance(c, str) for c in chunks) - assert ''.join(chunks) == 'hello world' + assert ''.join(chunks) == 'abcdef' -def test_response_iter_lines_bytes_default(status_transport_factory): +@pytest.mark.parametrize( + 'decode_unicode,expected', + [ + pytest.param(False, [b'a', b'b', b'c'], id='bytes'), + pytest.param(True, ['a', 'b', 'c'], id='decoded-unicode'), + ], +) +def test_response_iter_lines(status_transport_factory, decode_unicode, expected): transport = status_transport_factory(200, b'a\nb\nc') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/') - lines = list(response.iter_lines()) - assert lines == [b'a', b'b', b'c'] + assert list(response.iter_lines(decode_unicode=decode_unicode)) == expected -def test_response_iter_lines_decode_unicode(status_transport_factory): - transport = status_transport_factory(200, b'a\nb\nc') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - lines = list(response.iter_lines(decode_unicode=True)) - assert lines == ['a', 'b', 'c'] - - -def test_response_encoding_default_is_utf8(status_transport_factory): - transport = status_transport_factory(200, b'') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - assert response.encoding == 'utf-8' - - -def test_response_encoding_setter_propagates_to_inner_response(status_transport_factory): +def test_response_encoding_setter(status_transport_factory): transport = status_transport_factory(200, b'') http = HTTPXWrapper({}, {}, transport=transport) response = http.get('http://example.test/') response.encoding = 'latin-1' assert response.encoding == 'latin-1' - assert response._response.encoding == 'latin-1' - - -def test_response_url(status_transport_factory): - transport = status_transport_factory(200, b'') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/path?x=1') - assert str(response.url) == 'http://example.test/path?x=1' -def test_response_cookies_empty_by_default(status_transport_factory): - transport = status_transport_factory(200, b'') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - assert len(response.cookies) == 0 +def test_response_elapsed_returns_zero_on_runtime_error(): + class _FakeResponse: + @property + def elapsed(self): + raise RuntimeError('not measured') - -def test_response_elapsed(status_transport_factory): - from datetime import timedelta - - transport = status_transport_factory(200, b'') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - assert isinstance(response.elapsed, timedelta) - - -def test_response_elapsed_returns_zero_on_runtime_error(status_transport_factory): - from datetime import timedelta - - transport = status_transport_factory(200, b'hello') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - if hasattr(response._response, '_elapsed'): - delattr(response._response, '_elapsed') - assert response.elapsed == timedelta(0) - - -def test_response_close_marks_inner_response_closed(status_transport_factory): - transport = status_transport_factory(200, b'') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - response.close() - assert response._response.is_closed is True + adapter = HTTPXResponseAdapter(_FakeResponse()) # type: ignore[arg-type] + assert adapter.elapsed == timedelta(0) @pytest.mark.parametrize('status_code,expected_ok', [(200, True), (204, True), (301, True), (400, False), (500, False)]) @@ -179,8 +82,6 @@ def test_response_reason_from_httpx_response(status_transport_factory): def test_response_reason_falls_back_when_reason_phrase_missing(): - from datadog_checks.base.utils.http_httpx import HTTPXResponseAdapter - class _FakeResponseExposingReason: reason = 'Not Found' diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_tls.py b/datadog_checks_base/tests/base/utils/http_httpx/test_tls.py deleted file mode 100644 index 6f57a6ded7ad1..0000000000000 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_tls.py +++ /dev/null @@ -1,38 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.utils.http_httpx import HTTPXWrapper - - -def test_tls_verify_default_true(capturing_transport): - http = HTTPXWrapper({}, {}, transport=capturing_transport) - assert http.options['verify'] is True - - -def test_tls_verify_false(capturing_transport): - http = HTTPXWrapper({'tls_verify': False}, {}, transport=capturing_transport) - assert http.options['verify'] is False - - -def test_tls_ca_cert_uses_path(capturing_transport): - http = HTTPXWrapper({'tls_ca_cert': '/etc/ssl/ca.pem'}, {}, transport=capturing_transport) - assert http.options['verify'] == '/etc/ssl/ca.pem' - - -def test_tls_client_cert_string(capturing_transport): - http = HTTPXWrapper({'tls_cert': '/etc/ssl/client.pem'}, {}, transport=capturing_transport) - assert http.options['cert'] == '/etc/ssl/client.pem' - - -def test_tls_client_cert_with_key(capturing_transport): - http = HTTPXWrapper( - {'tls_cert': '/etc/ssl/client.pem', 'tls_private_key': '/etc/ssl/client.key'}, - {}, - transport=capturing_transport, - ) - assert http.options['cert'] == ('/etc/ssl/client.pem', '/etc/ssl/client.key') - - -def test_tls_no_cert_when_not_configured(capturing_transport): - http = HTTPXWrapper({}, {}, transport=capturing_transport) - assert http.options['cert'] is None From ade5123a362802b85625d9b8b7e7f030790c943b Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Tue, 26 May 2026 16:00:18 -0400 Subject: [PATCH 08/19] phase2 mvp: drop encoding-setter test (httpx passthrough) --- .../tests/base/utils/http_httpx/test_response.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py index 45c0385a76a74..e525e956952c6 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py @@ -48,14 +48,6 @@ def test_response_iter_lines(status_transport_factory, decode_unicode, expected) assert list(response.iter_lines(decode_unicode=decode_unicode)) == expected -def test_response_encoding_setter(status_transport_factory): - transport = status_transport_factory(200, b'') - http = HTTPXWrapper({}, {}, transport=transport) - response = http.get('http://example.test/') - response.encoding = 'latin-1' - assert response.encoding == 'latin-1' - - def test_response_elapsed_returns_zero_on_runtime_error(): class _FakeResponse: @property From f59e1b7beeb05c9db7082a3e890064d4d3b64abb Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 28 May 2026 11:24:22 -0400 Subject: [PATCH 09/19] scope MVP to design validation: revert kong/kuma + mock_http_response changes --- .../datadog_checks/dev/plugin/pytest.py | 17 +++-------------- kong/tests/conftest.py | 3 +-- kong/tests/test_unit.py | 4 ---- kuma/tests/conftest.py | 4 +--- 4 files changed, 5 insertions(+), 23 deletions(-) diff --git a/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py b/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py index 03f6aeacedf29..9e474ba6a2ef0 100644 --- a/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py +++ b/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py @@ -297,20 +297,9 @@ def mock_response(): @pytest.fixture def mock_http_response(mocker, mock_response): - """Patch the default HTTP entry-point (and httpx.Client.request) to return a MockHTTPResponse.""" - - def _patch(*args, **kwargs): - method = kwargs.pop('method', _DEFAULT_MOCK_METHOD) - response = mock_response(*args, **kwargs) - primary = mocker.patch(method, return_value=response) - if method == _DEFAULT_MOCK_METHOD: - try: - mocker.patch('httpx.Client.request', return_value=response) - except ImportError: - pass - return primary - - yield _patch + yield lambda *args, **kwargs: mocker.patch( + kwargs.pop('method', _DEFAULT_MOCK_METHOD), return_value=mock_response(*args, **kwargs) + ) @pytest.fixture diff --git a/kong/tests/conftest.py b/kong/tests/conftest.py index 4c00a120bc60e..762a3bf82e8ab 100644 --- a/kong/tests/conftest.py +++ b/kong/tests/conftest.py @@ -26,5 +26,4 @@ def dd_environment(): @pytest.fixture def instance_openmetrics_v2(): - # kong is one of the first integrations migrated to the httpx-backed HTTP client. - return {**common.openmetrics_instance, 'use_httpx': True} + return common.openmetrics_instance diff --git a/kong/tests/test_unit.py b/kong/tests/test_unit.py index 91c925c8939ef..41dfb02a77171 100644 --- a/kong/tests/test_unit.py +++ b/kong/tests/test_unit.py @@ -57,8 +57,6 @@ def test_check_v3(aggregator, dd_run_check, mock_http_response): instance = { 'openmetrics_endpoint': METRICS_URL, 'extra_metrics': [{'kong_memory_workers_lua_vms_bytes': 'memory.workers.lua.vms.bytes'}], - # kong is one of the first integrations migrated to the httpx-backed HTTP client. - 'use_httpx': True, } check = Kong('kong', {}, [instance]) @@ -81,8 +79,6 @@ def test_check(aggregator, dd_run_check, mock_http_response): instance = { 'openmetrics_endpoint': METRICS_URL, 'extra_metrics': [{'kong_memory_workers_lua_vms_bytes': 'memory.workers.lua.vms.bytes'}], - # kong is one of the first integrations migrated to the httpx-backed HTTP client. - 'use_httpx': True, } check = Kong('kong', {}, [instance]) dd_run_check(check) diff --git a/kuma/tests/conftest.py b/kuma/tests/conftest.py index b484746723c9a..e7f54ba7284fc 100644 --- a/kuma/tests/conftest.py +++ b/kuma/tests/conftest.py @@ -73,8 +73,7 @@ def dd_environment(dd_save_state): metrics_endpoint = f'http://{kuma_metrics_url}:{kuma_metrics_port}/metrics' - # kuma is one of the first integrations migrated to the httpx-backed HTTP client. - env_instance = {'openmetrics_endpoint': metrics_endpoint, 'use_httpx': True} + env_instance = {'openmetrics_endpoint': metrics_endpoint} dd_save_state("kuma_instance", env_instance) @@ -87,6 +86,5 @@ def instance(dd_get_state): 'kuma_instance', default={ 'openmetrics_endpoint': 'http://localhost:5680/metrics', - 'use_httpx': True, }, ) From 424f0bfd184a917802be1c9dff756215bfef65ef Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 28 May 2026 12:22:48 -0400 Subject: [PATCH 10/19] consolidate changelog into 22676.added for feature branch merge --- datadog_checks_base/changelog.d/22676.added | 2 +- datadog_checks_base/changelog.d/23822.added | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 datadog_checks_base/changelog.d/23822.added diff --git a/datadog_checks_base/changelog.d/22676.added b/datadog_checks_base/changelog.d/22676.added index ff63715e3c69d..6e65051afdc9e 100644 --- a/datadog_checks_base/changelog.d/22676.added +++ b/datadog_checks_base/changelog.d/22676.added @@ -1 +1 @@ -Add library-agnostic HTTP mocks/proto/exceptions and migrate intg tests. +Add library-agnostic HTTP mocks/proto/exceptions and migrate intg tests. Add ``HTTPXWrapper``, an httpx-backed HTTP client opt-in via the ``use_httpx`` instance config flag. The default remains the existing requests-based client. diff --git a/datadog_checks_base/changelog.d/23822.added b/datadog_checks_base/changelog.d/23822.added deleted file mode 100644 index 115c23bd94631..0000000000000 --- a/datadog_checks_base/changelog.d/23822.added +++ /dev/null @@ -1 +0,0 @@ -Add HTTPXWrapper, an httpx-backed HTTP client wrapper opt-in via the ``use_httpx`` instance config flag. The default remains the existing requests-based client. From 5d56e6daf7671fff1a74256ced62f3aca5314a01 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 28 May 2026 13:57:19 -0400 Subject: [PATCH 11/19] phase2 mvp: close httpx.Client explicitly in dispatch test --- .../tests/base/utils/http_httpx/test_lifecycle.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py index d14e68a6c97cf..aa63d82c20e56 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py @@ -31,4 +31,6 @@ def test_agentcheck_http_dispatch_returns_httpx_wrapper(): from datadog_checks.base import AgentCheck check = AgentCheck('test', {}, [{'use_httpx': True}]) - assert isinstance(check.http, HTTPXWrapper) + http = check.http + assert isinstance(http, HTTPXWrapper) + http.close() From 313b9dea7ce2a765c57d3f8e48732e71a2a45a4d Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 28 May 2026 16:36:02 -0400 Subject: [PATCH 12/19] phase2 mvp: tighten httpx wrapper kwargs + base_scraper raise --- .../openmetrics/v2/scraper/base_scraper.py | 4 +-- .../datadog_checks/base/utils/http_httpx.py | 34 ++++++++++++++----- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py index 3f12c8e9f9c8a..a3211f6c39405 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py @@ -408,9 +408,9 @@ def stream_connection_lines(self): yield line except (ConnectionError, HTTPConnectionError) as e: if self.ignore_connection_errors: - self.log.warning("OpenMetrics endpoint %s is not accessible", self.endpoint) + self.log.warning("OpenMetrics endpoint %s is not accessible: %s", self.endpoint, e) else: - raise e + raise def filter_connection_lines(self, line_streamer): """ diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py index 2f412f360a0ce..ae563aacc1485 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py @@ -4,9 +4,9 @@ from __future__ import annotations import logging -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from datetime import timedelta -from typing import Any, Iterator +from typing import Any import httpx @@ -43,6 +43,22 @@ DEFAULT_REMAPPED_FIELDS: dict[str, dict[str, Any]] = {} +REQUEST_KWARGS = frozenset( + { + 'params', + 'json', + 'data', + 'content', + 'files', + 'cookies', + 'headers', + 'extra_headers', + 'timeout', + 'follow_redirects', + 'allow_redirects', + } +) + def _build_basic_auth(config: dict[str, Any]) -> httpx.BasicAuth | None: if config['username'] is not None and config['password'] is not None: @@ -234,7 +250,7 @@ def __init__( timeout = _build_timeout(config) allow_redirects = is_affirmative(config['allow_redirects']) - # `proxies` is None for shape-parity with RequestsWrapper.options; proxy wiring is Phase 3. + # TODO(httpx-migration): proxies wiring deferred to Phase 3. self.options: dict[str, Any] = { 'auth': auth, 'cert': cert, @@ -305,6 +321,7 @@ def get_header(self, name: str, default: str | None = None) -> str | None: return default def set_header(self, name: str, value: str) -> None: + # Mirror into both stores: options['headers'] is the public shape, _client.headers is what httpx sends. for key in list(self.options['headers']): if key.lower() == name.lower(): self.options['headers'][key] = value @@ -346,7 +363,10 @@ def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPXRespo return HTTPXResponseAdapter(response) def _build_request_kwargs(self, options: dict[str, Any]) -> dict[str, Any]: - """Translate call-site options to httpx.Client.request kwargs; unknown kwargs are dropped.""" + """Translate call-site options to httpx.Client.request kwargs.""" + unknown = set(options) - REQUEST_KWARGS + if unknown: + raise TypeError(f"HTTPXWrapper does not support per-request kwargs: {sorted(unknown)}") kwargs: dict[str, Any] = {} passthrough = ('params', 'json', 'data', 'content', 'files', 'cookies') for key in passthrough: @@ -397,8 +417,4 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool | None: return None def __del__(self) -> None: - # AttributeError fires when __init__ raised before _client was assigned. - try: - self.close() - except AttributeError: - pass + self.close() From ac4a51849dc9abbdab784f21b4f737b5090579ef Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 28 May 2026 16:39:48 -0400 Subject: [PATCH 13/19] phase2 mvp: stream httpx wrapper body via client.send(stream=True) --- .../datadog_checks/base/utils/http_httpx.py | 29 ++++++++++--------- .../base/utils/http_httpx/test_response.py | 15 ++++++++++ 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py index ae563aacc1485..66b0ec1a85160 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py @@ -122,10 +122,11 @@ def status_code(self) -> int: @property def content(self) -> bytes: - return self._response.content + return self._response.read() @property def text(self) -> str: + self._response.read() return self._response.text @property @@ -167,6 +168,7 @@ def elapsed(self) -> timedelta: return timedelta(0) def json(self, **kwargs: Any) -> Any: + self._response.read() return self._response.json(**kwargs) def raise_for_status(self) -> None: @@ -182,12 +184,13 @@ def close(self) -> None: self._response.close() def iter_content(self, chunk_size: int | None = None, decode_unicode: bool = False) -> Iterator[bytes | str]: - content = self._response.content if chunk_size is None: + content = self._response.read() + if not content: + return yield content.decode('utf-8') if decode_unicode else content return - for i in range(0, len(content), max(chunk_size, 1)): - chunk = content[i : i + chunk_size] + for chunk in self._response.iter_bytes(chunk_size=chunk_size): yield chunk.decode('utf-8') if decode_unicode else chunk def iter_lines( @@ -196,15 +199,11 @@ def iter_lines( decode_unicode: bool = False, delimiter: bytes | str | None = None, ) -> Iterator[bytes | str]: - if isinstance(delimiter, str): - delimiter = delimiter.encode('utf-8') - sep = delimiter or b'\n' - content = self._response.content - lines = content.split(sep) - if lines and not lines[-1]: - lines.pop() - for line in lines: - yield line.decode('utf-8') if decode_unicode else line + if delimiter is not None: + raise NotImplementedError("HTTPXResponseAdapter.iter_lines does not support custom delimiters") + for line in self._response.iter_lines(): + # httpx.Response.iter_lines yields str; encode when caller wants bytes. + yield line if decode_unicode else line.encode('utf-8') def __enter__(self) -> 'HTTPXResponseAdapter': return self @@ -356,8 +355,10 @@ def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPXRespo self.logger.debug('Sending %s request to %s', method, url) request_kwargs = self._build_request_kwargs(options) + follow_redirects = request_kwargs.pop('follow_redirects', httpx.USE_CLIENT_DEFAULT) try: - response = self._client.request(method, url, **request_kwargs) + request = self._client.build_request(method, url, **request_kwargs) + response = self._client.send(request, stream=True, follow_redirects=follow_redirects) except httpx.HTTPError as exc: raise _map_httpx_exception(exc) from exc return HTTPXResponseAdapter(response) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py index e525e956952c6..f0fa1114a1421 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py @@ -48,6 +48,21 @@ def test_response_iter_lines(status_transport_factory, decode_unicode, expected) assert list(response.iter_lines(decode_unicode=decode_unicode)) == expected +def test_response_iter_content_empty_body_yields_nothing(status_transport_factory): + transport = status_transport_factory(200, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert list(response.iter_content()) == [] + + +def test_response_iter_lines_rejects_delimiter(status_transport_factory): + transport = status_transport_factory(200, b'a\nb\n') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + with pytest.raises(NotImplementedError): + list(response.iter_lines(delimiter=b'|')) + + def test_response_elapsed_returns_zero_on_runtime_error(): class _FakeResponse: @property From d92d80bf0f01f002314d5dfa074ec1c11c5c92f2 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 28 May 2026 16:41:32 -0400 Subject: [PATCH 14/19] phase2 mvp: address review iteration 2 test feedback --- .../tests/base/utils/http_httpx/conftest.py | 2 +- .../base/utils/http_httpx/test_config.py | 24 ++++++-- .../base/utils/http_httpx/test_exceptions.py | 10 +++- .../base/utils/http_httpx/test_methods.py | 5 +- .../base/utils/http_httpx/test_response.py | 56 +++++++++++++++++++ 5 files changed, 86 insertions(+), 11 deletions(-) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/conftest.py b/datadog_checks_base/tests/base/utils/http_httpx/conftest.py index 7d501050b5c69..552ee607e0eb9 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/conftest.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/conftest.py @@ -1,7 +1,7 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from typing import Callable +from collections.abc import Callable import httpx import pytest diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_config.py b/datadog_checks_base/tests/base/utils/http_httpx/test_config.py index 92757d83a4003..658e8967bf134 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_config.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_config.py @@ -1,6 +1,8 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + from datadog_checks.base.utils.http_httpx import HTTPXWrapper @@ -75,12 +77,18 @@ def test_tls_no_cert_when_not_configured(capturing_transport): assert http.options['cert'] is None -def test_get_header_case_insensitive(capturing_transport): +@pytest.mark.parametrize( + 'lookup_name,default,expected', + [ + pytest.param('x-foo', None, 'bar', id='lowercase-lookup'), + pytest.param('X-FOO', None, 'bar', id='uppercase-lookup'), + pytest.param('missing', None, None, id='missing-no-default'), + pytest.param('missing', 'fallback', 'fallback', id='missing-with-default'), + ], +) +def test_get_header(capturing_transport, lookup_name, default, expected): http = HTTPXWrapper({'extra_headers': {'X-Foo': 'bar'}}, {}, transport=capturing_transport) - assert http.get_header('x-foo') == 'bar' - assert http.get_header('X-FOO') == 'bar' - assert http.get_header('missing') is None - assert http.get_header('missing', default='fallback') == 'fallback' + assert http.get_header(lookup_name, default=default) == expected def test_set_header_overrides_existing(capturing_transport): @@ -93,3 +101,9 @@ def test_remapper_renames_field(capturing_transport): remapper = {'ssl_validation': {'name': 'tls_verify'}} http = HTTPXWrapper({'ssl_validation': False}, {}, remapper=remapper, transport=capturing_transport) assert http.options['verify'] is False + + +def test_request_rejects_unknown_kwarg(capturing_transport): + http = HTTPXWrapper({}, {}, transport=capturing_transport) + with pytest.raises(TypeError, match='proxies'): + http.get('http://example.test/', proxies={'http': 'http://proxy:8080'}) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py b/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py index 53a1afdcd3e36..f7396776cb4df 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py @@ -4,7 +4,11 @@ import httpx import pytest -from datadog_checks.base.utils.http_exceptions import HTTPConnectionError, HTTPError, HTTPTimeoutError +from datadog_checks.base.utils.http_exceptions import ( + HTTPConnectionError, + HTTPRequestError, + HTTPTimeoutError, +) from datadog_checks.base.utils.http_httpx import HTTPXWrapper @@ -15,8 +19,8 @@ pytest.param(httpx.ReadTimeout('slow'), HTTPTimeoutError, id='read-timeout'), pytest.param(httpx.PoolTimeout('pool'), HTTPTimeoutError, id='pool-timeout'), pytest.param(httpx.ConnectError('refused'), HTTPConnectionError, id='connect-error'), - pytest.param(httpx.LocalProtocolError('bad'), HTTPError, id='local-protocol-error'), - pytest.param(httpx.RequestError('generic'), HTTPError, id='request-error'), + pytest.param(httpx.LocalProtocolError('bad'), HTTPRequestError, id='local-protocol-error'), + pytest.param(httpx.RequestError('generic'), HTTPRequestError, id='request-error'), ], ) def test_request_exception_mapping(raising_transport_factory, raised, expected): diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py b/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py index 115f5711b71f6..52683aaf1b265 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_methods.py @@ -1,6 +1,8 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import json + import pytest from datadog_checks.base.utils.http_httpx import HTTPXWrapper @@ -31,5 +33,4 @@ def test_post_json_body_is_serialized(capturing_transport, captured_requests): http.post('http://example.test/path', json={'a': 1, 'b': 'two'}) req = captured_requests[0] assert req.headers['content-type'] == 'application/json' - assert b'"a":' in req.content - assert b'"b":' in req.content + assert json.loads(req.content) == {'a': 1, 'b': 'two'} diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py index f0fa1114a1421..a9390f63e8639 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py @@ -3,6 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from datetime import timedelta +import httpx import pytest from datadog_checks.base.utils.http_exceptions import HTTPStatusError @@ -94,3 +95,58 @@ class _FakeResponseExposingReason: adapter = HTTPXResponseAdapter(_FakeResponseExposingReason()) # type: ignore[arg-type] assert adapter.reason == 'Not Found' + + +def test_response_text_decodes_body(status_transport_factory): + transport = status_transport_factory(200, b'hello') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.text == 'hello' + + +def test_response_json_returns_decoded_object(): + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={'a': 1}) + + http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + response = http.get('http://example.test/') + assert response.json() == {'a': 1} + + +def test_response_content_returns_raw_bytes(status_transport_factory): + transport = status_transport_factory(200, b'\x00\x01\x02') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.content == b'\x00\x01\x02' + + +def test_response_headers_exposed(): + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, headers={'X-Custom': 'value'}, content=b'') + + http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + response = http.get('http://example.test/') + assert response.headers['X-Custom'] == 'value' + + +def test_response_url_reflects_request_url(status_transport_factory): + transport = status_transport_factory(200, b'') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/path') + assert str(response.url) == 'http://example.test/path' + + +def test_response_encoding_defaults_to_utf8(status_transport_factory): + transport = status_transport_factory(200, b'hello') + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + assert response.encoding in (None, 'utf-8', 'ascii') + + +def test_response_cookies_exposed(): + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, headers={'Set-Cookie': 'session=abc123'}, content=b'') + + http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + response = http.get('http://example.test/') + assert response.cookies['session'] == 'abc123' From 8e8c4a5df4ae1894f4d82d23632a33e770d0623f Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 28 May 2026 21:56:32 -0400 Subject: [PATCH 15/19] phase2 mvp: drop unreachable reason fallback --- .../datadog_checks/base/utils/http_httpx.py | 5 +---- .../tests/base/utils/http_httpx/test_response.py | 8 -------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py index 66b0ec1a85160..9600036482bd7 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py @@ -139,10 +139,7 @@ def ok(self) -> bool: @property def reason(self) -> str: - reason = getattr(self._response, 'reason_phrase', None) - if reason is not None: - return reason - return getattr(self._response, 'reason', '') or '' + return self._response.reason_phrase @property def encoding(self) -> str | None: diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py index a9390f63e8639..3a3924d9129c2 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py @@ -89,14 +89,6 @@ def test_response_reason_from_httpx_response(status_transport_factory): assert response.reason == 'OK' -def test_response_reason_falls_back_when_reason_phrase_missing(): - class _FakeResponseExposingReason: - reason = 'Not Found' - - adapter = HTTPXResponseAdapter(_FakeResponseExposingReason()) # type: ignore[arg-type] - assert adapter.reason == 'Not Found' - - def test_response_text_decodes_body(status_transport_factory): transport = status_transport_factory(200, b'hello') http = HTTPXWrapper({}, {}, transport=transport) From b6ec5ae64d2e5ba89ce1576cb24254a87269ffb9 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 28 May 2026 22:17:20 -0400 Subject: [PATCH 16/19] phase2 mvp: drop allow_redirects per-request kwarg + comment proxies placeholder --- .../datadog_checks/base/utils/http_httpx.py | 5 +---- .../tests/base/utils/http_httpx/test_config.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py index 9600036482bd7..ad8ffb50b1a4f 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py @@ -55,7 +55,6 @@ 'extra_headers', 'timeout', 'follow_redirects', - 'allow_redirects', } ) @@ -246,7 +245,7 @@ def __init__( timeout = _build_timeout(config) allow_redirects = is_affirmative(config['allow_redirects']) - # TODO(httpx-migration): proxies wiring deferred to Phase 3. + # proxies=None mirrors RequestsWrapper.options for consumers (e.g. http_check). Wiring is Phase 3. self.options: dict[str, Any] = { 'auth': auth, 'cert': cert, @@ -397,8 +396,6 @@ def _build_request_kwargs(self, options: dict[str, Any]) -> dict[str, Any]: if 'follow_redirects' in options: kwargs['follow_redirects'] = bool(options['follow_redirects']) - elif 'allow_redirects' in options: - kwargs['follow_redirects'] = bool(options['allow_redirects']) return kwargs diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_config.py b/datadog_checks_base/tests/base/utils/http_httpx/test_config.py index 658e8967bf134..f473ad9f4c871 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_config.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_config.py @@ -103,7 +103,14 @@ def test_remapper_renames_field(capturing_transport): assert http.options['verify'] is False -def test_request_rejects_unknown_kwarg(capturing_transport): +@pytest.mark.parametrize( + 'kwarg,value', + [ + pytest.param('proxies', {'http': 'http://proxy:8080'}, id='proxies'), + pytest.param('allow_redirects', False, id='allow-redirects-uses-httpx-name'), + ], +) +def test_request_rejects_unknown_kwarg(capturing_transport, kwarg, value): http = HTTPXWrapper({}, {}, transport=capturing_transport) - with pytest.raises(TypeError, match='proxies'): - http.get('http://example.test/', proxies={'http': 'http://proxy:8080'}) + with pytest.raises(TypeError, match=kwarg): + http.get('http://example.test/', **{kwarg: value}) From 66f2302fc04584f1156564035760d9238de3f87d Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 28 May 2026 23:04:18 -0400 Subject: [PATCH 17/19] phase2 mvp: route httpx.InvalidURL through HTTPInvalidURLError --- .../datadog_checks/base/utils/http_httpx.py | 7 +++++-- .../base/utils/http_httpx/test_exceptions.py | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py index ad8ffb50b1a4f..f27a7c07eeb36 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py @@ -16,6 +16,7 @@ from .http_exceptions import ( HTTPConnectionError, HTTPError, + HTTPInvalidURLError, HTTPRequestError, HTTPStatusError, HTTPTimeoutError, @@ -90,7 +91,7 @@ def _build_timeout(config: dict[str, Any]) -> tuple[float, float]: return connect, read -def _map_httpx_exception(exc: BaseException) -> HTTPError: +def _map_httpx_exception(exc: httpx.HTTPError | httpx.InvalidURL) -> HTTPError: """Translate an httpx exception into the library-agnostic equivalent.""" if isinstance(exc, httpx.TimeoutException): return HTTPTimeoutError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) @@ -102,6 +103,8 @@ def _map_httpx_exception(exc: BaseException) -> HTTPError: request=getattr(exc, 'request', None), response=getattr(exc, 'response', None), ) + if isinstance(exc, httpx.InvalidURL): + return HTTPInvalidURLError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) if isinstance(exc, httpx.RequestError): return HTTPRequestError(str(exc) or exc.__class__.__name__, request=getattr(exc, 'request', None)) return HTTPError(str(exc) or exc.__class__.__name__) @@ -355,7 +358,7 @@ def _request(self, method: str, url: str, options: dict[str, Any]) -> HTTPXRespo try: request = self._client.build_request(method, url, **request_kwargs) response = self._client.send(request, stream=True, follow_redirects=follow_redirects) - except httpx.HTTPError as exc: + except (httpx.HTTPError, httpx.InvalidURL) as exc: raise _map_httpx_exception(exc) from exc return HTTPXResponseAdapter(response) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py b/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py index f7396776cb4df..ef68a8728369f 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_exceptions.py @@ -6,10 +6,11 @@ from datadog_checks.base.utils.http_exceptions import ( HTTPConnectionError, + HTTPInvalidURLError, HTTPRequestError, HTTPTimeoutError, ) -from datadog_checks.base.utils.http_httpx import HTTPXWrapper +from datadog_checks.base.utils.http_httpx import HTTPXWrapper, _map_httpx_exception @pytest.mark.parametrize( @@ -28,3 +29,15 @@ def test_request_exception_mapping(raising_transport_factory, raised, expected): http = HTTPXWrapper({}, {}, transport=transport) with pytest.raises(expected): http.get('http://example.test/') + + +def test_map_httpx_exception_routes_invalid_url(): + mapped = _map_httpx_exception(httpx.InvalidURL('bad url')) + assert isinstance(mapped, HTTPInvalidURLError) + + +def test_request_raises_invalid_url_error(raising_transport_factory): + transport = raising_transport_factory(httpx.InvalidURL('bad url')) + http = HTTPXWrapper({}, {}, transport=transport) + with pytest.raises(HTTPInvalidURLError): + http.get('http://example.test/') From fd2e807aea7d988e708dbf0d7c06671046b65290 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 28 May 2026 23:05:54 -0400 Subject: [PATCH 18/19] phase2 mvp: align HTTPXResponseAdapter with HTTPResponseProtocol shape --- .../datadog_checks/base/utils/http_httpx.py | 24 ++++++------ .../base/utils/http_httpx/test_response.py | 39 +++++++++++++++++-- 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py index f27a7c07eeb36..5ccc9fbcb36da 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py @@ -25,6 +25,8 @@ LOGGER = logging.getLogger(__name__) DEFAULT_TIMEOUT = 10 +# Matches DEFAULT_CHUNK_SIZE in http.py to preserve iter_content/iter_lines parity with RequestsWrapper. +DEFAULT_CHUNK_SIZE = 16 STANDARD_FIELDS = { 'allow_redirects': True, @@ -164,6 +166,7 @@ def elapsed(self) -> timedelta: try: return self._response.elapsed except RuntimeError: + LOGGER.debug('elapsed unavailable for response from %s', self._response.url) return timedelta(0) def json(self, **kwargs: Any) -> Any: @@ -183,18 +186,14 @@ def close(self) -> None: self._response.close() def iter_content(self, chunk_size: int | None = None, decode_unicode: bool = False) -> Iterator[bytes | str]: - if chunk_size is None: - content = self._response.read() - if not content: - return - yield content.decode('utf-8') if decode_unicode else content - return - for chunk in self._response.iter_bytes(chunk_size=chunk_size): - yield chunk.decode('utf-8') if decode_unicode else chunk + effective_size = chunk_size if chunk_size is not None else DEFAULT_CHUNK_SIZE + encoding = self._response.encoding or 'utf-8' + for chunk in self._response.iter_bytes(chunk_size=effective_size): + yield chunk.decode(encoding) if decode_unicode else chunk def iter_lines( self, - chunk_size: int | None = None, + chunk_size: int | None = None, # noqa: ARG002 - httpx buffers lines internally; kept for HTTPResponseProtocol parity decode_unicode: bool = False, delimiter: bytes | str | None = None, ) -> Iterator[bytes | str]: @@ -414,5 +413,8 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool | None: self.close() return None - def __del__(self) -> None: - self.close() + def __del__(self) -> None: # no cov + try: + self.close() + except AttributeError: + pass diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py index 3a3924d9129c2..5bf83063b9806 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_response.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_response.py @@ -1,13 +1,14 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import logging from datetime import timedelta import httpx import pytest from datadog_checks.base.utils.http_exceptions import HTTPStatusError -from datadog_checks.base.utils.http_httpx import HTTPXResponseAdapter, HTTPXWrapper +from datadog_checks.base.utils.http_httpx import DEFAULT_CHUNK_SIZE, HTTPXResponseAdapter, HTTPXWrapper @pytest.mark.parametrize('status_code', [404, 500]) @@ -56,6 +57,34 @@ def test_response_iter_content_empty_body_yields_nothing(status_transport_factor assert list(response.iter_content()) == [] +def test_response_iter_content_default_chunk_size_uses_default(status_transport_factory): + body = b'X' * (DEFAULT_CHUNK_SIZE * 3 + 5) + transport = status_transport_factory(200, body) + http = HTTPXWrapper({}, {}, transport=transport) + response = http.get('http://example.test/') + chunks = list(response.iter_content()) + assert b''.join(chunks) == body + assert all(len(chunk) <= DEFAULT_CHUNK_SIZE for chunk in chunks) + assert any(len(chunk) == DEFAULT_CHUNK_SIZE for chunk in chunks) + + +@pytest.mark.parametrize( + 'charset,raw,expected', + [ + pytest.param('utf-8', 'café'.encode('utf-8'), 'café', id='utf-8'), + pytest.param('iso-8859-1', 'café'.encode('iso-8859-1'), 'café', id='iso-8859-1'), + ], +) +def test_response_iter_content_decode_uses_response_encoding(charset, raw, expected): + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=raw, headers={'Content-Type': f'text/plain; charset={charset}'}) + + http = HTTPXWrapper({}, {}, transport=httpx.MockTransport(handler)) + response = http.get('http://example.test/') + chunks = list(response.iter_content(chunk_size=64, decode_unicode=True)) + assert ''.join(chunks) == expected + + def test_response_iter_lines_rejects_delimiter(status_transport_factory): transport = status_transport_factory(200, b'a\nb\n') http = HTTPXWrapper({}, {}, transport=transport) @@ -64,14 +93,18 @@ def test_response_iter_lines_rejects_delimiter(status_transport_factory): list(response.iter_lines(delimiter=b'|')) -def test_response_elapsed_returns_zero_on_runtime_error(): +def test_response_elapsed_returns_zero_on_runtime_error(caplog): class _FakeResponse: + url = 'http://example.test/' + @property def elapsed(self): raise RuntimeError('not measured') adapter = HTTPXResponseAdapter(_FakeResponse()) # type: ignore[arg-type] - assert adapter.elapsed == timedelta(0) + with caplog.at_level(logging.DEBUG, logger='datadog_checks.base.utils.http_httpx'): + assert adapter.elapsed == timedelta(0) + assert any('elapsed unavailable' in record.message for record in caplog.records) @pytest.mark.parametrize('status_code,expected_ok', [(200, True), (204, True), (301, True), (400, False), (500, False)]) From 5f97af4836ae02bc01da99c1c38157888133e101 Mon Sep 17 00:00:00 2001 From: mwdd146980 Date: Thu, 28 May 2026 23:11:28 -0400 Subject: [PATCH 19/19] phase2 mvp: parametrize auth/dispatch tests, cover default-wrapper path --- .../base/utils/http_httpx/test_auth_basic.py | 26 +++++++++---------- .../base/utils/http_httpx/test_lifecycle.py | 16 ++++++++---- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py b/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py index 8c776c51b4d50..23081a10a34b1 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_auth_basic.py @@ -1,6 +1,8 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + from datadog_checks.base.utils.http_httpx import HTTPXWrapper from .common import parse_basic_auth @@ -18,19 +20,15 @@ def test_basic_auth_sent_on_request(capturing_transport, captured_requests): assert password == 'secret' -def test_no_auth_without_credentials(capturing_transport, captured_requests): - http = HTTPXWrapper({}, {}, transport=capturing_transport) - http.get('http://example.test/') - assert 'authorization' not in captured_requests[0].headers - - -def test_basic_auth_only_when_both_user_and_password_set(capturing_transport, captured_requests): - http = HTTPXWrapper({'username': 'alice'}, {}, transport=capturing_transport) - http.get('http://example.test/') - assert 'authorization' not in captured_requests[0].headers - - -def test_basic_auth_skipped_when_only_password_set(capturing_transport, captured_requests): - http = HTTPXWrapper({'password': 'secret'}, {}, transport=capturing_transport) +@pytest.mark.parametrize( + 'instance', + [ + pytest.param({}, id='no-credentials'), + pytest.param({'username': 'alice'}, id='username-only'), + pytest.param({'password': 'secret'}, id='password-only'), + ], +) +def test_no_authorization_header_set(instance, capturing_transport, captured_requests): + http = HTTPXWrapper(instance, {}, transport=capturing_transport) http.get('http://example.test/') assert 'authorization' not in captured_requests[0].headers diff --git a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py index aa63d82c20e56..e44d3e696c6d5 100644 --- a/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py +++ b/datadog_checks_base/tests/base/utils/http_httpx/test_lifecycle.py @@ -27,10 +27,16 @@ def test_module_import_fails_without_httpx(monkeypatch): import datadog_checks.base.utils.http_httpx # noqa: F401 -def test_agentcheck_http_dispatch_returns_httpx_wrapper(): +@pytest.mark.parametrize( + 'instance,expected_cls_name', + [ + pytest.param({'use_httpx': True}, 'HTTPXWrapper', id='opt-in'), + pytest.param({'use_httpx': False}, 'RequestsWrapper', id='explicit-default'), + pytest.param({}, 'RequestsWrapper', id='unset-default'), + ], +) +def test_agentcheck_http_dispatch(instance, expected_cls_name): from datadog_checks.base import AgentCheck - check = AgentCheck('test', {}, [{'use_httpx': True}]) - http = check.http - assert isinstance(http, HTTPXWrapper) - http.close() + check = AgentCheck('test', {}, [instance]) + assert type(check.http).__name__ == expected_cls_name