diff --git a/changelog.d/6809_datadog_agent_api_key_validation.feature.md b/changelog.d/6809_datadog_agent_api_key_validation.feature.md new file mode 100644 index 0000000000000..8a7c102e83f16 --- /dev/null +++ b/changelog.d/6809_datadog_agent_api_key_validation.feature.md @@ -0,0 +1 @@ +The `datadog_agent` source now supports validating incoming Datadog API keys. The new `valid_api_keys` option accepts a list of permitted API keys, and `drop_on_invalid_api_key` controls whether requests carrying an unrecognized key are rejected with a `403 Forbidden` response (when `true`) or accepted with the key simply not stored (when `false`, the default). When `valid_api_keys` is empty (the default), no validation is performed and all API keys are accepted, preserving existing behavior. diff --git a/src/sources/datadog_agent/logs.rs b/src/sources/datadog_agent/logs.rs index 0863fb636b815..ca84da38cf617 100644 --- a/src/sources/datadog_agent/logs.rs +++ b/src/sources/datadog_agent/logs.rs @@ -15,7 +15,10 @@ use vector_lib::{ use vrl::core::Value; use warp::{Filter, filters::BoxedFilter, path as warp_path, path::FullPath, reply::Response}; -use super::{ApiKeyQueryParams, DatadogAgentConfig, DatadogAgentSource, LogMsg, RequestHandler}; +use super::{ + ApiKeyQueryParams, ApiKeyValidation, DatadogAgentConfig, DatadogAgentSource, LogMsg, + RequestHandler, invalid_api_key_error, +}; use crate::{ common::{datadog::DDTAGS, http::ErrorMessage}, event::Event, @@ -43,15 +46,15 @@ pub(super) fn build_warp_filter( let events = source .decode(&encoding_header, body, path.as_str()) .and_then(|body| { - decode_log_body( - body, - source.api_key_extractor.extract( - path.as_str(), - api_token, - query_params.dd_api_key, - ), - &source, - ) + let api_key = match source.api_key_extractor.extract_and_validate( + path.as_str(), + api_token, + query_params.dd_api_key, + ) { + ApiKeyValidation::Accepted(api_key) => api_key, + ApiKeyValidation::Rejected => return Err(invalid_api_key_error()), + }; + decode_log_body(body, api_key, &source) }); handler.clone().handle_request(events, super::LOGS) }, diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index 6f377e826a8b8..24462dda2d687 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -14,7 +14,9 @@ use vector_lib::{ use warp::{Filter, filters::BoxedFilter, path, path::FullPath, reply::Response}; use super::ddmetric_proto::{Metadata, MetricPayload, SketchPayload, metric_payload}; -use super::{ApiKeyQueryParams, DatadogAgentSource, RequestHandler}; +use super::{ + ApiKeyQueryParams, ApiKeyValidation, DatadogAgentSource, RequestHandler, invalid_api_key_error, +}; use crate::{ common::{ datadog::{DatadogMetricType, DatadogSeriesMetric}, @@ -70,13 +72,17 @@ fn sketches_service( let events = source .decode(&encoding_header, body, path.as_str()) .and_then(|body| { + let api_key = match source.api_key_extractor.extract_and_validate( + path.as_str(), + api_token, + query_params.dd_api_key, + ) { + ApiKeyValidation::Accepted(api_key) => api_key, + ApiKeyValidation::Rejected => return Err(invalid_api_key_error()), + }; decode_datadog_sketches( body, - source.api_key_extractor.extract( - path.as_str(), - api_token, - query_params.dd_api_key, - ), + api_key, source.split_metric_namespace, &source.events_received, ) @@ -107,13 +113,17 @@ fn series_v1_service( let events = source .decode(&encoding_header, body, path.as_str()) .and_then(|body| { + let api_key = match source.api_key_extractor.extract_and_validate( + path.as_str(), + api_token, + query_params.dd_api_key, + ) { + ApiKeyValidation::Accepted(api_key) => api_key, + ApiKeyValidation::Rejected => return Err(invalid_api_key_error()), + }; decode_datadog_series_v1( body, - source.api_key_extractor.extract( - path.as_str(), - api_token, - query_params.dd_api_key, - ), + api_key, // Currently metrics do not have schemas defined, so for now we just pass a // default one. &Arc::new(schema::Definition::default_legacy_namespace()), @@ -147,13 +157,17 @@ fn series_v2_service( let events = source .decode(&encoding_header, body, path.as_str()) .and_then(|body| { + let api_key = match source.api_key_extractor.extract_and_validate( + path.as_str(), + api_token, + query_params.dd_api_key, + ) { + ApiKeyValidation::Accepted(api_key) => api_key, + ApiKeyValidation::Rejected => return Err(invalid_api_key_error()), + }; decode_datadog_series_v2( body, - source.api_key_extractor.extract( - path.as_str(), - api_token, - query_params.dd_api_key, - ), + api_key, source.split_metric_namespace, &source.events_received, ) diff --git a/src/sources/datadog_agent/mod.rs b/src/sources/datadog_agent/mod.rs index ac84188df7c78..86ebb8de68900 100644 --- a/src/sources/datadog_agent/mod.rs +++ b/src/sources/datadog_agent/mod.rs @@ -17,7 +17,10 @@ pub(crate) mod ddtrace_proto { include!(concat!(env!("OUT_DIR"), "/dd_trace.rs")); } -use std::{convert::Infallible, fmt::Debug, io::Read, net::SocketAddr, sync::Arc, time::Duration}; +use std::{ + collections::HashSet, convert::Infallible, fmt::Debug, io::Read, net::SocketAddr, sync::Arc, + time::Duration, +}; use bytes::{Buf, Bytes}; use chrono::{DateTime, Utc, serde::ts_milliseconds}; @@ -91,6 +94,31 @@ pub struct DatadogAgentConfig { #[serde(default = "crate::serde::default_true")] store_api_key: bool, + /// A list of API keys that are permitted to send events to this source. + /// + /// When this list is non-empty, the API key carried by an incoming request (in the URL, + /// the `dd-api-key` header, or the `dd-api-key` query parameter) is checked against it. + /// When the list is empty (the default), all API keys are accepted and no validation is + /// performed. + /// + /// The behavior when a request carries an API key that is not in this list is controlled by + /// `drop_on_invalid_api_key`. + #[configurable(metadata(docs::advanced))] + #[serde(default)] + valid_api_keys: Vec, + + /// Controls what happens when a request carries an API key that is not present in + /// `valid_api_keys`. + /// + /// When set to `true`, requests with an unrecognized API key are rejected with a + /// `403 Forbidden` response. When set to `false` (the default), the unrecognized key is + /// simply not stored in the event metadata, but the events are still accepted. + /// + /// This option has no effect when `valid_api_keys` is empty. + #[configurable(metadata(docs::advanced))] + #[serde(default = "crate::serde::default_false")] + drop_on_invalid_api_key: bool, + /// If this is set to `true`, logs are not accepted by the component. #[configurable(metadata(docs::advanced))] #[serde(default = "crate::serde::default_false")] @@ -173,6 +201,8 @@ impl GenerateConfig for DatadogAgentConfig { address: "0.0.0.0:8080".parse().unwrap(), tls: None, store_api_key: true, + valid_api_keys: Vec::new(), + drop_on_invalid_api_key: false, framing: default_framing_message_based(), decoding: default_decoding(), acknowledgements: SourceAcknowledgementsConfig::default(), @@ -209,6 +239,8 @@ impl SourceConfig for DatadogAgentConfig { let tls = MaybeTlsSettings::from_config(self.tls.as_ref(), true)?; let source = DatadogAgentSource::new( self.store_api_key, + self.valid_api_keys.clone(), + self.drop_on_invalid_api_key, decoder, tls.http_protocol_name(), logs_schema_definition, @@ -386,9 +418,41 @@ pub(crate) struct DatadogAgentSource { pub struct ApiKeyExtractor { matcher: Regex, store_api_key: bool, + valid_api_keys: Arc>, + drop_on_invalid_api_key: bool, +} + +/// The result of checking an extracted API key against the configured allow list. +pub(crate) enum ApiKeyValidation { + /// The key is accepted. The contained value is the key to store in the event + /// metadata (which may be `None` when `store_api_key` is disabled or no key was present). + Accepted(Option>), + /// The request carried an API key that is not in `valid_api_keys` and + /// `drop_on_invalid_api_key` is enabled, so the request must be rejected. + Rejected, +} + +/// The error returned to the Datadog Agent when a request is rejected because its API key is not +/// in the configured `valid_api_keys` list and `drop_on_invalid_api_key` is enabled. +pub(crate) fn invalid_api_key_error() -> ErrorMessage { + ErrorMessage::new( + StatusCode::FORBIDDEN, + "API key is not in the configured `valid_api_keys` list.".to_string(), + ) } impl ApiKeyExtractor { + #[cfg(test)] + pub(crate) fn for_test(valid_api_keys: Vec, drop_on_invalid_api_key: bool) -> Self { + Self { + matcher: Regex::new(r"^/v1/input/(?P[[:alnum:]]{32})/??") + .expect("static regex always compiles"), + store_api_key: true, + valid_api_keys: Arc::new(valid_api_keys.into_iter().collect()), + drop_on_invalid_api_key, + } + } + pub fn extract( &self, path: &str, @@ -407,11 +471,41 @@ impl ApiKeyExtractor { // Try from header next .or_else(|| header.map(Arc::from)) } + + /// Extracts the API key from the request and validates it against `valid_api_keys`. + /// + /// When `valid_api_keys` is empty, no validation is performed and the extracted key (if any) + /// is accepted. Otherwise, the key is checked against the allow list: a missing or + /// unrecognized key results in `Rejected` when `drop_on_invalid_api_key` is set, or an + /// `Accepted(None)` (the key is not stored) otherwise. + pub(crate) fn extract_and_validate( + &self, + path: &str, + header: Option, + query_params: Option, + ) -> ApiKeyValidation { + let api_key = self.extract(path, header, query_params); + + if self.valid_api_keys.is_empty() { + return ApiKeyValidation::Accepted(api_key); + } + + match &api_key { + Some(key) if self.valid_api_keys.contains(key.as_ref()) => { + ApiKeyValidation::Accepted(api_key) + } + _ if self.drop_on_invalid_api_key => ApiKeyValidation::Rejected, + _ => ApiKeyValidation::Accepted(None), + } + } } impl DatadogAgentSource { + #[allow(clippy::too_many_arguments)] pub(crate) fn new( store_api_key: bool, + valid_api_keys: Vec, + drop_on_invalid_api_key: bool, decoder: Decoder, protocol: &'static str, logs_schema_definition: Option, @@ -424,6 +518,8 @@ impl DatadogAgentSource { store_api_key, matcher: Regex::new(r"^/v1/input/(?P[[:alnum:]]{32})/??") .expect("static regex always compiles"), + valid_api_keys: Arc::new(valid_api_keys.into_iter().collect()), + drop_on_invalid_api_key, }, log_schema_host_key: log_schema() .host_key_target_path() diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index b2dbf9691db00..19158809061a8 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -48,8 +48,9 @@ use crate::{ schema::Definition, serde::{default_decoding, default_framing_message_based}, sources::datadog_agent::{ - DatadogAgentConfig, DatadogAgentSource, LOGS, LogMsg, METRICS, TRACES, ddmetric_proto, - ddtrace_proto, logs::decode_log_body, metrics::DatadogSeriesRequest, + ApiKeyExtractor, ApiKeyValidation, DatadogAgentConfig, DatadogAgentSource, LOGS, LogMsg, + METRICS, TRACES, ddmetric_proto, ddtrace_proto, logs::decode_log_body, + metrics::DatadogSeriesRequest, }, test_util::{ addr::{PortGuard, next_addr}, @@ -108,6 +109,8 @@ fn test_decode_log_body() { let source = DatadogAgentSource::new( true, + Vec::new(), + false, decoder, "http", Some(test_logs_schema_definition()), @@ -164,6 +167,8 @@ fn test_decode_log_body_parse_ddtags() { let source = DatadogAgentSource::new( true, + Vec::new(), + false, decoder, "http", Some(test_logs_schema_definition()), @@ -201,6 +206,8 @@ fn test_decode_log_body_empty_object() { let source = DatadogAgentSource::new( true, + Vec::new(), + false, decoder, "http", Some(test_logs_schema_definition()), @@ -1609,6 +1616,8 @@ fn test_config_outputs_with_disabled_data_types() { address: "0.0.0.0:8080".parse().unwrap(), tls: None, store_api_key: true, + valid_api_keys: Vec::new(), + drop_on_invalid_api_key: false, framing: default_framing_message_based(), decoding: default_decoding(), acknowledgements: Default::default(), @@ -2053,6 +2062,8 @@ fn test_config_outputs() { address: "0.0.0.0:8080".parse().unwrap(), tls: None, store_api_key: true, + valid_api_keys: Vec::new(), + drop_on_invalid_api_key: false, framing: default_framing_message_based(), decoding, acknowledgements: Default::default(), @@ -2780,6 +2791,8 @@ impl ValidatableComponent for DatadogAgentConfig { address: "0.0.0.0:9007".parse().unwrap(), tls: None, store_api_key: false, + valid_api_keys: Vec::new(), + drop_on_invalid_api_key: false, framing: CharacterDelimitedDecoderConfig { character_delimited: CharacterDelimitedDecoderOptions { delimiter: b',', @@ -2832,3 +2845,48 @@ impl ValidatableComponent for DatadogAgentConfig { } register_validatable_component!(DatadogAgentConfig); + +#[test] +fn api_key_validation() { + let valid = "0123456789abcdef0123456789abcdef".to_string(); + let invalid = "ffffffffffffffffffffffffffffffff".to_string(); + + // No `valid_api_keys` configured: any key (or none) is accepted as-is. + let extractor = ApiKeyExtractor::for_test(vec![], false); + assert!(matches!( + extractor.extract_and_validate("/v1/input", Some(invalid.clone()), None), + ApiKeyValidation::Accepted(Some(key)) if key.as_ref() == invalid + )); + + // Allow list set, key matches (via header): accepted and stored. + let extractor = ApiKeyExtractor::for_test(vec![valid.clone()], true); + assert!(matches!( + extractor.extract_and_validate("/v1/input", Some(valid.clone()), None), + ApiKeyValidation::Accepted(Some(key)) if key.as_ref() == valid + )); + + // Allow list set, key matches (via URL path): accepted and stored. + assert!(matches!( + extractor.extract_and_validate(&format!("/v1/input/{valid}"), None, None), + ApiKeyValidation::Accepted(Some(key)) if key.as_ref() == valid + )); + + // Allow list set, unknown key, drop_on_invalid_api_key = true: rejected. + assert!(matches!( + extractor.extract_and_validate("/v1/input", Some(invalid.clone()), None), + ApiKeyValidation::Rejected + )); + + // Allow list set, no key present, drop_on_invalid_api_key = true: rejected. + assert!(matches!( + extractor.extract_and_validate("/v1/input", None, None), + ApiKeyValidation::Rejected + )); + + // Allow list set, unknown key, drop_on_invalid_api_key = false: accepted but key not stored. + let extractor = ApiKeyExtractor::for_test(vec![valid], false); + assert!(matches!( + extractor.extract_and_validate("/v1/input", Some(invalid), None), + ApiKeyValidation::Accepted(None) + )); +} diff --git a/src/sources/datadog_agent/traces.rs b/src/sources/datadog_agent/traces.rs index 5f0bea7ab0824..79b279748b43d 100644 --- a/src/sources/datadog_agent/traces.rs +++ b/src/sources/datadog_agent/traces.rs @@ -13,7 +13,10 @@ use vector_lib::{ use vrl::event_path; use warp::{Filter, Rejection, Reply, filters::BoxedFilter, path, path::FullPath, reply::Response}; -use super::{ApiKeyQueryParams, DatadogAgentSource, RequestHandler, ddtrace_proto}; +use super::{ + ApiKeyQueryParams, ApiKeyValidation, DatadogAgentSource, RequestHandler, ddtrace_proto, + invalid_api_key_error, +}; use crate::{ common::http::ErrorMessage, event::{Event, ObjectMap, TraceEvent, Value}, @@ -53,22 +56,21 @@ fn build_trace_filter( let events = source .decode(&encoding_header, body, path.as_str()) .and_then(|body| { - handle_dd_trace_payload( - body, - source.api_key_extractor.extract( - path.as_str(), - api_token, - query_params.dd_api_key, - ), - reported_language.as_ref(), - &source, - ) - .map_err(|error| { - ErrorMessage::new( - StatusCode::UNPROCESSABLE_ENTITY, - format!("Error decoding Datadog traces: {error:?}"), - ) - }) + let api_key = match source.api_key_extractor.extract_and_validate( + path.as_str(), + api_token, + query_params.dd_api_key, + ) { + ApiKeyValidation::Accepted(api_key) => api_key, + ApiKeyValidation::Rejected => return Err(invalid_api_key_error()), + }; + handle_dd_trace_payload(body, api_key, reported_language.as_ref(), &source) + .map_err(|error| { + ErrorMessage::new( + StatusCode::UNPROCESSABLE_ENTITY, + format!("Error decoding Datadog traces: {error:?}"), + ) + }) }); handler.clone().handle_request(events, super::TRACES) } diff --git a/website/cue/reference/components/sources/generated/datadog_agent.cue b/website/cue/reference/components/sources/generated/datadog_agent.cue index fa6a519fe1e75..04155c217fd5b 100644 --- a/website/cue/reference/components/sources/generated/datadog_agent.cue +++ b/website/cue/reference/components/sources/generated/datadog_agent.cue @@ -360,6 +360,20 @@ generated: components: sources: datadog_agent: configuration: { required: false type: bool: default: false } + drop_on_invalid_api_key: { + description: """ + Controls what happens when a request carries an API key that is not present in + `valid_api_keys`. + + When set to `true`, requests with an unrecognized API key are rejected with a + `403 Forbidden` response. When set to `false` (the default), the unrecognized key is + simply not stored in the event metadata, but the events are still accepted. + + This option has no effect when `valid_api_keys` is empty. + """ + required: false + type: bool: default: false + } framing: { description: """ Framing configuration. @@ -736,4 +750,22 @@ generated: components: sources: datadog_agent: configuration: { } } } + valid_api_keys: { + description: """ + A list of API keys that are permitted to send events to this source. + + When this list is non-empty, the API key carried by an incoming request (in the URL, + the `dd-api-key` header, or the `dd-api-key` query parameter) is checked against it. + When the list is empty (the default), all API keys are accepted and no validation is + performed. + + The behavior when a request carries an API key that is not in this list is controlled by + `drop_on_invalid_api_key`. + """ + required: false + type: array: { + default: [] + items: type: string: {} + } + } }