From e56b4f2b8d1f841364c9f93009d139a655316a00 Mon Sep 17 00:00:00 2001 From: Milo Blake Date: Fri, 3 Jul 2026 13:04:06 +0000 Subject: [PATCH] feat: Move health and metrics to dedicated port Health and metrics endpoints were previously attached to the public API router, allowing end users to access /health, /ready, and /metrics directly, which is a potential DDoS vector. This moves them to a dedicated listener on port 8099, separate from the public API on port 8080. Adds a new config section to configure the address. Updates the k8s StatefulSet manifest to point liveness, readiness, and startup probes at the new port. Co-developed-by: Claude (Anthropic) Closes #719 Signed-off-by: Milo Blake --- crates/config/src/interface.rs | 20 ++++++++++ crates/config/src/lib.rs | 6 ++- crates/keystone/src/api/health.rs | 2 +- crates/keystone/src/api/mod.rs | 6 ++- crates/keystone/src/bin/keystone.rs | 44 +++++++++++++++++---- tools/k8s/keystone/base/statefulset-rs.yaml | 9 +++-- 6 files changed, 73 insertions(+), 14 deletions(-) diff --git a/crates/config/src/interface.rs b/crates/config/src/interface.rs index 88dab13ce..de66ed264 100644 --- a/crates/config/src/interface.rs +++ b/crates/config/src/interface.rs @@ -66,6 +66,26 @@ impl Default for PublicInterface { } } +/// Keystone health/metrics API interface. +#[derive(Debug, Deserialize, Clone)] +pub struct MetricsInterface { + /// Default address for the health/metrics endpoint. Defaults to `0.0.0.0:8082`. + #[serde(default = "default_metrics_tcp_address")] + pub tcp_address: SocketAddr, +} + +impl Default for MetricsInterface { + fn default() -> Self { + Self { + tcp_address: default_metrics_tcp_address(), + } + } +} + +fn default_metrics_tcp_address() -> SocketAddr { + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8099) +} + fn default_public_tcp_address() -> SocketAddr { SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8080) } diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index d7b3c49af..743061278 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -206,7 +206,7 @@ pub struct Config { #[serde(rename = "interface_internal", default)] pub interface_internal: Option, - /// Server listener configuration for the internal interface. + /// Server listener configuration for the public interface. #[serde(rename = "interface_public", default)] pub interface_public: PublicInterface, @@ -214,6 +214,10 @@ pub struct Config { #[serde(rename = "interface_admin", default)] pub interface_admin: Option, + /// Server listener configuration for the health/metrics interface. + #[serde(rename = "interface_metrics", default)] + pub interface_metrics: MetricsInterface, + /// Resource provider configuration. #[serde(default)] pub resource: ResourceProvider, diff --git a/crates/keystone/src/api/health.rs b/crates/keystone/src/api/health.rs index af129b076..4e8bbe898 100644 --- a/crates/keystone/src/api/health.rs +++ b/crates/keystone/src/api/health.rs @@ -295,7 +295,7 @@ pub(crate) mod tests { #[tokio::test] async fn health_returns_service_unavailable_for_disconnected_db() { let state = get_mocked_state(Provider::mocked_builder(), true, None).await; - let (router, _api) = super::super::openapi_router().split_for_parts(); + let (router, _api) = super::super::metrics_router().split_for_parts(); let app = router.with_state(state); let response = app diff --git a/crates/keystone/src/api/mod.rs b/crates/keystone/src/api/mod.rs index a3bcde006..c30618ae9 100644 --- a/crates/keystone/src/api/mod.rs +++ b/crates/keystone/src/api/mod.rs @@ -70,10 +70,14 @@ pub fn openapi_router() -> OpenApiRouter { OpenApiRouter::new() .nest("/v3", v3::openapi_router()) .nest("/v4", v4::openapi_router()) - .merge(health::openapi_router()) .routes(routes!(version)) } +/// Dedicated health/metrics router for the internal health/metrics interface. +pub fn metrics_router() -> OpenApiRouter { + health::openapi_router() +} + /// Version discovery endpoint. #[utoipa::path( get, diff --git a/crates/keystone/src/bin/keystone.rs b/crates/keystone/src/bin/keystone.rs index 590711191..9ed51b879 100644 --- a/crates/keystone/src/bin/keystone.rs +++ b/crates/keystone/src/bin/keystone.rs @@ -291,6 +291,7 @@ async fn main() -> Result<(), Report> { spawn_opa_subprocess(&cfg, &token, &mut handles)?; spawn_public_listener(&cfg, app.clone(), &token, &mut handles).await?; spawn_internal_listener(&cfg, app.clone(), &token, &mut handles)?; + spawn_metrics_listener(&cfg, &shared_state, &token, &mut handles).await?; spawn_admin_listener(&cfg, app, &token, &mut handles); // Wait for both (or handle errors) @@ -644,13 +645,7 @@ async fn build_router( .layer(PropagateRequestIdLayer::new(x_request_id)); //.layer(middleware::from_fn(cert_extension_middleware)); - let metrics_router = Router::new() - .route("/metrics", axum::routing::get(metrics_handler)) - .with_state(shared_state.clone()); - - let mut app = Router::new() - .merge(main_router.with_state(shared_state.clone())) - .merge(metrics_router); + let mut app = Router::new().merge(main_router.with_state(shared_state.clone())); if shared_state .config_manager @@ -869,6 +864,39 @@ fn spawn_internal_listener( Ok(()) } +/// Start the metrics and health interface listener on a dedicated port. +async fn spawn_metrics_listener( + cfg: &Config, + shared_state: &ServiceState, + token: &CancellationToken, + handles: &mut tokio::task::JoinSet<()>, +) -> Result<(), Report> { + info!( + "Starting metrics/health API at {}", + cfg.interface_metrics.tcp_address + ); + let listener = TcpListener::bind(&cfg.interface_metrics.tcp_address).await?; + let cancel_token = token.clone(); + + let (metrics_router, _) = api::metrics_router().split_for_parts(); + let metrics_app = Router::new() + .merge(metrics_router.with_state(shared_state.clone())) + .route("/metrics", axum::routing::get(metrics_handler)) + .with_state(shared_state.clone()); + + handles.spawn(async move { + if let Err(e) = axum::serve(listener, metrics_app.into_make_service()) + .with_graceful_shutdown(async move { + cancel_token.cancelled().await; + }) + .await + { + error!("Metrics/health listener error: {:#}", e); + } + }); + Ok(()) +} + /// Start the SPIFFE mTLS listener on the admin Unix-domain-socket interface, /// when configured. fn spawn_admin_listener( @@ -1109,7 +1137,7 @@ mod tests { let response = app .oneshot( Request::builder() - .uri("/metrics/") + .uri("/v3/") .body(axum::body::Body::empty()) .unwrap(), ) diff --git a/tools/k8s/keystone/base/statefulset-rs.yaml b/tools/k8s/keystone/base/statefulset-rs.yaml index ff4603072..a526e2cfc 100644 --- a/tools/k8s/keystone/base/statefulset-rs.yaml +++ b/tools/k8s/keystone/base/statefulset-rs.yaml @@ -43,19 +43,22 @@ spec: - containerPort: 8300 name: keystone-raft protocol: TCP + - containerPort: 8099 + name: keystone-metrics + protocol: TCP livenessProbe: failureThreshold: 3 httpGet: path: /health - port: keystone-rs + port: keystone-metrics scheme: HTTP periodSeconds: 10 readinessProbe: failureThreshold: 3 httpGet: path: /ready - port: keystone-rs + port: keystone-metrics scheme: HTTP periodSeconds: 10 successThreshold: 1 @@ -64,7 +67,7 @@ spec: failureThreshold: 600 httpGet: path: /ready - port: keystone-rs + port: keystone-metrics scheme: HTTP periodSeconds: 1 successThreshold: 1