Moves health and metrics endpoints to dedicated listener on separate port

crates/config/src/interface.rs

@@ -66,6 +66,26 @@ impl Default for PublicInterface {
     }
 }
 
+/// Keystone health/metrics API interface.
+#[derive(Debug, Deserialize, Clone)]
+pub struct MetricsInterface {
+    /// Default address for the health/metrics endpoint. Defaults to `0.0.0.0:8082`.
+    #[serde(default = "default_metrics_tcp_address")]
+    pub tcp_address: SocketAddr,
+}
+
+impl Default for MetricsInterface {
+    fn default() -> Self {
+        Self {
+            tcp_address: default_metrics_tcp_address(),
+        }
+    }
+}
+
+fn default_metrics_tcp_address() -> SocketAddr {
+    SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8099)
+}
+
 fn default_public_tcp_address() -> SocketAddr {
     SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8080)
 }

crates/config/src/lib.rs

@@ -206,14 +206,18 @@ pub struct Config {
     #[serde(rename = "interface_internal", default)]
     pub interface_internal: Option<InternalInterface>,
 
-    /// Server listener configuration for the internal interface.
+    /// Server listener configuration for the public interface.
     #[serde(rename = "interface_public", default)]
     pub interface_public: PublicInterface,
 
     /// Server listener configuration for the admin interface.
     #[serde(rename = "interface_admin", default)]
     pub interface_admin: Option<AdminInterface>,
 
+    /// Server listener configuration for the health/metrics interface.
+    #[serde(rename = "interface_metrics", default)]
+    pub interface_metrics: MetricsInterface,
+
     /// Resource provider configuration.
     #[serde(default)]
     pub resource: ResourceProvider,

crates/keystone/src/api/health.rs

@@ -295,7 +295,7 @@ pub(crate) mod tests {
     #[tokio::test]
     async fn health_returns_service_unavailable_for_disconnected_db() {
         let state = get_mocked_state(Provider::mocked_builder(), true, None).await;
-        let (router, _api) = super::super::openapi_router().split_for_parts();
+        let (router, _api) = super::super::metrics_router().split_for_parts();
         let app = router.with_state(state);
 
         let response = app

crates/keystone/src/api/mod.rs

@@ -70,10 +70,14 @@ pub fn openapi_router() -> OpenApiRouter<ServiceState> {
     OpenApiRouter::new()
         .nest("/v3", v3::openapi_router())
         .nest("/v4", v4::openapi_router())
-        .merge(health::openapi_router())
         .routes(routes!(version))
 }
 
+/// Dedicated health/metrics router for the internal health/metrics interface.
+pub fn metrics_router() -> OpenApiRouter<ServiceState> {
+    health::openapi_router()
+}
+
 /// Version discovery endpoint.
 #[utoipa::path(
     get,

crates/keystone/src/bin/keystone.rs

@@ -291,6 +291,7 @@ async fn main() -> Result<(), Report> {
     spawn_opa_subprocess(&cfg, &token, &mut handles)?;
     spawn_public_listener(&cfg, app.clone(), &token, &mut handles).await?;
     spawn_internal_listener(&cfg, app.clone(), &token, &mut handles)?;
+    spawn_metrics_listener(&cfg, &shared_state, &token, &mut handles).await?;
     spawn_admin_listener(&cfg, app, &token, &mut handles);
 
     // Wait for both (or handle errors)
@@ -644,13 +645,7 @@ async fn build_router(
         .layer(PropagateRequestIdLayer::new(x_request_id));
     //.layer(middleware::from_fn(cert_extension_middleware));
 
-    let metrics_router = Router::new()
-        .route("/metrics", axum::routing::get(metrics_handler))
-        .with_state(shared_state.clone());
-
-    let mut app = Router::new()
-        .merge(main_router.with_state(shared_state.clone()))
-        .merge(metrics_router);
+    let mut app = Router::new().merge(main_router.with_state(shared_state.clone()));
 
     if shared_state
         .config_manager
@@ -869,6 +864,39 @@ fn spawn_internal_listener(
     Ok(())
 }
 
+/// Start the metrics and health interface listener on a dedicated port.
+async fn spawn_metrics_listener(
+    cfg: &Config,
+    shared_state: &ServiceState,
+    token: &CancellationToken,
+    handles: &mut tokio::task::JoinSet<()>,
+) -> Result<(), Report> {
+    info!(
+        "Starting metrics/health API at {}",
+        cfg.interface_metrics.tcp_address
+    );
+    let listener = TcpListener::bind(&cfg.interface_metrics.tcp_address).await?;
+    let cancel_token = token.clone();
+
+    let (metrics_router, _) = api::metrics_router().split_for_parts();
+    let metrics_app = Router::new()
+        .merge(metrics_router.with_state(shared_state.clone()))
+        .route("/metrics", axum::routing::get(metrics_handler))
+        .with_state(shared_state.clone());
+
+    handles.spawn(async move {
+        if let Err(e) = axum::serve(listener, metrics_app.into_make_service())
+            .with_graceful_shutdown(async move {
+                cancel_token.cancelled().await;
+            })
+            .await
+        {
+            error!("Metrics/health listener error: {:#}", e);
+        }
+    });
+    Ok(())
+}
+
 /// Start the SPIFFE mTLS listener on the admin Unix-domain-socket interface,
 /// when configured.
 fn spawn_admin_listener(
@@ -1109,7 +1137,7 @@ mod tests {
         let response = app
             .oneshot(
                 Request::builder()
-                    .uri("/metrics/")
+                    .uri("/v3/")
                     .body(axum::body::Body::empty())
                     .unwrap(),
             )

tools/k8s/keystone/base/statefulset-rs.yaml

@@ -43,19 +43,22 @@ spec:
             - containerPort: 8300
               name: keystone-raft
               protocol: TCP
+            - containerPort: 8099
+              name: keystone-metrics
+              protocol: TCP
 
           livenessProbe:
             failureThreshold: 3
             httpGet:
               path: /health
-              port: keystone-rs
+              port: keystone-metrics
               scheme: HTTP
             periodSeconds: 10
           readinessProbe:
             failureThreshold: 3
             httpGet:
               path: /ready
-              port: keystone-rs
+              port: keystone-metrics
               scheme: HTTP
             periodSeconds: 10
             successThreshold: 1
@@ -64,7 +67,7 @@ spec:
             failureThreshold: 600
             httpGet:
               path: /ready
-              port: keystone-rs
+              port: keystone-metrics
               scheme: HTTP
             periodSeconds: 1
             successThreshold: 1