Increase and stabilize test timeouts and waits

Raise various test timeouts and replace fragile polling with explicit waits to reduce CI flakiness. Changes include:

- backend: increased integration/misc test timeouts (user consume test 10s→20s, runtime metrics 5s→10s) to accommodate shared-server startup delays under CI.
- link/kalam-client: widen AUTO_SERVER_READY_TIMEOUT (15s→30s), AUTO_SERVER_HTTP_TIMEOUT (10s→15s) and add AUTO_SERVER_BLOCKING_TIMEOUT (60s); use blocking timeout for server startup/token waits.
- proxied tests/helpers: increase RECONNECT_WAIT_TIMEOUT (15s→30s), connection_timeout (10s→15s), subscribe/auth handshake budgets (5s→10s) to tolerate slower CI runs; raise overall test timeout for subscribe_during_reconnect (15s→45s).
- double_outage test: replace manual polling loops with explicit wait helpers (wait_for_active_connections / wait_for_reconnect) and add an assert to ensure the client begins reconnecting before the second outage.

These changes aim to make tests more robust against intermittent delays in CI and reduce spurious failures.

Author: jamals86

backend/tests/integration_tests/topic_pubsub.rs

@@ -320,7 +320,7 @@ async fn test_drop_topic() {
 
 /// Test that user role is forbidden from consuming topics
 #[tokio::test]
-#[ntest::timeout(10000)]
+#[ntest::timeout(20000)]
 async fn test_consume_user_role_forbidden() {
     let server = TestServer::new_shared().await;
 

backend/tests/misc/system/test_runtime_metrics.rs

@@ -3,7 +3,7 @@ use kalam_client::models::ResponseStatus;
 use serial_test::serial;
 
 #[tokio::test]
-#[ntest::timeout(5000)] // healthy shared-server startup + query now lands around 3.1s
+#[ntest::timeout(10000)] // local runs are fast, but shared-server startup can spike under CI load
 #[serial(memory_metrics)]
 async fn test_system_stats_expose_memory_breakdown_and_allocator_metrics() {
     let server = TestServer::new_shared().await;

link/kalam-client/tests/common/mod.rs

@@ -29,9 +29,13 @@ struct AutoTestServer {
     _running: RunningTestHttpServer,
 }
 
-const AUTO_SERVER_READY_TIMEOUT: Duration = Duration::from_secs(15);
+// Full-suite CI can delay isolated bootstrap and login well past 20s even when
+// the server eventually becomes healthy, so keep the blocking bridge wider than
+// the HTTP polling window used inside the helper itself.
+const AUTO_SERVER_READY_TIMEOUT: Duration = Duration::from_secs(30);
 const AUTO_SERVER_RETRY_INTERVAL: Duration = Duration::from_millis(100);
-const AUTO_SERVER_HTTP_TIMEOUT: Duration = Duration::from_secs(10);
+const AUTO_SERVER_HTTP_TIMEOUT: Duration = Duration::from_secs(15);
+const AUTO_SERVER_BLOCKING_TIMEOUT: Duration = Duration::from_secs(60);
 
 fn should_auto_start_test_server() -> bool {
     if std::env::var("KALAMDB_SERVER_URL").is_ok() {
@@ -74,7 +78,7 @@ fn ensure_auto_test_server(
                 let _ = tx.send(result);
             });
 
-            match rx.recv_timeout(Duration::from_secs(20)) {
+            match rx.recv_timeout(AUTO_SERVER_BLOCKING_TIMEOUT) {
                 Ok(result) => result,
                 Err(err) => Err(format!("Timed out starting test server: {}", err)),
             }
@@ -307,7 +311,7 @@ fn root_access_token_blocking_for_base_url(
             let _ = tx.send(result);
         });
 
-        match rx.recv_timeout(Duration::from_secs(20)) {
+        match rx.recv_timeout(AUTO_SERVER_BLOCKING_TIMEOUT) {
             Ok(Ok(token)) => return Ok(token),
             Ok(Err(err)) => return Err(err.into()),
             Err(err) => return Err(err.to_string().into()),

link/kalam-client/tests/proxied/double_outage.rs

@@ -91,12 +91,12 @@ async fn test_proxy_server_down_while_reconnecting() {
     // Briefly resume the proxy so the client starts its reconnect attempt,
     // then kill it again immediately.
     proxy.simulate_server_up();
-    for _ in 0..20 {
-        if connect_count.load(Ordering::SeqCst) >= 2 || proxy.active_count().await >= 1 {
-            break;
-        }
-        sleep(Duration::from_millis(100)).await;
-    }
+    assert!(
+        proxy
+            .wait_for_active_connections(1, Duration::from_secs(10))
+            .await,
+        "client should begin reconnecting before the second outage"
+    );
 
     // ── Second outage while reconnecting ────────────────────────────────
     let dc2 = disconnect_count.load(Ordering::SeqCst);
@@ -126,13 +126,13 @@ async fn test_proxy_server_down_while_reconnecting() {
     let expected_connects = connect_count.load(Ordering::SeqCst) + 1;
     proxy.simulate_server_up();
 
-    for _ in 0..100 {
-        if connect_count.load(Ordering::SeqCst) >= expected_connects && client.is_connected().await
-        {
-            break;
-        }
-        sleep(Duration::from_millis(100)).await;
-    }
+    wait_for_reconnect(
+        &client,
+        &connect_count,
+        expected_connects,
+        "double outage final recovery",
+    )
+    .await;
     assert!(client.is_connected().await, "client should recover after double outage");
 
     let mut resumed_ids = Vec::<String>::new();

link/kalam-client/tests/proxied/helpers.rs

@@ -14,7 +14,7 @@ use std::time::Duration;
 use tokio::time::{sleep, Instant};
 
 pub const TEST_TIMEOUT: Duration = Duration::from_secs(10);
-pub const RECONNECT_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
+pub const RECONNECT_WAIT_TIMEOUT: Duration = Duration::from_secs(30);
 
 fn reconnect_test_timeouts() -> KalamLinkTimeouts {
     KalamLinkTimeouts {
@@ -23,14 +23,15 @@ fn reconnect_test_timeouts() -> KalamLinkTimeouts {
         // suite) do not time out before the in-process isolated server finishes
         // the WebSocket handshake.  When the TCP proxy is paused, connections
         // fail immediately regardless of this value.
-        connection_timeout: Duration::from_secs(10),
+        connection_timeout: Duration::from_secs(15),
         receive_timeout: Duration::from_secs(5),
         send_timeout: Duration::from_secs(2),
-        // Reconnect involves auth + resubscribe handshakes. Keep these high
-        // enough for a loaded debug build, but do not inflate the overall test
-        // wall-clock timeout; the outer test timeout remains the guardrail.
-        subscribe_timeout: Duration::from_secs(5),
-        auth_timeout: Duration::from_secs(5),
+        // Reconnect involves auth + resubscribe handshakes. CI full-suite runs
+        // regularly push the isolated server beyond 5s here, so keep the
+        // handshake budget above that while the outer test timeout remains the
+        // main guardrail.
+        subscribe_timeout: Duration::from_secs(10),
+        auth_timeout: Duration::from_secs(10),
         initial_data_timeout: Duration::from_secs(30),
         idle_timeout: Duration::ZERO,
         keepalive_interval: Duration::from_secs(1),

link/kalam-client/tests/proxied/subscribe_during_reconnect.rs

@@ -9,7 +9,7 @@ use tokio::time::{sleep, timeout};
 /// outage. The new subscription must eventually be established and deliver its
 /// data once the connection stabilises.
 #[tokio::test]
-#[ntest::timeout(15000)]
+#[ntest::timeout(45000)]
 async fn test_subscribe_during_reconnect_eventually_delivers() {
     let result = timeout(Duration::from_secs(60), async {
         let writer = match create_test_client() {