From 1a56b040c5892cd8289462f5ae9538ce2b8ca6b7 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 22 Oct 2026 13:20:14 -0700 Subject: [PATCH 1/3] feat(monitoring): log event-loop lag, stalls, and in-flight route handlers --- apps/sim/instrumentation-node.ts | 3 + apps/sim/lib/core/utils/with-route-handler.ts | 4 +- .../lib/monitoring/event-loop-telemetry.ts | 134 ++++++++++++++++ .../lib/monitoring/in-flight-requests.test.ts | 81 ++++++++++ apps/sim/lib/monitoring/in-flight-requests.ts | 148 ++++++++++++++++++ 5 files changed, 369 insertions(+), 1 deletion(-) create mode 100644 apps/sim/lib/monitoring/event-loop-telemetry.ts create mode 100644 apps/sim/lib/monitoring/in-flight-requests.test.ts create mode 100644 apps/sim/lib/monitoring/in-flight-requests.ts diff --git a/apps/sim/instrumentation-node.ts b/apps/sim/instrumentation-node.ts index d361721efba..52050f6a23d 100644 --- a/apps/sim/instrumentation-node.ts +++ b/apps/sim/instrumentation-node.ts @@ -403,6 +403,9 @@ export async function register() { const { startMemoryTelemetry } = await import('./lib/monitoring/memory-telemetry') startMemoryTelemetry() + const { startEventLoopTelemetry } = await import('@/lib/monitoring/event-loop-telemetry') + startEventLoopTelemetry() + // Not awaited: the connection is warmed in the background so the first request // that needs Redis does not pay the handshake inside its own command deadline, // but boot never waits on Redis to serve requests that do not touch it. diff --git a/apps/sim/lib/core/utils/with-route-handler.ts b/apps/sim/lib/core/utils/with-route-handler.ts index 47a35f4be7c..d0d2110322e 100644 --- a/apps/sim/lib/core/utils/with-route-handler.ts +++ b/apps/sim/lib/core/utils/with-route-handler.ts @@ -9,6 +9,7 @@ import { FILE_DELIVERY_CSP_PATH_PATTERN, getMainCSPPolicy } from '@/lib/core/sec import { HttpError } from '@/lib/core/utils/http-error' import { generateRequestId } from '@/lib/core/utils/request' import { MAX_CALL_CHAIN_DEPTH, parseCallChain, SIM_VIA_HEADER } from '@/lib/execution/call-chain' +import { trackInFlightRequest } from '@/lib/monitoring/in-flight-requests' import { withPermissionGroupScope } from '@/lib/permission-groups/request-scope.server' const fileDeliveryPath = new RegExp(`^${FILE_DELIVERY_CSP_PATH_PATTERN}$`) @@ -178,6 +179,7 @@ export function withRouteHandler( callChain: callChainFor(request), } + const settleInFlight = trackInFlightRequest(requestId, method, path) return runWithRequestContext(requestContext, async () => { let response: NextResponse | Response try { @@ -242,6 +244,6 @@ export function withRouteHandler( applyResponseHeaders(response, request, requestId) return response - }) + }).finally(settleInFlight) } } diff --git a/apps/sim/lib/monitoring/event-loop-telemetry.ts b/apps/sim/lib/monitoring/event-loop-telemetry.ts new file mode 100644 index 00000000000..4e7ccef7a43 --- /dev/null +++ b/apps/sim/lib/monitoring/event-loop-telemetry.ts @@ -0,0 +1,134 @@ +/** + * Event-loop health telemetry. One event loop serves every request in the + * process, so a synchronous CPU burst in one request stalls all of them; this + * makes that visible from the process itself. + * + * - Every {@link SNAPSHOT_INTERVAL_MS}, one INFO `Event loop snapshot` line with + * delay percentiles from `monitorEventLoopDelay`, process CPU share, and the + * in-flight route handler count. + * - Whenever a {@link STALL_PROBE_MS} probe timer fires at least + * {@link STALL_WARN_MS} late, one WARN `Event loop stalled` line (at most + * {@link MAX_STALL_WARNINGS_PER_SNAPSHOT} per window) listing the route + * handlers in flight during the stall as concurrency context; see + * `in-flight-requests.ts` for what that list cannot attribute. + * + * Both lines are emitted after the loop recovers, since nothing on a blocked + * loop can run; a stall that never ends shows up as a missing snapshot instead. + * + * Reading the numbers: + * - On Bun the delay histogram records only the excess delay beyond the + * sampling timer, so an idle loop reads near 0 ms. On Node each sample + * includes the {@link HISTOGRAM_RESOLUTION_MS} resolution, so an idle loop + * reads about 20 ms. Compare values only within one runtime. + * - A block is counted as a stall only once the probe is a full + * {@link STALL_WARN_MS} late, which takes a block of 1.0–1.25 s depending on + * where in the probe interval it starts. Shorter blocks show only in + * `lagMaxMs`. + * - `performance.eventLoopUtilization()` is a stub of zeros on Bun, so `elu` is + * emitted only where it reports real values. `cpuPct` (process CPU share, + * which also counts GC and other threads) is the signal on both runtimes. + */ + +import { monitorEventLoopDelay, performance } from 'node:perf_hooks' +import { createLogger } from '@sim/logger' +import { + findRequestsSpanningStall, + getInFlightCount, + takeInFlightWindow, +} from '@/lib/monitoring/in-flight-requests' + +const logger = createLogger('EventLoopTelemetry', { logLevel: 'INFO' }) + +const SNAPSHOT_INTERVAL_MS = 60_000 +const HISTOGRAM_RESOLUTION_MS = 20 +const STALL_PROBE_MS = 250 +const STALL_WARN_MS = 1_000 +/** Caps WARN volume when the loop is stalling continuously; the snapshot still counts every stall. */ +const MAX_STALL_WARNINGS_PER_SNAPSHOT = 10 + +const NS_PER_MS = 1e6 + +/** On `globalThis` so a second module instance (dev reloads, a second bundle graph) cannot start a second set of timers. */ +const g = globalThis as typeof globalThis & { _simEventLoopTelemetryStarted?: boolean } + +const round = (value: number, digits = 1) => { + const factor = 10 ** digits + return Math.round(value * factor) / factor +} + +/** + * Starts the snapshot and stall-probe timers once per process. Both timers are + * unref'd, so they never keep the process alive. Called from instrumentation + * at server start. + */ +export function startEventLoopTelemetry() { + if (g._simEventLoopTelemetryStarted) return + g._simEventLoopTelemetryStarted = true + + const histogram = monitorEventLoopDelay({ resolution: HISTOGRAM_RESOLUTION_MS }) + histogram.enable() + + let stallCount = 0 + let stallMaxMs = 0 + let stallTotalMs = 0 + let warningsThisWindow = 0 + + let lastProbe = performance.now() + const probe = setInterval(() => { + const now = performance.now() + const lateMs = now - lastProbe - STALL_PROBE_MS + lastProbe = now + if (lateMs < STALL_WARN_MS) return + + stallCount++ + stallTotalMs += lateMs + if (lateMs > stallMaxMs) stallMaxMs = lateMs + if (warningsThisWindow >= MAX_STALL_WARNINGS_PER_SNAPSHOT) return + warningsThisWindow++ + + logger.warn('Event loop stalled', { + stallMs: Math.round(lateMs), + inFlight: getInFlightCount(), + inFlightDuringStall: findRequestsSpanningStall(now - lateMs, now), + }) + }, STALL_PROBE_MS) + probe.unref() + + let lastWall = performance.now() + let lastCpu = process.cpuUsage() + let lastElu = performance.eventLoopUtilization() + + const snapshot = setInterval(() => { + const wallMs = performance.now() - lastWall + const cpu = process.cpuUsage(lastCpu) + const elu = performance.eventLoopUtilization(lastElu) + const count = histogram.count + const inFlightWindow = takeInFlightWindow() + + logger.info('Event loop snapshot', { + lagP50Ms: count ? round(histogram.percentile(50) / NS_PER_MS) : 0, + lagP99Ms: count ? round(histogram.percentile(99) / NS_PER_MS) : 0, + lagMaxMs: count ? round(histogram.max / NS_PER_MS) : 0, + lagMeanMs: count ? round(histogram.mean / NS_PER_MS) : 0, + stalls: stallCount, + stallMaxMs: Math.round(stallMaxMs), + stallTotalMs: Math.round(stallTotalMs), + cpuPct: wallMs > 0 ? round(((cpu.user + cpu.system) / 1000 / wallMs) * 100) : 0, + ...(elu.idle + elu.active > 0 ? { elu: round(elu.utilization, 3) } : {}), + inFlight: getInFlightCount(), + inFlightPeak: inFlightWindow.peak, + ...(inFlightWindow.dropped > 0 ? { inFlightDropped: inFlightWindow.dropped } : {}), + windowMs: Math.round(wallMs), + }) + + histogram.reset() + stallCount = 0 + stallMaxMs = 0 + stallTotalMs = 0 + warningsThisWindow = 0 + lastWall = performance.now() + lastCpu = process.cpuUsage() + lastElu = performance.eventLoopUtilization() + }, SNAPSHOT_INTERVAL_MS) + snapshot.unref() +} diff --git a/apps/sim/lib/monitoring/in-flight-requests.test.ts b/apps/sim/lib/monitoring/in-flight-requests.test.ts new file mode 100644 index 00000000000..3082534caa5 --- /dev/null +++ b/apps/sim/lib/monitoring/in-flight-requests.test.ts @@ -0,0 +1,81 @@ +/** + * Failure modes guarded here: + * - Next instantiates this file once per bundle graph (instrumentation vs each + * route), so per-module state would leave the telemetry reader seeing an + * empty registry while routes write to their own copy. + * - Request ids repeat when route wrappers nest, so keying entries by request id + * lets the inner settle erase the outer entry. + * - A handler that never settles must not grow the registry without bound. + * - A synchronous burst settles its own request a few milliseconds before the + * stall probe runs, so a near-zero tolerance never reports it. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +type InFlightModule = typeof import('@/lib/monitoring/in-flight-requests') + +async function loadFreshModuleInstance(): Promise { + vi.resetModules() + return import('@/lib/monitoring/in-flight-requests') +} + +describe('in-flight request registry', () => { + beforeEach(() => { + for (const key of Object.keys(globalThis)) { + if (key.startsWith('_simInFlight')) Reflect.deleteProperty(globalThis, key) + } + }) + + it('shares state between separately instantiated copies of the module', async () => { + const routeInstance = await loadFreshModuleInstance() + const telemetryInstance = await loadFreshModuleInstance() + expect(routeInstance).not.toBe(telemetryInstance) + + const settle = routeInstance.trackInFlightRequest('req-1', 'POST', '/api/a') + expect(telemetryInstance.getInFlightCount()).toBe(1) + + settle() + expect(telemetryInstance.getInFlightCount()).toBe(0) + }) + + it('keeps an outer entry when a nested wrapper reuses the same request id', async () => { + const registry = await loadFreshModuleInstance() + const settleOuter = registry.trackInFlightRequest('req-1', 'POST', '/api/a') + const settleInner = registry.trackInFlightRequest('req-1', 'POST', '/api/a') + + settleInner() + expect(registry.getInFlightCount()).toBe(1) + + settleOuter() + expect(registry.getInFlightCount()).toBe(0) + }) + + it('bounds the registry when handlers never settle', async () => { + const registry = await loadFreshModuleInstance() + for (let i = 0; i < 10_000; i++) registry.trackInFlightRequest(`req-${i}`, 'GET', '/api/hang') + + const tracked = registry.getInFlightCount() + expect(tracked).toBeLessThan(10_000) + expect(registry.takeInFlightWindow().dropped).toBe(10_000 - tracked) + }) + + it('reports a request that settled shortly before the stall was observed', async () => { + const registry = await loadFreshModuleInstance() + const settle = registry.trackInFlightRequest('req-burst', 'POST', '/api/burst') + const stallStart = performance.now() + settle() + const stallEnd = performance.now() + 20 + + const spanning = registry.findRequestsSpanningStall(stallStart, stallEnd) + expect(spanning.map((entry) => entry.requestId)).toEqual(['req-burst']) + expect(spanning[0].finished).toBe(true) + }) + + it('does not report a request that finished well before the stall ended', async () => { + const registry = await loadFreshModuleInstance() + const settle = registry.trackInFlightRequest('req-early', 'GET', '/api/early') + const stallStart = performance.now() + settle() + + expect(registry.findRequestsSpanningStall(stallStart, performance.now() + 500)).toEqual([]) + }) +}) diff --git a/apps/sim/lib/monitoring/in-flight-requests.ts b/apps/sim/lib/monitoring/in-flight-requests.ts new file mode 100644 index 00000000000..60c8380d21b --- /dev/null +++ b/apps/sim/lib/monitoring/in-flight-requests.ts @@ -0,0 +1,148 @@ +/** + * Process-wide registry of route handlers currently running, plus a short ring + * of recently settled ones, so an event-loop stall can list the requests that + * were in flight while it happened. + * + * State lives on `globalThis`: Next instantiates this file once in the + * instrumentation graph and again in each app-route graph, and module-level + * state would leave the telemetry reader looking at an empty copy. + * + * What it can and cannot see: + * - An entry settles when the handler returns its `Response` (headers), not + * when a streamed body finishes, so work done while streaming is never + * attributed. Page renders, server actions, and background loops are not + * tracked at all. + * - The list is concurrency context, not a culprit: a long-poll or SSE handshake + * that merely waited through a stall appears alongside the request that + * caused it. + * - Entries are keyed by a per-call token, so nested wrappers that share a + * request id each keep their own entry (and each count toward `inFlight`). + * - A handler that never settles keeps its entry until the registry holds + * {@link MAX_TRACKED_IN_FLIGHT} entries; the oldest are then dropped and + * counted in {@link InFlightWindow.dropped}. + * + * Paths are already present on every route log line, so naming them here + * widens no exposure. + */ + +interface TrackedRequest { + requestId: string + method: string + path: string + startedAt: number + endedAt?: number +} + +/** A request that was in flight for the whole of a stall window. */ +interface RequestSpanningStall { + requestId: string + method: string + path: string + /** How long the request had been running when the stall ended. */ + ageMs: number + /** True when it settled at (or within the tolerance before) the end of the stall. */ + finished: boolean +} + +/** Per-snapshot-window counters, reset by {@link takeInFlightWindow}. */ +interface InFlightWindow { + /** Highest concurrent in-flight count during the window. */ + peak: number + /** Entries evicted because the registry was full (handlers that never settled). */ + dropped: number +} + +interface InFlightState { + live: Map + recent: (TrackedRequest | undefined)[] + recentNext: number + nextToken: number + peak: number + dropped: number +} + +const MAX_TRACKED_IN_FLIGHT = 4_096 +const RECENT_CAPACITY = 64 +const MAX_SPANNING_RESULTS = 5 +/** + * A synchronous burst settles its own request in the turn that frees the loop, + * and the probe timer runs some milliseconds later, so a request counts as + * spanning the stall when it settled this close to the stall's end. + */ +const SETTLE_TOLERANCE_MS = 50 + +const g = globalThis as typeof globalThis & { _simInFlightRequestState?: InFlightState } +if (!g._simInFlightRequestState) { + g._simInFlightRequestState = { + live: new Map(), + recent: new Array(RECENT_CAPACITY), + recentNext: 0, + nextToken: 0, + peak: 0, + dropped: 0, + } +} +const state = g._simInFlightRequestState + +/** + * Registers a running route handler. Call the returned function once when the + * handler settles; later calls are no-ops. + */ +export function trackInFlightRequest(requestId: string, method: string, path: string): () => void { + const token = state.nextToken++ + const entry: TrackedRequest = { requestId, method, path, startedAt: performance.now() } + state.live.set(token, entry) + if (state.live.size > MAX_TRACKED_IN_FLIGHT) { + const oldest = state.live.keys().next().value + if (oldest !== undefined) state.live.delete(oldest) + state.dropped++ + } + if (state.live.size > state.peak) state.peak = state.live.size + return () => { + if (!state.live.delete(token)) return + entry.endedAt = performance.now() + state.recent[state.recentNext] = entry + state.recentNext = (state.recentNext + 1) % RECENT_CAPACITY + } +} + +/** Number of route handlers currently registered as running. */ +export function getInFlightCount(): number { + return state.live.size +} + +/** Returns the window's counters and starts a new window (peak restarts at the current count). */ +export function takeInFlightWindow(): InFlightWindow { + const window = { peak: state.peak, dropped: state.dropped } + state.peak = state.live.size + state.dropped = 0 + return window +} + +/** + * Requests in flight for the whole stall window: started before it began and + * still running, or settled within {@link SETTLE_TOLERANCE_MS} of its end. + * Settled ones come first (the likeliest source of a synchronous burst), then + * longest-running first. Concurrency context only; see the module TSDoc. + */ +export function findRequestsSpanningStall( + stallStart: number, + stallEnd: number +): RequestSpanningStall[] { + const spansStall = (entry: TrackedRequest) => + entry.startedAt <= stallStart && (entry.endedAt ?? stallEnd) >= stallEnd - SETTLE_TOLERANCE_MS + const matches: TrackedRequest[] = [] + for (const entry of state.live.values()) if (spansStall(entry)) matches.push(entry) + for (const entry of state.recent) if (entry && spansStall(entry)) matches.push(entry) + matches.sort( + (a, b) => + Number(b.endedAt !== undefined) - Number(a.endedAt !== undefined) || a.startedAt - b.startedAt + ) + return matches.slice(0, MAX_SPANNING_RESULTS).map((entry) => ({ + requestId: entry.requestId, + method: entry.method, + path: entry.path, + ageMs: Math.round(stallEnd - entry.startedAt), + finished: entry.endedAt !== undefined, + })) +} From 375743d065a18c2039e0f9a462437a3896573f4c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 22 Oct 2026 13:31:58 -0700 Subject: [PATCH 2/3] fix(monitoring): bound stall window at the previous probe --- apps/sim/lib/monitoring/event-loop-telemetry.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/sim/lib/monitoring/event-loop-telemetry.ts b/apps/sim/lib/monitoring/event-loop-telemetry.ts index 4e7ccef7a43..3a843c571b1 100644 --- a/apps/sim/lib/monitoring/event-loop-telemetry.ts +++ b/apps/sim/lib/monitoring/event-loop-telemetry.ts @@ -76,7 +76,8 @@ export function startEventLoopTelemetry() { let lastProbe = performance.now() const probe = setInterval(() => { const now = performance.now() - const lateMs = now - lastProbe - STALL_PROBE_MS + const previousProbe = lastProbe + const lateMs = now - previousProbe - STALL_PROBE_MS lastProbe = now if (lateMs < STALL_WARN_MS) return @@ -86,10 +87,11 @@ export function startEventLoopTelemetry() { if (warningsThisWindow >= MAX_STALL_WARNINGS_PER_SNAPSHOT) return warningsThisWindow++ + // The loop was last seen free at the previous probe, so the block began at or after it. logger.warn('Event loop stalled', { stallMs: Math.round(lateMs), inFlight: getInFlightCount(), - inFlightDuringStall: findRequestsSpanningStall(now - lateMs, now), + inFlightDuringStall: findRequestsSpanningStall(previousProbe, now), }) }, STALL_PROBE_MS) probe.unref() From b202572c7a88a64ae884ca3c5f217727c7a5b088 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 22 Oct 2026 13:44:41 -0700 Subject: [PATCH 3/3] fix(monitoring): widen the stall-start edge instead of moving it to the previous probe --- .../lib/monitoring/event-loop-telemetry.ts | 6 ++---- .../lib/monitoring/in-flight-requests.test.ts | 21 +++++++++++++++++++ apps/sim/lib/monitoring/in-flight-requests.ts | 16 ++++++++------ 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/apps/sim/lib/monitoring/event-loop-telemetry.ts b/apps/sim/lib/monitoring/event-loop-telemetry.ts index 3a843c571b1..4e7ccef7a43 100644 --- a/apps/sim/lib/monitoring/event-loop-telemetry.ts +++ b/apps/sim/lib/monitoring/event-loop-telemetry.ts @@ -76,8 +76,7 @@ export function startEventLoopTelemetry() { let lastProbe = performance.now() const probe = setInterval(() => { const now = performance.now() - const previousProbe = lastProbe - const lateMs = now - previousProbe - STALL_PROBE_MS + const lateMs = now - lastProbe - STALL_PROBE_MS lastProbe = now if (lateMs < STALL_WARN_MS) return @@ -87,11 +86,10 @@ export function startEventLoopTelemetry() { if (warningsThisWindow >= MAX_STALL_WARNINGS_PER_SNAPSHOT) return warningsThisWindow++ - // The loop was last seen free at the previous probe, so the block began at or after it. logger.warn('Event loop stalled', { stallMs: Math.round(lateMs), inFlight: getInFlightCount(), - inFlightDuringStall: findRequestsSpanningStall(previousProbe, now), + inFlightDuringStall: findRequestsSpanningStall(now - lateMs, now), }) }, STALL_PROBE_MS) probe.unref() diff --git a/apps/sim/lib/monitoring/in-flight-requests.test.ts b/apps/sim/lib/monitoring/in-flight-requests.test.ts index 3082534caa5..b1774f237c8 100644 --- a/apps/sim/lib/monitoring/in-flight-requests.test.ts +++ b/apps/sim/lib/monitoring/in-flight-requests.test.ts @@ -70,6 +70,27 @@ describe('in-flight request registry', () => { expect(spanning[0].finished).toBe(true) }) + it('reports a request that started just after the probe was due, but not one queued behind the stall', async () => { + vi.useFakeTimers({ toFake: ['performance'] }) + try { + const registry = await loadFreshModuleInstance() + const probeDue = performance.now() + vi.advanceTimersByTime(20) + const settleBlocker = registry.trackInFlightRequest('req-blocker', 'POST', '/api/blocker') + vi.advanceTimersByTime(1_500) + settleBlocker() + const settleQueued = registry.trackInFlightRequest('req-queued', 'GET', '/api/queued') + vi.advanceTimersByTime(5) + settleQueued() + vi.advanceTimersByTime(5) + + const spanning = registry.findRequestsSpanningStall(probeDue, performance.now()) + expect(spanning.map((entry) => entry.requestId)).toEqual(['req-blocker']) + } finally { + vi.useRealTimers() + } + }) + it('does not report a request that finished well before the stall ended', async () => { const registry = await loadFreshModuleInstance() const settle = registry.trackInFlightRequest('req-early', 'GET', '/api/early') diff --git a/apps/sim/lib/monitoring/in-flight-requests.ts b/apps/sim/lib/monitoring/in-flight-requests.ts index 60c8380d21b..85131a946d6 100644 --- a/apps/sim/lib/monitoring/in-flight-requests.ts +++ b/apps/sim/lib/monitoring/in-flight-requests.ts @@ -65,11 +65,13 @@ const MAX_TRACKED_IN_FLIGHT = 4_096 const RECENT_CAPACITY = 64 const MAX_SPANNING_RESULTS = 5 /** - * A synchronous burst settles its own request in the turn that frees the loop, - * and the probe timer runs some milliseconds later, so a request counts as - * spanning the stall when it settled this close to the stall's end. + * Slack applied at both edges of a stall window. The probe timer runs some + * milliseconds after the burst that freed the loop settled its own request, and + * a request can start a few milliseconds after the probe was due (other + * callbacks ran first) and then block. Requests queued behind the stall start + * at its end, a full second or more later, so they stay out. */ -const SETTLE_TOLERANCE_MS = 50 +const STALL_EDGE_TOLERANCE_MS = 50 const g = globalThis as typeof globalThis & { _simInFlightRequestState?: InFlightState } if (!g._simInFlightRequestState) { @@ -121,7 +123,8 @@ export function takeInFlightWindow(): InFlightWindow { /** * Requests in flight for the whole stall window: started before it began and - * still running, or settled within {@link SETTLE_TOLERANCE_MS} of its end. + * still running, or settled by its end, each edge widened by + * {@link STALL_EDGE_TOLERANCE_MS}. `stallStart` is when the probe was due. * Settled ones come first (the likeliest source of a synchronous burst), then * longest-running first. Concurrency context only; see the module TSDoc. */ @@ -130,7 +133,8 @@ export function findRequestsSpanningStall( stallEnd: number ): RequestSpanningStall[] { const spansStall = (entry: TrackedRequest) => - entry.startedAt <= stallStart && (entry.endedAt ?? stallEnd) >= stallEnd - SETTLE_TOLERANCE_MS + entry.startedAt <= stallStart + STALL_EDGE_TOLERANCE_MS && + (entry.endedAt ?? stallEnd) >= stallEnd - STALL_EDGE_TOLERANCE_MS const matches: TrackedRequest[] = [] for (const entry of state.live.values()) if (spansStall(entry)) matches.push(entry) for (const entry of state.recent) if (entry && spansStall(entry)) matches.push(entry)