From 569826ecda332991dc803e62daaca02f1b7e818e Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 27 May 2026 21:26:03 -0500 Subject: [PATCH 1/7] feat: Replace WebSocket push with Server-Sent Events (SSE) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the WebSocket-based real-time push infrastructure with SSE: Backend: - Add SseConnection (Channel-based, bounded queue, write loop) - Add SseConnectionManager (keep-alive timer, connection limits) - Add SseMiddleware (handles GET /api/v2/push with auth) - Update MessageBusBroker to use SSE connections - Remove WebSocketConnectionManager and MessageBusBrokerMiddleware - Remove UseWebSockets() from Startup - Maintain throttle exemption for /api/v2/push Frontend (Svelte 5): - Add sse-client.svelte.ts using fetch+ReadableStream - Use Authorization header (no token-in-URL) - Exponential backoff, auth failure detection, visibility integration - Remove old web-socket-client.svelte.ts Frontend (Angular legacy): - Rewrite websocket-service.js to use fetch+ReadableStream SSE - Same event dispatch pattern ($rootScope.$emit) - Authorization header auth, exponential backoff Tests: - 11 backend unit tests (SseConnectionManager + SseBroker) - 6 integration tests (require Elasticsearch) - 15 frontend unit tests (connection lifecycle, reconnection, auth) - 5 throttling tests pass (no regression) Security audit passed: proper auth pipeline, bounded channels, per-user connection limits (10), no SSE injection, token revocation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> fix: Address SSE review blockers - dead conn cleanup, config rename, UserMembershipChanged Blocker fixes from adversarial review: 1. Config flag renamed: EnableWebSockets → EnablePush (backward compat reads both) - k8s api.yaml now sets EnablePush=true (SSE was accidentally disabled) - Test appsettings also enables push so integration tests can exercise SSE - Default remains true for environments without explicit config 2. Dead connection cleanup: WriteLoopAsync now cancels CTS in finally block - Ensures ConnectionAborted fires even when write loop exits due to IOException/ObjectDisposedException, so middleware cleanup always runs - Keep-alive sweep now proactively prunes connections that fail writes (previously only pruned already-canceled connections) - Prevents stale connection IDs from accumulating in Redis/IConnectionMapping and poisoning the per-user connection limit 3. Svelte UserMembershipChanged parity with Angular - Org/project queries are now invalidated when membership changes - Matches the Angular client's fan-out behavior 4. Documented soft connection limit (race is acceptable - distributed lock would add latency without security benefit for a cap of 10) 5. Added DroppedMessages counter to SseConnection for observability of backpressure events under burst load Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- k8s/exceptionless/templates/api.yaml | 4 +- src/Exceptionless.Core/Bootstrapper.cs | 4 +- .../Configuration/AppOptions.cs | 9 +- src/Exceptionless.Web/Bootstrapper.cs | 2 +- .../components/websocket/websocket-service.js | 247 +++++---- .../features/websockets/sse-client.svelte.ts | 276 ++++++++++ .../features/websockets/sse-client.test.ts | 344 ++++++++++++ .../websockets/web-socket-client.svelte.ts | 218 -------- .../websockets/web-socket-client.test.ts | 494 ------------------ .../ClientApp/src/routes/(app)/+layout.svelte | 28 +- .../Hubs/MessageBusBroker.cs | 36 +- .../Hubs/MessageBusBrokerMiddleware.cs | 154 ------ src/Exceptionless.Web/Hubs/SseConnection.cs | 271 ++++++++++ .../Hubs/SseConnectionManager.cs | 191 +++++++ src/Exceptionless.Web/Hubs/SseMiddleware.cs | 127 +++++ .../Hubs/WebSocketConnectionManager.cs | 193 ------- src/Exceptionless.Web/Startup.cs | 5 +- .../Utility/Handlers/ThrottlingMiddleware.cs | 4 +- .../Hubs/FakeHttpResponse.cs | 45 ++ .../Hubs/SseIntegrationTests.cs | 196 +++++++ tests/Exceptionless.Tests/Hubs/SseTests.cs | 459 ++++++++++++++++ .../Exceptionless.Tests/Hubs/TestWebSocket.cs | 53 -- .../Hubs/WebSocketConnectionManagerTests.cs | 99 ---- .../Hubs/WebSocketTests.cs | 121 ----- tests/Exceptionless.Tests/appsettings.yml | 2 +- 25 files changed, 2083 insertions(+), 1499 deletions(-) create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/websockets/web-socket-client.svelte.ts delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/websockets/web-socket-client.test.ts delete mode 100644 src/Exceptionless.Web/Hubs/MessageBusBrokerMiddleware.cs create mode 100644 src/Exceptionless.Web/Hubs/SseConnection.cs create mode 100644 src/Exceptionless.Web/Hubs/SseConnectionManager.cs create mode 100644 src/Exceptionless.Web/Hubs/SseMiddleware.cs delete mode 100644 src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs create mode 100644 tests/Exceptionless.Tests/Hubs/FakeHttpResponse.cs create mode 100644 tests/Exceptionless.Tests/Hubs/SseIntegrationTests.cs create mode 100644 tests/Exceptionless.Tests/Hubs/SseTests.cs delete mode 100644 tests/Exceptionless.Tests/Hubs/TestWebSocket.cs delete mode 100644 tests/Exceptionless.Tests/Hubs/WebSocketConnectionManagerTests.cs delete mode 100644 tests/Exceptionless.Tests/Hubs/WebSocketTests.cs diff --git a/k8s/exceptionless/templates/api.yaml b/k8s/exceptionless/templates/api.yaml index eb6979280a..f131ce253b 100644 --- a/k8s/exceptionless/templates/api.yaml +++ b/k8s/exceptionless/templates/api.yaml @@ -82,8 +82,8 @@ spec: {{- include "exceptionless.otel-env" . | indent 12 }} - name: RunJobsInProcess value: 'false' - - name: EnableWebSockets - value: 'false' + - name: EnablePush + value: 'true' {{- if (empty .Values.storage.connectionString) }} volumeMounts: - mountPath: "/app/storage" diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index bfd443ef16..84f86d8e60 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -224,8 +224,8 @@ public static void LogConfiguration(IServiceProvider serviceProvider, AppOptions if (String.IsNullOrEmpty(appOptions.StorageOptions.Provider)) logger.LogWarning("Distributed storage is NOT enabled on {MachineName}", Environment.MachineName); - if (!appOptions.EnableWebSockets) - logger.LogWarning("Web Sockets is NOT enabled on {MachineName}", Environment.MachineName); + if (!appOptions.EnablePush) + logger.LogWarning("Real-time push (SSE) is NOT enabled on {MachineName}", Environment.MachineName); if (String.IsNullOrEmpty(appOptions.EmailOptions.SmtpHost)) logger.LogWarning("Emails will NOT be sent until the SmtpHost is configured on {MachineName}", Environment.MachineName); diff --git a/src/Exceptionless.Core/Configuration/AppOptions.cs b/src/Exceptionless.Core/Configuration/AppOptions.cs index 818b27951d..05dd0f7c1b 100644 --- a/src/Exceptionless.Core/Configuration/AppOptions.cs +++ b/src/Exceptionless.Core/Configuration/AppOptions.cs @@ -55,7 +55,11 @@ public class AppOptions public bool EnableRepositoryNotifications { get; internal set; } - public bool EnableWebSockets { get; internal set; } + /// + /// Controls whether real-time push (SSE) is enabled. Reads from either 'EnablePush' + /// or legacy 'EnableWebSockets' config key for backward compatibility. + /// + public bool EnablePush { get; internal set; } public string? Version { get; internal set; } @@ -111,7 +115,8 @@ public static AppOptions ReadFromConfiguration(IConfiguration config) options.BulkBatchSize = config.GetValue(nameof(options.BulkBatchSize), 1000); options.EnableRepositoryNotifications = config.GetValue(nameof(options.EnableRepositoryNotifications), true); - options.EnableWebSockets = config.GetValue(nameof(options.EnableWebSockets), true); + // Support both new 'EnablePush' and legacy 'EnableWebSockets' config keys + options.EnablePush = config.GetValue(nameof(options.EnablePush), config.GetValue("EnableWebSockets", true)); try { diff --git a/src/Exceptionless.Web/Bootstrapper.cs b/src/Exceptionless.Web/Bootstrapper.cs index f83edd3615..8004ff3d2e 100644 --- a/src/Exceptionless.Web/Bootstrapper.cs +++ b/src/Exceptionless.Web/Bootstrapper.cs @@ -14,7 +14,7 @@ public class Bootstrapper { public static void RegisterServices(IServiceCollection services, AppOptions appOptions, ILoggerFactory loggerFactory) { - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js b/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js index 0cee86b9a6..f720bb459a 100644 --- a/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js +++ b/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js @@ -4,155 +4,152 @@ angular .module("exceptionless.websocket", ["app.config", "exceptionless", "exceptionless.auth"]) .factory("websocketService", function ($ExceptionlessClient, $rootScope, $timeout, authService, BASE_URL) { - var ResilientWebSocket = (function () { - function ResilientWebSocket(url, protocols) { - if (protocols === void 0) { - protocols = []; - } - this.reconnectInterval = 1000; - this.timeoutInterval = 2000; - this.forcedClose = false; - this.timedOut = false; - this.protocols = []; - this.onopen = function (event) {}; - this.onclose = function (event) {}; - this.onconnecting = function () {}; - this.onmessage = function (event) {}; - this.onerror = function (event) {}; - this.url = url; - this.protocols = protocols; - this.readyState = WebSocket.CONNECTING; - this.connect(false); - } - - ResilientWebSocket.prototype.connect = function (reconnectAttempt) { - var _this = this; - this.ws = new WebSocket(this.url, this.protocols); - this.onconnecting(); - var localWs = this.ws; - var timeout = setTimeout(function () { - _this.timedOut = true; - localWs.close(); - _this.timedOut = false; - }, this.timeoutInterval); - this.ws.onopen = function (event) { - clearTimeout(timeout); - _this.readyState = WebSocket.OPEN; - reconnectAttempt = false; - _this.onopen(event); - }; - this.ws.onclose = function (event) { - clearTimeout(timeout); - _this.ws = null; - if (_this.forcedClose) { - _this.readyState = WebSocket.CLOSED; - _this.onclose(event); - } else { - _this.readyState = WebSocket.CONNECTING; - _this.onconnecting(); - if (!reconnectAttempt && !_this.timedOut) { - _this.onclose(event); - } - setTimeout(function () { - _this.connect(true); - }, _this.reconnectInterval); - } - }; - this.ws.onmessage = function (event) { - _this.onmessage(event); - }; - this.ws.onerror = function (event) { - _this.onerror(event); - }; - }; - ResilientWebSocket.prototype.send = function (data) { - if (this.ws) { - return this.ws.send(data); - } - throw new Error("INVALID_STATE_ERR : Pausing to reconnect websocket"); - }; - ResilientWebSocket.prototype.close = function () { - if (this.ws) { - this.forcedClose = true; - this.ws.close(); - return true; - } - return false; - }; - ResilientWebSocket.prototype.refresh = function () { - if (this.ws) { - this.ws.close(); - return true; - } - - return false; - }; - - return ResilientWebSocket; - })(); - var source = "exceptionless.websocket.websocketService"; - var _connection; - var _websocketTimeout; + var _abortController; + var _reconnectTimeout; + var _reconnectAttempts = 0; + var _forcedClose = false; function start() { startDelayed(1); } function startDelayed(delay) { - function startImpl() { - _connection = new ResilientWebSocket(getPushUrl()); - _connection.onmessage = function (ev) { - var data = ev.data ? JSON.parse(ev.data) : null; - if (!data || !data.type) { - return; + if (_abortController || _reconnectTimeout) { + stop(); + } + + _reconnectTimeout = $timeout(function () { + _reconnectTimeout = null; + connect(); + }, delay || 1000); + } + + function connect() { + _forcedClose = false; + _abortController = new AbortController(); + var signal = _abortController.signal; + + var url = BASE_URL + "/api/v2/push"; + var token = authService.getToken(); + + fetch(url, { + headers: { + Accept: "text/event-stream", + Authorization: "Bearer " + token, + }, + signal: signal, + }) + .then(function (response) { + if (!response.ok) { + if (response.status === 401 || response.status === 403) { + // Auth failure - don't reconnect + return; + } + throw new Error("SSE connection failed: " + response.status); } - if (data.message && data.message.change_type >= 0) { - data.message.added = data.message.change_type === 0; - data.message.updated = data.message.change_type === 1; - data.message.deleted = data.message.change_type === 2; + _reconnectAttempts = 0; + var reader = response.body.getReader(); + var decoder = new TextDecoder(); + var buffer = ""; + + function readChunk() { + return reader.read().then(function (result) { + if (result.done) { + if (!_forcedClose) { + scheduleReconnect(); + } + return; + } + + buffer += decoder.decode(result.value, { stream: true }); + + var messages = buffer.split("\n\n"); + buffer = messages.pop() || ""; + + messages.forEach(function (message) { + if (!message.trim()) return; + + var lines = message.split("\n"); + var data = ""; + + lines.forEach(function (line) { + if (line.indexOf("data: ") === 0) { + data += line.slice(6); + } else if (line.indexOf("data:") === 0) { + data += line.slice(5); + } + // Comments (: keepalive) are ignored + }); + + if (data) { + var parsed = JSON.parse(data); + if (!parsed || !parsed.type) { + return; + } + + if (parsed.message && parsed.message.change_type >= 0) { + parsed.message.added = parsed.message.change_type === 0; + parsed.message.updated = parsed.message.change_type === 1; + parsed.message.deleted = parsed.message.change_type === 2; + } + + $rootScope.$emit(parsed.type, parsed.message); + + // This event is fired when a user is added or removed from an organization. + if (parsed.type === "UserMembershipChanged" && parsed.message && parsed.message.organization_id) { + $rootScope.$emit("OrganizationChanged", parsed.message); + $rootScope.$emit("ProjectChanged", parsed.message); + } + } + }); + + return readChunk(); + }); } - $rootScope.$emit(data.type, data.message); + return readChunk(); + }) + .catch(function (error) { + if (signal.aborted && _forcedClose) { + return; + } - // This event is fired when a user is added or removed from an organization. - if (data.type === "UserMembershipChanged" && data.message && data.message.organization_id) { - $rootScope.$emit("OrganizationChanged", data.message); - $rootScope.$emit("ProjectChanged", data.message); + if (!_forcedClose) { + scheduleReconnect(); } - }; - } + }); + } - if (_connection || _websocketTimeout) { - stop(); + function scheduleReconnect() { + var multiplier = 1; + + _reconnectAttempts++; + for (var attempt = 1; attempt < _reconnectAttempts; attempt++) { + multiplier *= 2; } - _websocketTimeout = $timeout(startImpl, delay || 1000); + var delay = Math.min(1000 * multiplier, 30000); + _reconnectTimeout = $timeout(function () { + _reconnectTimeout = null; + connect(); + }, delay); } function stop() { - if (_websocketTimeout) { - $timeout.cancel(_websocketTimeout); - _websocketTimeout = null; + if (_reconnectTimeout) { + $timeout.cancel(_reconnectTimeout); + _reconnectTimeout = null; } - if (_connection) { - _connection.close(); - _connection = null; + if (_abortController) { + _forcedClose = true; + _abortController.abort(); + _abortController = null; } } - function getPushUrl() { - var pushUrl = BASE_URL + "/api/v2/push?access_token=" + authService.getToken(); - var protoMatch = /^(https?):\/\//; - if (BASE_URL.startsWith("https:")) { - return pushUrl.replace(protoMatch, "wss://"); - } - - return pushUrl.replace(protoMatch, "ws://"); - } - var service = { start: start, startDelayed: startDelayed, diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts new file mode 100644 index 0000000000..b94a609a06 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts @@ -0,0 +1,276 @@ +import { DocumentVisibility } from '$shared/document-visibility.svelte'; + +import { accessToken } from '../auth/index.svelte'; + +export interface SseClientOptions { + /** + * Base URL for SSE connection (e.g., 'http://localhost:5200') + * If not provided, constructs from window.location + */ + baseUrl?: string; + /** + * Connection timeout in milliseconds + * Default: 10000ms (10 seconds) + */ + connectionTimeout?: number; + /** + * Custom reconnection delay calculator + * Default uses exponential backoff: 1s, 2s, 4s, 8s, 16s, max 30s + * For testing, can return 0 to reconnect immediately + */ + reconnectDelay?: (attempt: number) => number; +} + +// SSE connection state constants (same values as EventSource.*) +export const SSE_CONNECTING = 0; +export const SSE_OPEN = 1; +export const SSE_CLOSED = 2; + +export class SseClient { + public readyState = $state(SSE_CLOSED); + + /** + * Lazy getter for SSE URL. + */ + public get url(): string { + if (this._url === null) { + if (this._options.baseUrl) { + this._url = `${this._options.baseUrl}${this._path}`; + } else { + const { host, protocol } = window.location; + this._url = `${protocol}//${host}${this._path}`; + } + } + + return this._url; + } + + private _options: SseClientOptions; + private _path: string; + private _url: null | string = null; + private accessToken: null | string = null; + private authFailed: boolean = false; + private connectionTimeoutId: null | ReturnType = null; + private forcedClose: boolean = false; + private hasConnectedBefore: boolean = false; + private reconnectAttempts: number = 0; + private reconnectTimeoutId: null | ReturnType = null; + + private abortController: AbortController | null = null; + + /** + * @param path - SSE endpoint path (default: '/api/v2/push') + * @param options - Optional configuration + */ + constructor(path: string = '/api/v2/push', options: SseClientOptions = {}) { + this._path = path; + this._options = options; + + const visibility = new DocumentVisibility(); + + $effect(() => { + if (this.accessToken !== accessToken.current) { + this.accessToken = accessToken.current; + this.reconnectAttempts = 0; + this.authFailed = false; + this.close(); + this.forcedClose = false; // Allow reconnect with new token + } else if (!visibility.visible) { + this.close(); + this.forcedClose = false; // Allow reconnect when visible again + } + + if (this.accessToken && visibility.visible && this.readyState === SSE_CLOSED && this.reconnectTimeoutId === null && !this.authFailed && !this.forcedClose) { + this.connect(); + } + }); + } + + public close(): boolean { + clearTimeout(this.reconnectTimeoutId!); + this.reconnectTimeoutId = null; + clearTimeout(this.connectionTimeoutId!); + this.connectionTimeoutId = null; + + if (this.abortController) { + this.forcedClose = true; + this.abortController.abort(); + this.abortController = null; + this.readyState = SSE_CLOSED; + return true; + } + + return false; + } + + public connect() { + const isReconnect: boolean = this.hasConnectedBefore; + + this.readyState = SSE_CONNECTING; + this.forcedClose = false; + + this.abortController = new AbortController(); + const { signal } = this.abortController; + + this.onConnecting(isReconnect); + + // Connection timeout + clearTimeout(this.connectionTimeoutId!); + const timeout = this._options.connectionTimeout ?? 10000; + this.connectionTimeoutId = setTimeout(() => { + this.connectionTimeoutId = null; + if (this.readyState === SSE_CONNECTING) { + console.warn(`[SseClient] Connection timeout after ${timeout}ms`); + this.abortController?.abort(); + } + }, timeout); + + this.startStream(signal, isReconnect); + } + + public onClose: () => void = () => {}; + public onConnecting: (isReconnect: boolean) => void = () => {}; + public onError: (error: unknown) => void = () => {}; + public onMessage: (ev: MessageEvent) => void = () => {}; + public onOpen: (isReconnect: boolean) => void = () => {}; + + /** + * Calculate reconnection delay using exponential backoff + */ + private getReconnectDelay(attempt: number): number { + if (this._options.reconnectDelay) { + return this._options.reconnectDelay(attempt); + } + + // Default: exponential backoff 1s, 2s, 4s, 8s, 16s, max 30s + return Math.min(1000 * Math.pow(2, attempt - 1), 30000); + } + + private async startStream(signal: AbortSignal, isReconnect: boolean) { + try { + const token = this.accessToken ?? accessToken.current; + const response = await fetch(this.url, { + headers: { + Accept: 'text/event-stream', + Authorization: `Bearer ${token}` + }, + signal + }); + + clearTimeout(this.connectionTimeoutId!); + this.connectionTimeoutId = null; + + if (!response.ok) { + // Auth failures - don't reconnect + if (response.status === 401 || response.status === 403) { + console.warn('[SseClient] Auth failure, not reconnecting', { status: response.status }); + this.authFailed = true; + this.readyState = SSE_CLOSED; + this.onClose(); + return; + } + + // Rate limited + if (response.status === 429) { + console.warn('[SseClient] Rate limited, will retry'); + this.scheduleReconnect(); + return; + } + + throw new Error(`SSE connection failed: ${response.status}`); + } + + if (!response.body) { + throw new Error('SSE response has no body'); + } + + this.readyState = SSE_OPEN; + this.reconnectAttempts = 0; + this.hasConnectedBefore = true; + this.onOpen(isReconnect); + + // Read the stream + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + + if (done) { + break; + } + + buffer += decoder.decode(value, { stream: true }); + + // Process complete SSE messages (separated by double newline) + const messages = buffer.split('\n\n'); + buffer = messages.pop() ?? ''; + + for (const message of messages) { + if (!message.trim()) continue; + + // Parse SSE format + const lines = message.split('\n'); + let data = ''; + + for (const line of lines) { + if (line.startsWith('data: ')) { + data += line.slice(6); + } else if (line.startsWith('data:')) { + data += line.slice(5); + } else if (line.startsWith(':')) { + // Comment (keep-alive), ignore + continue; + } + } + + if (data) { + // Create a MessageEvent-like object for compatibility + const event = new MessageEvent('message', { data }); + this.onMessage(event); + } + } + } + } catch (error: unknown) { + clearTimeout(this.connectionTimeoutId!); + this.connectionTimeoutId = null; + + if (signal.aborted && this.forcedClose) { + // Intentional close - don't reconnect + this.readyState = SSE_CLOSED; + this.onClose(); + return; + } + + if (signal.aborted) { + // Timeout or other abort - try reconnect + this.scheduleReconnect(); + return; + } + + console.error('[SseClient] Stream error', error); + this.onError(error); + } + + // Stream ended (server closed connection) - reconnect + if (!this.forcedClose) { + this.scheduleReconnect(); + } + } + + private scheduleReconnect() { + this.reconnectAttempts++; + const delay = this.getReconnectDelay(this.reconnectAttempts); + + this.readyState = SSE_CONNECTING; + this.onConnecting(true); + this.onClose(); + + clearTimeout(this.reconnectTimeoutId!); + this.reconnectTimeoutId = setTimeout(() => { + this.reconnectTimeoutId = null; + this.connect(); + }, delay); + } +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts new file mode 100644 index 0000000000..3b8a8907d0 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts @@ -0,0 +1,344 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { SSE_CLOSED, SSE_CONNECTING, SSE_OPEN, SseClient, type SseClientOptions } from './sse-client.svelte'; + +// Mock the auth module +vi.mock('../auth/index.svelte', () => ({ + accessToken: { + current: 'test-token-123' + } +})); + +// Mock DocumentVisibility to always return visible +vi.mock('$shared/document-visibility.svelte', () => { + return { + DocumentVisibility: class { + visible = true; + } + }; +}); + +// Helper to create a mock fetch response that streams SSE data +function createSseResponse(events: string[] = [], options: { status?: number; delay?: number } = {}) { + const { status = 200, delay = 0 } = options; + + return new Response( + new ReadableStream({ + async start(controller) { + for (const event of events) { + if (delay > 0) { + await new Promise((resolve) => setTimeout(resolve, delay)); + } + controller.enqueue(new TextEncoder().encode(event)); + } + controller.close(); + } + }), + { + status, + headers: { 'Content-Type': 'text/event-stream' } + } + ); +} + +// Creates a response whose stream stays open indefinitely (for testing open connections) +function createOpenSseResponse(initialEvents: string[] = []) { + return new Response( + new ReadableStream({ + start(controller) { + for (const event of initialEvents) { + controller.enqueue(new TextEncoder().encode(event)); + } + // intentionally never close + } + }), + { + status: 200, + headers: { 'Content-Type': 'text/event-stream' } + } + ); +} + +function createClient(options?: SseClientOptions): SseClient { + return new SseClient('/api/v2/push', { + baseUrl: 'http://localhost:5200', + reconnectDelay: () => 50, + ...options + }); +} + +describe('SseClient', () => { + let fetchMock: ReturnType; + let activeClients: SseClient[] = []; + + beforeEach(() => { + fetchMock = vi.fn(); + global.fetch = fetchMock; + }); + + afterEach(() => { + for (const client of activeClients) { + client.close(); + } + activeClients = []; + vi.restoreAllMocks(); + }); + + function trackedClient(options?: SseClientOptions): SseClient { + const client = createClient(options); + activeClients.push(client); + return client; + } + + describe('Connection Lifecycle', () => { + it('should connect successfully and call onOpen', async () => { + const onOpen = vi.fn(); + fetchMock.mockResolvedValue(createOpenSseResponse([': keepalive\n\n'])); + + const client = trackedClient(); + client.onOpen = onOpen; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(fetchMock).toHaveBeenCalledWith( + 'http://localhost:5200/api/v2/push', + expect.objectContaining({ + headers: expect.objectContaining({ + Accept: 'text/event-stream', + Authorization: 'Bearer test-token-123' + }) + }) + ); + expect(onOpen).toHaveBeenCalledWith(false); + + client.close(); + }); + + it('should set readyState to CONNECTING then OPEN', async () => { + fetchMock.mockResolvedValue(createOpenSseResponse([': keepalive\n\n'])); + + const client = trackedClient(); + client.connect(); + + expect(client.readyState).toBe(SSE_CONNECTING); + + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(client.readyState).toBe(SSE_OPEN); + + client.close(); + }); + + it('should call onConnecting with isReconnect=false on first connection', async () => { + const onConnecting = vi.fn(); + fetchMock.mockResolvedValue(createSseResponse([])); + + const client = trackedClient(); + client.onConnecting = onConnecting; + client.connect(); + + expect(onConnecting).toHaveBeenCalledWith(false); + client.close(); + }); + }); + + describe('Disconnection', () => { + it('should close when close() is called', async () => { + // Create a response that never closes + fetchMock.mockResolvedValue( + new Response( + new ReadableStream({ + start() { + // Never close - simulate long-lived connection + } + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } } + ) + ); + + const client = trackedClient(); + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 50)); + const result = client.close(); + + expect(result).toBe(true); + expect(client.readyState).toBe(SSE_CLOSED); + }); + + it('should return false when closing already closed connection', () => { + const client = trackedClient(); + const result = client.close(); + + expect(result).toBe(false); + }); + + it('should NOT reconnect after manual close', async () => { + // Use a stream that stays open (never closes) so we can test manual close + fetchMock.mockResolvedValue( + new Response( + new ReadableStream({ + start() { + // intentionally never close - stream stays open + } + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } } + ) + ); + + const client = trackedClient(); + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 50)); + client.close(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(client.readyState).toBe(SSE_CLOSED); + // fetch should only be called once (no reconnect) + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + }); + + describe('Auth Failure Handling', () => { + it('should NOT reconnect on 401', async () => { + fetchMock.mockImplementation(() => Promise.resolve(new Response(null, { status: 401 }))); + + const onClose = vi.fn(); + const client = trackedClient(); + client.onClose = onClose; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(onClose).toHaveBeenCalledTimes(1); + expect(client.readyState).toBe(SSE_CLOSED); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('should NOT reconnect on 403', async () => { + fetchMock.mockImplementation(() => Promise.resolve(new Response(null, { status: 403 }))); + + const client = trackedClient(); + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(client.readyState).toBe(SSE_CLOSED); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + }); + + describe('Reconnection Logic', () => { + it('should reconnect when stream ends normally', async () => { + let callCount = 0; + fetchMock.mockImplementation(() => { + callCount++; + return Promise.resolve(createSseResponse([': keepalive\n\n'])); + }); + + const client = trackedClient({ baseUrl: 'http://localhost:5200', reconnectDelay: () => 10 }); + client.connect(); + + // Wait for initial connection + stream end + reconnect + await new Promise((resolve) => setTimeout(resolve, 200)); + + expect(callCount).toBeGreaterThan(1); + client.close(); + }); + + it('should use custom reconnectDelay', async () => { + const reconnectDelay = vi.fn(() => 50); + fetchMock.mockResolvedValue(createSseResponse([])); + + const client = trackedClient({ baseUrl: 'http://localhost:5200', reconnectDelay }); + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 150)); + + expect(reconnectDelay).toHaveBeenCalled(); + client.close(); + }); + }); + + describe('Message Handling', () => { + it('should parse SSE data messages and call onMessage', async () => { + const onMessage = vi.fn(); + const sseData = 'data: {"type":"StackChanged","message":{"id":"123"}}\n\n'; + fetchMock.mockResolvedValue(createSseResponse([sseData])); + + const client = trackedClient(); + client.onMessage = onMessage; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(onMessage).toHaveBeenCalledWith( + expect.objectContaining({ + data: '{"type":"StackChanged","message":{"id":"123"}}' + }) + ); + client.close(); + }); + + it('should ignore keep-alive comments', async () => { + const onMessage = vi.fn(); + const sseData = ': keepalive\n\ndata: {"type":"test","message":{}}\n\n'; + fetchMock.mockResolvedValue(createSseResponse([sseData])); + + const client = trackedClient(); + client.onMessage = onMessage; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Should only get the data message, not the keepalive + expect(onMessage).toHaveBeenCalledTimes(1); + expect(onMessage).toHaveBeenCalledWith( + expect.objectContaining({ + data: '{"type":"test","message":{}}' + }) + ); + client.close(); + }); + + it('should handle multiple messages in one chunk', async () => { + const onMessage = vi.fn(); + const sseData = 'data: {"type":"A","message":{}}\n\ndata: {"type":"B","message":{}}\n\n'; + fetchMock.mockResolvedValue(createSseResponse([sseData])); + + const client = trackedClient(); + client.onMessage = onMessage; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(onMessage).toHaveBeenCalledTimes(2); + client.close(); + }); + + it('should handle messages split across chunks', async () => { + const onMessage = vi.fn(); + fetchMock.mockResolvedValue(createSseResponse(['data: {"type":"Sp', 'lit","message":{}}\n\n'])); + + const client = trackedClient(); + client.onMessage = onMessage; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(onMessage).toHaveBeenCalledWith( + expect.objectContaining({ + data: '{"type":"Split","message":{}}' + }) + ); + client.close(); + }); + }); + + describe('URL Construction', () => { + it('should construct correct SSE URL with base URL', () => { + const client = trackedClient({ baseUrl: 'http://localhost:5200' }); + expect(client.url).toBe('http://localhost:5200/api/v2/push'); + }); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/web-socket-client.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/web-socket-client.svelte.ts deleted file mode 100644 index c816c5bcb5..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/web-socket-client.svelte.ts +++ /dev/null @@ -1,218 +0,0 @@ -import { DocumentVisibility } from '$shared/document-visibility.svelte'; - -import { accessToken } from '../auth/index.svelte'; - -export interface WebSocketClientOptions { - /** - * Base URL for WebSocket connection (e.g., 'ws://localhost:1234') - * If not provided, constructs from window.location - */ - baseUrl?: string; - /** - * Connection timeout in milliseconds - * Default: 10000ms (10 seconds) - */ - connectionTimeout?: number; - /** - * Custom reconnection delay calculator - * Default uses exponential backoff: 1s, 2s, 4s, 8s, 16s, max 30s - * For testing, can return 0 to reconnect immediately - */ - reconnectDelay?: (attempt: number) => number; -} - -export class WebSocketClient { - public readyState = $state(WebSocket.CLOSED); - - /** - * Lazy getter for WebSocket URL. - * Constructed on first access. Uses baseUrl from options if provided, otherwise constructs from window.location. - */ - public get url(): string { - if (this._url === null) { - if (this._options.baseUrl) { - this._url = `${this._options.baseUrl}${this._path}`; - } else { - const { host, protocol } = window.location; - const wsProtocol = protocol === 'https:' ? 'wss://' : 'ws://'; - this._url = `${wsProtocol}${host}${this._path}`; - } - } - - return this._url; - } - - private _options: WebSocketClientOptions; - private _path: string; - private _url: null | string = null; - private accessToken: null | string = null; - private connectionTimeoutId: null | ReturnType = null; - private forcedClose: boolean = false; - private hasConnectedBefore: boolean = false; - private reconnectAttempts: number = 0; - private reconnectTimeoutId: null | ReturnType = null; - - private ws: null | WebSocket = null; - - /** - * @param path - WebSocket path (default: '/api/v2/push') - * @param options - Optional configuration - */ - constructor(path: string = '/api/v2/push', options: WebSocketClientOptions = {}) { - this._path = path; - this._options = options; - - const visibility = new DocumentVisibility(); - - $effect(() => { - if (this.accessToken !== accessToken.current) { - this.accessToken = accessToken.current; - this.reconnectAttempts = 0; // Reset backoff on token change - this.close(); - } else if (!visibility.visible) { - this.close(); - } - - // Only auto-connect if we're fully closed and don't have a pending reconnect attempt - // Don't try to connect if we're CONNECTING, OPEN, or CLOSING - if (this.accessToken && visibility.visible && this.readyState === WebSocket.CLOSED && this.reconnectTimeoutId === null) { - this.connect(); - } - }); - } - - public close(): boolean { - clearTimeout(this.reconnectTimeoutId!); - this.reconnectTimeoutId = null; - clearTimeout(this.connectionTimeoutId!); - this.connectionTimeoutId = null; - - if (this.ws) { - this.forcedClose = true; - this.ws.close(); - return true; - } - - return false; - } - - public connect() { - // isReconnect means: have we successfully connected before? - const isReconnect: boolean = this.hasConnectedBefore; - - // Reset state - this.readyState = WebSocket.CONNECTING; - this.forcedClose = false; - - try { - this.ws = new WebSocket(`${this.url}?access_token=${this.accessToken}`); - this.onConnecting(isReconnect); - } catch (error) { - console.error('[WebSocketClient] Failed to create WebSocket', error); - throw error; - } - - // Connection timeout: if we don't connect within configured timeout, force close - clearTimeout(this.connectionTimeoutId!); - const timeout = this._options.connectionTimeout ?? 10000; - this.connectionTimeoutId = setTimeout(() => { - this.connectionTimeoutId = null; - if (this.ws && this.readyState === WebSocket.CONNECTING) { - console.warn(`[WebSocketClient] Connection timeout after ${timeout}ms`); - this.ws.close(); - } - }, timeout); - - this.ws.onopen = (event: Event) => { - clearTimeout(this.connectionTimeoutId!); - this.connectionTimeoutId = null; - this.readyState = WebSocket.OPEN; - this.reconnectAttempts = 0; // Reset backoff on successful connection - this.hasConnectedBefore = true; // Mark that we've connected successfully - this.onOpen(event, isReconnect); - }; - - this.ws.onclose = (event: CloseEvent) => { - clearTimeout(this.connectionTimeoutId!); - this.connectionTimeoutId = null; - this.ws = null; - - if (this.forcedClose) { - this.readyState = WebSocket.CLOSED; - this.onClose(event); - return; - } - - // Don't retry on authentication/authorization failures - // Code 1008 (Policy Violation) is explicit auth failure - // Code 1006 (Abnormal Closure) during handshake could be 401/403 - // Codes 4xxx are custom application codes (e.g., 4401=401, 4403=403) - const isAuthFailure = event.code === 1008 || (event.code === 1006 && event.wasClean === false) || (event.code >= 4400 && event.code < 4500); - if (isAuthFailure) { - console.warn('[WebSocketClient] Auth failure detected, not reconnecting', { - code: event.code, - reason: event.reason - }); - this.readyState = WebSocket.CLOSED; - this.onClose(event); - return; // Let the auth system handle redirect to login - } - - // Calculate reconnection delay with exponential backoff - this.reconnectAttempts++; - const delay = this.getReconnectDelay(this.reconnectAttempts); - - this.onConnecting(true); // Always true when reconnecting after close - this.onClose(event); - - // Schedule reconnect - clear any existing timeout first - clearTimeout(this.reconnectTimeoutId!); - this.reconnectTimeoutId = setTimeout(() => { - this.reconnectTimeoutId = null; - this.connect(); - }, delay); - }; - - this.ws.onmessage = (event) => { - this.onMessage(event); - }; - - this.ws.onerror = (event) => { - console.error('[WebSocketClient] onerror triggered', { - event, - readyState: this.readyState, - reconnectAttempts: this.reconnectAttempts - }); - this.onError(event); - }; - } - - public onClose: (ev: CloseEvent) => void = () => {}; - - public onConnecting: (isReconnect: boolean) => void = () => {}; - public onError: (ev: Event) => void = () => {}; - public onMessage: (ev: MessageEvent) => void = () => {}; - - public onOpen: (ev: Event, isReconnect: boolean) => void = () => {}; - - public send(data: Parameters[0]) { - if (this.ws) { - return this.ws.send(data); - } else { - throw new Error('INVALID_STATE_ERR : Pausing to reconnect websocket'); - } - } - - /** - * Calculate reconnection delay using exponential backoff - * Can be overridden via options for testing - */ - private getReconnectDelay(attempt: number): number { - if (this._options.reconnectDelay) { - return this._options.reconnectDelay(attempt); - } - - // Default: exponential backoff 1s, 2s, 4s, 8s, 16s, max 30s - return Math.min(1000 * Math.pow(2, attempt - 1), 30000); - } -} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/web-socket-client.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/web-socket-client.test.ts deleted file mode 100644 index c7138d1206..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/web-socket-client.test.ts +++ /dev/null @@ -1,494 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import WS from 'vitest-websocket-mock'; - -import { WebSocketClient, type WebSocketClientOptions } from './web-socket-client.svelte'; - -// Mock the auth module -vi.mock('../auth/index.svelte', () => ({ - accessToken: { - current: 'test-token-123' - } -})); - -// Mock DocumentVisibility to always return visible -vi.mock('$shared/document-visibility.svelte', () => { - return { - DocumentVisibility: class { - visible = true; - } - }; -}); - -let server: WS; - -beforeEach(() => { - server = new WS('ws://localhost:1234/api/v2/push'); -}); - -afterEach(() => { - WS.clean(); -}); - -function createClient(path?: string, options?: WebSocketClientOptions): WebSocketClient { - return new WebSocketClient(path, { - baseUrl: 'ws://localhost:1234', - reconnectDelay: () => 0, - ...options - }); -} - -describe('WebSocketClient', () => { - describe('Connection Lifecycle', () => { - it('should connect successfully', async () => { - const client = createClient(); - client.connect(); - await server.connected; - - expect(client.readyState).toBe(WebSocket.OPEN); - client.close(); - }); - - it('should set readyState to CONNECTING then OPEN', async () => { - const client = createClient(); - client.connect(); - - expect(client.readyState).toBe(WebSocket.CONNECTING); - await server.connected; - expect(client.readyState).toBe(WebSocket.OPEN); - - client.close(); - }); - - it('should call onConnecting with isReconnect=false on first connection', async () => { - const onConnecting = vi.fn(); - const client = createClient(); - client.onConnecting = onConnecting; - - client.connect(); - expect(onConnecting).toHaveBeenCalledWith(false); - await server.connected; - - client.close(); - }); - - it('should call onOpen with isReconnect=false on first connection', async () => { - const onOpen = vi.fn(); - const client = createClient(); - client.onOpen = onOpen; - - client.connect(); - await server.connected; - - expect(onOpen).toHaveBeenCalledWith(expect.anything(), false); - client.close(); - }); - - it('should handle multiple connect calls gracefully', async () => { - const client = createClient(); - - client.connect(); - client.connect(); - client.connect(); - - await server.connected; - expect(client.readyState).toBe(WebSocket.OPEN); - - client.close(); - }); - - it('should use custom connectionTimeout option', async () => { - const onConnecting = vi.fn(); - const client = new WebSocketClient('/api/v2/push', { - baseUrl: 'ws://localhost:9999', - connectionTimeout: 75, // Very short timeout - reconnectDelay: () => 1000 // Prevent immediate reconnect - }); - client.onConnecting = onConnecting; - - client.connect(); - expect(client.readyState).toBe(WebSocket.CONNECTING); - - // Wait for custom timeout to expire and close to be triggered - await new Promise((resolve) => setTimeout(resolve, 150)); - - // onConnecting was called with isReconnect=false for initial connect - expect(onConnecting).toHaveBeenCalledWith(false); - }); - }); - - describe('Disconnection', () => { - it('should close WebSocket when close() is called', async () => { - const client = createClient(); - client.connect(); - await server.connected; - - const result = client.close(); - await new Promise((resolve) => setTimeout(resolve, 10)); - - expect(result).toBe(true); - expect(client.readyState).toBe(WebSocket.CLOSED); - }); - - it('should return false when closing already closed connection', () => { - const client = createClient(); - client.close(); - const result = client.close(); - - expect(result).toBe(false); - }); - - it('should call onClose callback', async () => { - const onClose = vi.fn(); - const client = createClient(); - client.onClose = onClose; - - client.connect(); - await server.connected; - - server.close({ code: 1000, reason: 'Test', wasClean: true }); - await new Promise((resolve) => setTimeout(resolve, 10)); - - expect(onClose).toHaveBeenCalledWith( - expect.objectContaining({ - code: 1000, - reason: 'Test', - wasClean: true - }) - ); - }); - - it('should NOT reconnect after manual close', async () => { - const client = createClient(); - client.connect(); - await server.connected; - - client.close(); - await new Promise((resolve) => setTimeout(resolve, 50)); - - expect(client.readyState).toBe(WebSocket.CLOSED); - }); - }); - - describe('Reconnection Logic', () => { - it('should NOT reconnect on policy violation (code 1008) - auth failure', async () => { - const client = createClient(); - const onClose = vi.fn(); - client.onClose = onClose; - - client.connect(); - await server.connected; - - server.close({ code: 1008, reason: 'Policy Violation', wasClean: false }); - await new Promise((resolve) => setTimeout(resolve, 50)); - - expect(onClose).toHaveBeenCalledWith( - expect.objectContaining({ - code: 1008, - reason: 'Policy Violation', - wasClean: false - }) - ); - expect(client.readyState).toBe(WebSocket.CLOSED); - }); - - it('should NOT reconnect on abnormal closure (code 1006, wasClean=false) - connection lost unexpectedly', async () => { - const client = createClient(); - const onClose = vi.fn(); - client.onClose = onClose; - - client.connect(); - await server.connected; - - server.close({ code: 1006, reason: 'Abnormal Closure', wasClean: false }); - await new Promise((resolve) => setTimeout(resolve, 50)); - - expect(onClose).toHaveBeenCalledWith( - expect.objectContaining({ - code: 1006, - reason: 'Abnormal Closure', - wasClean: false - }) - ); - expect(client.readyState).toBe(WebSocket.CLOSED); - }); - - it('should NOT reconnect on unauthorized (code 4401) - 401 HTTP equivalent', async () => { - const client = createClient(); - const onClose = vi.fn(); - client.onClose = onClose; - - client.connect(); - await server.connected; - - server.close({ code: 4401, reason: 'Unauthorized', wasClean: false }); - await new Promise((resolve) => setTimeout(resolve, 50)); - - expect(onClose).toHaveBeenCalledWith( - expect.objectContaining({ - code: 4401, - reason: 'Unauthorized', - wasClean: false - }) - ); - expect(client.readyState).toBe(WebSocket.CLOSED); - }); - - it('should NOT reconnect on forbidden (code 4403) - 403 HTTP equivalent', async () => { - const client = createClient(); - const onClose = vi.fn(); - client.onClose = onClose; - - client.connect(); - await server.connected; - - server.close({ code: 4403, reason: 'Forbidden', wasClean: false }); - await new Promise((resolve) => setTimeout(resolve, 50)); - - expect(onClose).toHaveBeenCalledWith( - expect.objectContaining({ - code: 4403, - reason: 'Forbidden', - wasClean: false - }) - ); - expect(client.readyState).toBe(WebSocket.CLOSED); - }); - - it('should reconnect on normal closure (code 1000) - server initiated graceful close', async () => { - const onConnecting = vi.fn(); - const client = createClient(); - client.onConnecting = onConnecting; - - client.connect(); - await server.connected; - onConnecting.mockClear(); - - server.close({ code: 1000, reason: 'Normal Closure', wasClean: true }); - await new Promise((resolve) => setTimeout(resolve, 10)); - await server.connected; - - expect(onConnecting).toHaveBeenCalledWith(true); - client.close(); - }); - - it('should reconnect on going away (code 1001) - server restart', async () => { - const onConnecting = vi.fn(); - const client = createClient(); - client.onConnecting = onConnecting; - - client.connect(); - await server.connected; - onConnecting.mockClear(); - - server.close({ code: 1001, reason: 'Going Away', wasClean: true }); - await new Promise((resolve) => setTimeout(resolve, 10)); - await server.connected; - - expect(onConnecting).toHaveBeenCalledWith(true); - client.close(); - }); - - it('should call onConnecting with isReconnect=true on reconnection', async () => { - const onConnecting = vi.fn(); - const client = createClient(); - client.onConnecting = onConnecting; - - client.connect(); - await server.connected; - expect(onConnecting).toHaveBeenCalledWith(false); - onConnecting.mockClear(); - - server.close({ code: 1000, reason: 'Test', wasClean: true }); - await new Promise((resolve) => setTimeout(resolve, 10)); - - expect(onConnecting).toHaveBeenCalledWith(true); - await server.connected; - client.close(); - }); - }); - - describe('Message Handling', () => { - it('should send messages when connected', async () => { - const client = createClient(); - client.connect(); - await server.connected; - - client.send('test message'); - await expect(server).toReceiveMessage('test message'); - - client.close(); - }); - - it('should throw error when sending while disconnected', () => { - const client = createClient(); - - expect(() => client.send('test')).toThrow('INVALID_STATE_ERR'); - }); - - it('should call onMessage callback when receiving messages', async () => { - const onMessage = vi.fn(); - const client = createClient(); - client.onMessage = onMessage; - - client.connect(); - await server.connected; - - server.send('test data'); - await new Promise((resolve) => setTimeout(resolve, 10)); - - expect(onMessage).toHaveBeenCalledWith( - expect.objectContaining({ - data: 'test data' - }) - ); - - client.close(); - }); - - it('should receive JSON messages', async () => { - const onMessage = vi.fn(); - const client = createClient(); - client.onMessage = onMessage; - - client.connect(); - await server.connected; - - const message = JSON.stringify({ data: 'hello', type: 'test' }); - server.send(message); - await new Promise((resolve) => setTimeout(resolve, 10)); - - expect(onMessage).toHaveBeenCalledWith( - expect.objectContaining({ - data: message - }) - ); - - client.close(); - }); - }); - - describe('Error Handling', () => { - it('should call onError callback', async () => { - const onError = vi.fn(); - const client = createClient(); - client.onError = onError; - - client.connect(); - await server.connected; - - server.error(); - await new Promise((resolve) => setTimeout(resolve, 10)); - - expect(onError).toHaveBeenCalled(); - client.close(); - }); - }); - - describe('URL Construction', () => { - it('should construct correct WebSocket URL', () => { - const client = createClient('/api/v2/push'); - - expect(client.url).toBe('ws://localhost:1234/api/v2/push'); - }); - - it('should use custom base URL', async () => { - const customClient = new WebSocketClient('/api/v2/push', { - baseUrl: 'ws://custom-host:5000', - reconnectDelay: () => 0 - }); - - const customServer = new WS('ws://custom-host:5000/api/v2/push'); - customClient.connect(); - await customServer.connected; - - expect(customClient.readyState).toBe(WebSocket.OPEN); - - customClient.close(); - customServer.close(); - }); - - it('should handle custom path', async () => { - const client = createClient('/custom/path'); - const customServer = new WS('ws://localhost:1234/custom/path'); - - client.connect(); - await customServer.connected; - - expect(client.readyState).toBe(WebSocket.OPEN); - - client.close(); - customServer.close(); - }); - }); - - describe('Options - getReconnectDelay', () => { - it('should use custom getReconnectDelay from options', async () => { - const getReconnectDelay = vi.fn(() => 100); - const client = new WebSocketClient('/api/v2/push', { - baseUrl: 'ws://localhost:1234', - reconnectDelay: getReconnectDelay - }); - - client.connect(); - await server.connected; - - server.close({ code: 1000, reason: 'Test', wasClean: true }); - await new Promise((resolve) => setTimeout(resolve, 10)); - await server.connected; - - expect(getReconnectDelay).toHaveBeenCalled(); - client.close(); - }); - - it('should use immediate reconnection with getReconnectDelay: () => 0', async () => { - const onConnecting = vi.fn(); - const client = createClient(); - client.onConnecting = onConnecting; - - client.connect(); - await server.connected; - onConnecting.mockClear(); - - const start = Date.now(); - server.close({ code: 1000, reason: 'Test', wasClean: true }); - - // Wait for reconnection attempt - await new Promise((resolve) => setTimeout(resolve, 50)); - - // Verify reconnection happened quickly (within 50ms) - const elapsed = Date.now() - start; - expect(onConnecting).toHaveBeenCalledWith(true); - expect(elapsed).toBeLessThan(100); - - client.close(); - }); - }); - - describe('Edge Cases', () => { - it('should handle rapid connect/disconnect cycles', async () => { - const client = createClient(); - - client.connect(); - client.close(); - client.connect(); - await server.connected; - - expect(client.readyState).toBe(WebSocket.OPEN); - client.close(); - }); - - it('should maintain connection state correctly', async () => { - const client = createClient(); - - expect(client.readyState).toBe(WebSocket.CLOSED); - - client.connect(); - await server.connected; - expect(client.readyState).toBe(WebSocket.OPEN); - - client.close(); - await new Promise((resolve) => setTimeout(resolve, 10)); - expect(client.readyState).toBe(WebSocket.CLOSED); - }); - }); -}); diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte index 9b6e0982bd..3c21033a9d 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte @@ -29,7 +29,7 @@ import { getGravatarFromCurrentUser } from '$features/users/gravatar.svelte'; import { invalidateWebhookQueries } from '$features/webhooks/api.svelte'; import { isEntityChangedType, type WebSocketMessageType } from '$features/websockets/models'; - import { WebSocketClient } from '$features/websockets/web-socket-client.svelte'; + import { SseClient } from '$features/websockets/sse-client.svelte'; import { useMiddleware } from '@exceptionless/fetchclient'; import { useQueryClient } from '@tanstack/svelte-query'; import { tick } from 'svelte'; @@ -153,11 +153,15 @@ } } - // This event is fired when a user is added or removed from an organization. - // if (data.type === "UserMembershipChanged" && data.message?.organization_id) { - // $rootScope.$emit("OrganizationChanged", data.message); - // $rootScope.$emit("ProjectChanged", data.message); - // } + // When a user is added or removed from an organization, invalidate org/project caches + // so the UI reflects the membership change without a manual reload. + if (data.type === 'UserMembershipChanged') { + const msg = data.message as { organization_id?: string }; + if (msg?.organization_id) { + await invalidateOrganizationQueries(queryClient, msg); + await invalidateProjectQueries(queryClient, msg); + } + } } // Close Sidebar on page change on mobile @@ -185,7 +189,7 @@ } }); - // WebSocket + keyboard shortcuts — only depends on token, not navigation + // SSE + keyboard shortcuts — only depends on token, not navigation $effect(() => { const currentToken = accessToken.current; @@ -246,15 +250,15 @@ document.addEventListener('keydown', handleKeydown, { capture: true }); - const ws = new WebSocketClient(); - ws.onMessage = onMessage; - ws.onOpen = (_, isReconnect) => { + const sse = new SseClient(); + sse.onMessage = onMessage; + sse.onOpen = (isReconnect) => { if (isReconnect) { queryClient.invalidateQueries(); document.dispatchEvent( new CustomEvent('refresh', { bubbles: true, - detail: 'WebSocket Connected' + detail: 'SSE Connected' }) ); } @@ -262,7 +266,7 @@ return () => { document.removeEventListener('keydown', handleKeydown, { capture: true }); - ws?.close(); + sse?.close(); }; }); diff --git a/src/Exceptionless.Web/Hubs/MessageBusBroker.cs b/src/Exceptionless.Web/Hubs/MessageBusBroker.cs index fe059935db..2c0010cdf1 100644 --- a/src/Exceptionless.Web/Hubs/MessageBusBroker.cs +++ b/src/Exceptionless.Web/Hubs/MessageBusBroker.cs @@ -13,13 +13,13 @@ public sealed class MessageBusBroker : IStartupAction { private static readonly string TokenTypeName = nameof(Token); private static readonly string UserTypeName = nameof(User); - private readonly WebSocketConnectionManager _connectionManager; + private readonly SseConnectionManager _connectionManager; private readonly IConnectionMapping _connectionMapping; private readonly IMessageSubscriber _subscriber; private readonly AppOptions _options; private readonly ILogger _logger; - public MessageBusBroker(WebSocketConnectionManager connectionManager, IConnectionMapping connectionMapping, IMessageSubscriber subscriber, AppOptions options, ILogger logger) + public MessageBusBroker(SseConnectionManager connectionManager, IConnectionMapping connectionMapping, IMessageSubscriber subscriber, AppOptions options, ILogger logger) { _connectionManager = connectionManager; _connectionMapping = connectionMapping; @@ -30,7 +30,7 @@ public MessageBusBroker(WebSocketConnectionManager connectionManager, IConnectio public async Task RunAsync(CancellationToken shutdownToken = default) { - if (!_options.EnableWebSockets) + if (!_options.EnablePush) return; _logger.LogDebug("Subscribing to message bus notifications"); @@ -91,7 +91,7 @@ internal async Task OnEntityChangedAsync(EntityChanged ec, CancellationToken can var userConnectionIds = await _connectionMapping.GetUserIdConnectionsAsync(entityChanged.Id); _logger.LogTrace("Sending {UserTypeName} message to user: {UserId} (to {UserConnectionCount} connections)", UserTypeName, entityChanged.Id, userConnectionIds.Count); foreach (string connectionId in userConnectionIds) - await TypedSendAsync(connectionId, entityChanged); + TypedSend(connectionId, entityChanged); return; } @@ -106,11 +106,11 @@ internal async Task OnEntityChangedAsync(EntityChanged ec, CancellationToken can { var userConnectionIds = await _connectionMapping.GetUserIdConnectionsAsync(userId); - // Auth token removed = logout. Close sockets immediately without sending; + // Auth token removed = logout. Close connections immediately without sending; // there is no point delivering a message to a connection we are about to tear down. if (isAuthToken && entityChanged.ChangeType is ChangeType.Removed) { - _logger.LogTrace("Auth token removed for user {UserId}; closing {ConnectionCount} WebSocket connection(s)", userId, userConnectionIds.Count); + _logger.LogTrace("Auth token removed for user {UserId}; closing {ConnectionCount} SSE connection(s)", userId, userConnectionIds.Count); string? organizationId = entityChanged.OrganizationId; foreach (string connectionId in userConnectionIds) { @@ -118,7 +118,7 @@ internal async Task OnEntityChangedAsync(EntityChanged ec, CancellationToken can await _connectionMapping.GroupRemoveAsync(organizationId, connectionId); await _connectionMapping.UserIdRemoveAsync(userId, connectionId); - await _connectionManager.RemoveWebSocketAsync(connectionId); + await _connectionManager.RemoveConnectionAsync(connectionId); } return; @@ -126,7 +126,7 @@ internal async Task OnEntityChangedAsync(EntityChanged ec, CancellationToken can _logger.LogTrace("Sending {TokenTypeName} message for user: {UserId} (to {UserConnectionCount} connections)", TokenTypeName, userId, userConnectionIds.Count); foreach (string connectionId in userConnectionIds) - await TypedSendAsync(connectionId, entityChanged); + TypedSend(connectionId, entityChanged); return; } @@ -172,13 +172,15 @@ private Task OnPlanChangedAsync(PlanChanged planChanged, CancellationToken cance private Task OnReleaseNotificationAsync(ReleaseNotification notification, CancellationToken cancellationToken = default) { _logger.LogTrace("Sending release notification message: {Message}", notification.Message); - return TypedBroadcastAsync(notification); + TypedBroadcast(notification); + return Task.CompletedTask; } private Task OnSystemNotificationAsync(SystemNotification notification, CancellationToken cancellationToken = default) { _logger.LogTrace("Sending system notification message: {Message}", notification.Message); - return TypedBroadcastAsync(notification); + TypedBroadcast(notification); + return Task.CompletedTask; } private async Task GroupSendAsync(string group, object value) @@ -190,22 +192,22 @@ private async Task GroupSendAsync(string group, object value) return; } - await TypedSendAsync(connectionIds.ToList(), value); + TypedSend(connectionIds, value); } - public Task TypedSendAsync(string connectionId, object value) + public void TypedSend(string connectionId, object value) { - return _connectionManager.SendMessageAsync(connectionId, new TypedMessage { Type = GetMessageType(value), Message = value }); + _connectionManager.SendMessage(connectionId, new TypedMessage { Type = GetMessageType(value), Message = value }); } - public Task TypedSendAsync(IList connectionIds, object value) + public void TypedSend(IEnumerable connectionIds, object value) { - return _connectionManager.SendMessageAsync(connectionIds, new TypedMessage { Type = GetMessageType(value), Message = value }); + _connectionManager.SendMessage(connectionIds, new TypedMessage { Type = GetMessageType(value), Message = value }); } - public Task TypedBroadcastAsync(object value) + public void TypedBroadcast(object value) { - return _connectionManager.SendMessageToAllAsync(new TypedMessage { Type = GetMessageType(value), Message = value }); + _connectionManager.SendMessageToAll(new TypedMessage { Type = GetMessageType(value), Message = value }); } private static string GetMessageType(object value) diff --git a/src/Exceptionless.Web/Hubs/MessageBusBrokerMiddleware.cs b/src/Exceptionless.Web/Hubs/MessageBusBrokerMiddleware.cs deleted file mode 100644 index 227a5372ae..0000000000 --- a/src/Exceptionless.Web/Hubs/MessageBusBrokerMiddleware.cs +++ /dev/null @@ -1,154 +0,0 @@ -using System.Net.WebSockets; -using System.Text; -using Exceptionless.Core.Extensions; -using Exceptionless.Core.Utility; - -namespace Exceptionless.Web.Hubs; - -public class MessageBusBrokerMiddleware -{ - private readonly ILogger _logger; - private readonly WebSocketConnectionManager _connectionManager; - private readonly IConnectionMapping _connectionMapping; - private readonly RequestDelegate _next; - - public MessageBusBrokerMiddleware(RequestDelegate next, WebSocketConnectionManager connectionManager, IConnectionMapping connectionMapping, ILogger logger) - { - _next = next; - _connectionManager = connectionManager; - _connectionMapping = connectionMapping; - _logger = logger; - } - - public async Task Invoke(HttpContext context) - { - if (!context.WebSockets.IsWebSocketRequest || !context.User.IsAuthenticated()) - { - await _next(context); - return; - } - - using var socket = await context.WebSockets.AcceptWebSocketAsync(); - string connectionId = _connectionManager.AddWebSocket(socket); - await OnConnected(context, socket, connectionId); - bool disconnected = false; - - try - { - await ReceiveAsync(socket, async (result, data) => - { - switch (result.MessageType) - { - case WebSocketMessageType.Text: - _logger.LogTrace("WebSocket got message {ConnectionId}", connectionId); - // ignore incoming messages - return; - case WebSocketMessageType.Close: - await OnDisconnected(context, socket, connectionId); - await _connectionManager.RemoveWebSocketAsync(connectionId); - disconnected = true; - return; - } - }); - } - catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) { } - - // This will be hit when the connection is lost. - if (!disconnected) - { - await OnDisconnected(context, socket, connectionId); - await _connectionManager.RemoveWebSocketAsync(connectionId); - } - } - - private async Task OnConnected(HttpContext context, WebSocket socket, string connectionId) - { - _logger.LogTrace("WebSocket connected {ConnectionId} ({State})", connectionId, socket.State); - - try - { - foreach (string organizationId in context.User.GetOrganizationIds()) - await _connectionMapping.GroupAddAsync(organizationId, connectionId); - - string? userId = context.User.GetUserId(); - if (!String.IsNullOrEmpty(userId)) - await _connectionMapping.UserIdAddAsync(userId, connectionId); - } - catch (Exception ex) - { - _logger.LogError(ex, "OnConnected Error: {Message}", ex.Message); - throw; - } - } - - private async Task OnDisconnected(HttpContext context, WebSocket socket, string connectionId) - { - _logger.LogTrace("WebSocket disconnected {ConnectionId} ({State})", connectionId, socket.State); - - try - { - foreach (string organizationId in context.User.GetOrganizationIds()) - await _connectionMapping.GroupRemoveAsync(organizationId, connectionId); - - string? userId = context.User.GetUserId(); - if (!String.IsNullOrEmpty(userId)) - await _connectionMapping.UserIdRemoveAsync(userId, connectionId); - } - catch (Exception ex) - { - _logger.LogError(ex, "OnDisconnected Error: {Message}", ex.Message); - throw; - } - } - - private async Task ReceiveAsync(WebSocket socket, Func handleMessage) - { - var buffer = new ArraySegment(new byte[1024 * 4]); - var result = await socket.ReceiveAsync(buffer, CancellationToken.None); - LogFrame(result, buffer.Array); - - while (!result.CloseStatus.HasValue) - { - string data; - - using (var ms = new MemoryStream()) - { - do - { - result = await socket.ReceiveAsync(buffer, CancellationToken.None); - LogFrame(result, buffer.Array); - - if (buffer.Array is not null) - await ms.WriteAsync(buffer.Array, buffer.Offset, result.Count); - } while (!result.EndOfMessage); - - ms.Seek(0, SeekOrigin.Begin); - - using (var reader = new StreamReader(ms, Encoding.UTF8)) - data = await reader.ReadToEndAsync(); - } - - await handleMessage(result, data); - } - } - - private void LogFrame(WebSocketReceiveResult frame, byte[]? buffer) - { - if (!_logger.IsEnabled(LogLevel.Debug)) - return; - - if (frame.CloseStatus.HasValue) - { - _logger.LogDebug("Close: {CloseStatus} {CloseStatusDescription}", frame.CloseStatus.Value, frame.CloseStatusDescription); - } - else - { - string? content = "<>"; - if (frame.MessageType == WebSocketMessageType.Text) - content = buffer is not null ? Encoding.UTF8.GetString(buffer, 0, frame.Count) : null; - - _logger.LogDebug("Received Frame {MessageType}: length={FrameCount}, end={FrameEndOfMessage}: {Content}", frame.MessageType, frame.Count, frame.EndOfMessage, content); - } - - } -} diff --git a/src/Exceptionless.Web/Hubs/SseConnection.cs b/src/Exceptionless.Web/Hubs/SseConnection.cs new file mode 100644 index 0000000000..075b4ec2cb --- /dev/null +++ b/src/Exceptionless.Web/Hubs/SseConnection.cs @@ -0,0 +1,271 @@ +using Foundatio.Serializer; + +namespace Exceptionless.Web.Hubs; + +/// +/// Represents a single SSE client connection. Owns a write loop that serializes +/// all sends through a bounded dedup queue, preventing concurrent writes to the +/// underlying HttpResponse stream. +/// +/// Design: delivery is best-effort. Under burst load, oldest unwritten events are +/// dropped. This is intentional — SSE push messages trigger client-side cache +/// invalidation refetches, so a dropped message results in stale cache until the +/// next push or manual refresh, not data loss. +/// +/// Deduplication: messages with the same serialized payload are coalesced — if an +/// identical message is already queued, the newer duplicate is skipped. This reduces +/// redundant client refreshes during burst scenarios (e.g., rapid entity updates). +/// +public sealed class SseConnection : IAsyncDisposable +{ + private static readonly byte[] KeepAliveBytes = ": keepalive\n\n"u8.ToArray(); + private readonly HttpResponse _response; + private readonly ITextSerializer _serializer; + private readonly DedupQueue _queue; + private readonly CancellationTokenSource _cts; + private readonly CancellationToken _connectionAborted; + private readonly Task _writeLoop; + private readonly ILogger _logger; + private long _droppedMessages; + private long _dedupedMessages; + private int _disposeState; + + public string ConnectionId { get; } + public CancellationToken ConnectionAborted => _connectionAborted; + + /// Number of messages dropped due to backpressure (queue full). + public long DroppedMessages => Interlocked.Read(ref _droppedMessages); + + /// Number of messages skipped due to deduplication. + public long DedupedMessages => Interlocked.Read(ref _dedupedMessages); + + public SseConnection(string connectionId, HttpResponse response, ITextSerializer serializer, CancellationToken requestAborted, ILogger logger, int capacity = 64) + { + ConnectionId = connectionId; + _response = response; + _serializer = serializer; + _logger = logger; + _queue = new DedupQueue(capacity); + + _cts = CancellationTokenSource.CreateLinkedTokenSource(requestAborted); + _connectionAborted = _cts.Token; + _writeLoop = Task.Run(() => WriteLoopAsync(_cts.Token)); + } + + /// + /// Enqueue a message to be written. Returns false if the connection is closed. + /// If an identical message (same serialized payload) is already queued, the new + /// one is skipped (deduped) and this returns true. + /// + public bool TryWrite(object message) + { + if (_cts.IsCancellationRequested) + return false; + + string data = _serializer.SerializeToString(message); + var result = _queue.TryEnqueue(new SseEvent { Data = data, DedupeKey = data }); + + if (result == EnqueueResult.Deduped) + { + Interlocked.Increment(ref _dedupedMessages); + return true; + } + + if (result == EnqueueResult.DroppedOldest) + Interlocked.Increment(ref _droppedMessages); + + return true; + } + + /// + /// Send a keep-alive comment to prevent proxy/LB timeouts. + /// Keep-alives bypass dedup (always enqueued). + /// + public bool TryWriteKeepAlive() + { + if (_cts.IsCancellationRequested) + return false; + + _queue.TryEnqueue(SseEvent.KeepAlive); + return true; + } + + /// + /// Abort the connection. The write loop will complete and the middleware will clean up. + /// + public void Abort() + { + try { _cts.Cancel(); } + catch (ObjectDisposedException) { } + _queue.Complete(); + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposeState, 1) != 0) + return; + + Abort(); + try + { + await _writeLoop.ConfigureAwait(false); + } + catch (OperationCanceledException) { } + finally + { + _queue.Dispose(); + _cts.Dispose(); + } + } + + private async Task WriteLoopAsync(CancellationToken ct) + { + try + { + while (!ct.IsCancellationRequested) + { + var evt = await _queue.DequeueAsync(ct); + if (evt is null) + break; // Queue completed + + byte[] bytes; + if (evt.Value.IsKeepAlive) + { + bytes = KeepAliveBytes; + } + else + { + bytes = System.Text.Encoding.UTF8.GetBytes($"data: {evt.Value.Data}\n\n"); + } + + await _response.Body.WriteAsync(bytes, ct); + await _response.Body.FlushAsync(ct); + } + } + catch (OperationCanceledException) { } + catch (ObjectDisposedException) { } + catch (IOException) { } + catch (Exception ex) + { + _logger.LogDebug(ex, "SSE write loop ended for connection {ConnectionId}", ConnectionId); + } + finally + { + // Always signal ConnectionAborted so the middleware's Task.Delay unblocks + // and cleanup (IConnectionMapping removal) happens reliably. + _queue.Complete(); + if (!_cts.IsCancellationRequested) + { + try { _cts.Cancel(); } + catch (ObjectDisposedException) { } + } + } + } + + internal readonly record struct SseEvent + { + public string? Data { get; init; } + + /// + /// Key used for deduplication. If null, no dedup is applied (e.g., keep-alive). + /// For data messages, this is the serialized payload — identical payloads trigger + /// the same client-side cache invalidation, so coalescing is safe. + /// + public string? DedupeKey { get; init; } + + public bool IsKeepAlive { get; init; } + public static SseEvent KeepAlive => new() { IsKeepAlive = true }; + } + + internal enum EnqueueResult + { + Enqueued, + Deduped, + DroppedOldest + } + + /// + /// Bounded FIFO queue with deduplication. Thread-safe for multiple writers and a single reader. + /// When full, drops the oldest item to make room (like BoundedChannelFullMode.DropOldest). + /// If an item with the same DedupeKey is already queued, the new item is skipped. + /// + internal sealed class DedupQueue : IDisposable + { + private readonly object _lock = new(); + private readonly LinkedList _list = new(); + private readonly Dictionary> _index = new(); + private readonly SemaphoreSlim _signal = new(0); + private readonly int _capacity; + private bool _completed; + + public DedupQueue(int capacity) + { + _capacity = capacity; + } + + public EnqueueResult TryEnqueue(SseEvent evt) + { + lock (_lock) + { + if (_completed) + return EnqueueResult.Enqueued; + + // Dedup check: if same key is already queued, skip + if (evt.DedupeKey is not null && _index.ContainsKey(evt.DedupeKey)) + return EnqueueResult.Deduped; + + var result = EnqueueResult.Enqueued; + + // Enforce capacity: drop oldest if full + if (_list.Count >= _capacity) + { + var oldest = _list.First!; + _list.RemoveFirst(); + if (oldest.Value.DedupeKey is not null) + _index.Remove(oldest.Value.DedupeKey); + result = EnqueueResult.DroppedOldest; + } + + var node = _list.AddLast(evt); + if (evt.DedupeKey is not null) + _index[evt.DedupeKey] = node; + + _signal.Release(); + return result; + } + } + + public async Task DequeueAsync(CancellationToken ct) + { + await _signal.WaitAsync(ct); + + lock (_lock) + { + if (_list.Count == 0) + return null; // Completed + + var node = _list.First!; + _list.RemoveFirst(); + if (node.Value.DedupeKey is not null) + _index.Remove(node.Value.DedupeKey); + return node.Value; + } + } + + public void Complete() + { + lock (_lock) + { + if (_completed) + return; + _completed = true; + _signal.Release(); // Wake up the reader so it sees null + } + } + + public void Dispose() + { + _signal.Dispose(); + } + } +} diff --git a/src/Exceptionless.Web/Hubs/SseConnectionManager.cs b/src/Exceptionless.Web/Hubs/SseConnectionManager.cs new file mode 100644 index 0000000000..ba0a7f4297 --- /dev/null +++ b/src/Exceptionless.Web/Hubs/SseConnectionManager.cs @@ -0,0 +1,191 @@ +using System.Collections.Concurrent; +using Exceptionless.Core; +using Foundatio.Serializer; + +namespace Exceptionless.Web.Hubs; + +/// +/// Manages active SSE connections. Replaces WebSocketConnectionManager. +/// Sends keep-alive comments every 15 seconds to prevent proxy/LB disconnects. +/// Proactively prunes dead connections during keep-alive sweeps. +/// +public sealed class SseConnectionManager : IDisposable +{ + private readonly ConcurrentDictionary _connections = new(); + private readonly ConcurrentDictionary> _pendingDisposals = new(); + private readonly Timer? _timer; + private readonly ITextSerializer _serializer; + private readonly ILogger _logger; + + /// + /// Maximum number of concurrent connections per user to prevent resource exhaustion. + /// This is a soft limit — under concurrent connection bursts, a few extra connections + /// may be admitted briefly. This is acceptable because the alternative (distributed + /// locking) would add latency to every SSE connect without meaningful security benefit. + /// + public int MaxConnectionsPerUser { get; init; } = 10; + + public SseConnectionManager(AppOptions options, ITextSerializer serializer, ILoggerFactory loggerFactory) + { + _serializer = serializer; + _logger = loggerFactory.CreateLogger(); + + if (!options.EnablePush) + return; + + _timer = new Timer(SendKeepAlive, null, TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(15)); + } + + private void SendKeepAlive(object? state) + { + if (_connections.IsEmpty) + return; + + int sent = 0; + int pruned = 0; + + foreach (var (connectionId, connection) in _connections) + { + if (connection.ConnectionAborted.IsCancellationRequested) + { + TryRemove(connectionId); + pruned++; + continue; + } + + if (!connection.TryWriteKeepAlive()) + { + // Write failed — connection is dead, prune it + TryRemove(connectionId); + pruned++; + } + else + { + sent++; + } + } + + if (_logger.IsEnabled(LogLevel.Trace)) + _logger.LogTrace("SSE keep-alive: sent={SentCount}, pruned={PrunedCount}, active={ActiveCount}", sent, pruned, _connections.Count); + } + + public SseConnection? GetConnectionById(string connectionId) + { + return _connections.TryGetValue(connectionId, out var connection) ? connection : null; + } + + public ICollection GetAll() + { + return _connections.Values; + } + + public int ConnectionCount => _connections.Count; + + public SseConnection AddConnection(string connectionId, HttpResponse response, CancellationToken requestAborted) + { + var connection = new SseConnection(connectionId, response, _serializer, requestAborted, _logger); + _connections.TryAdd(connectionId, connection); + return connection; + } + + public async Task RemoveConnectionAsync(string connectionId) + { + if (_connections.TryRemove(connectionId, out var connection)) + { + await DisposeConnectionAsync(connectionId, connection).ConfigureAwait(false); + return; + } + + if (_pendingDisposals.TryGetValue(connectionId, out var pendingDisposal)) + await pendingDisposal.Value.ConfigureAwait(false); + } + + private void TryRemove(string connectionId) + { + if (_connections.TryRemove(connectionId, out var connection)) + _ = ObserveDisposeAsync(connectionId, DisposeConnectionAsync(connectionId, connection)); + } + + public bool SendMessage(string connectionId, object message) + { + if (!_connections.TryGetValue(connectionId, out var connection)) + return false; + + if (connection.ConnectionAborted.IsCancellationRequested) + { + TryRemove(connectionId); + return false; + } + + return connection.TryWrite(message); + } + + public void SendMessage(IEnumerable connectionIds, object message) + { + foreach (string connectionId in connectionIds) + SendMessage(connectionId, message); + } + + public void SendMessageToAll(object message) + { + foreach (var (connectionId, connection) in _connections) + { + if (connection.ConnectionAborted.IsCancellationRequested) + { + TryRemove(connectionId); + continue; + } + + connection.TryWrite(message); + } + } + + public void Dispose() + { + _timer?.Dispose(); + + var disposeTasks = new HashSet(); + + foreach (var (connectionId, connection) in _connections) + { + if (_connections.TryRemove(connectionId, out var activeConnection)) + disposeTasks.Add(DisposeConnectionAsync(connectionId, activeConnection)); + } + + foreach (var pendingDisposal in _pendingDisposals.Values) + disposeTasks.Add(pendingDisposal.Value); + + Task.WhenAll(disposeTasks).GetAwaiter().GetResult(); + } + + private Task DisposeConnectionAsync(string connectionId, SseConnection connection) + { + var pendingDisposal = _pendingDisposals.GetOrAdd(connectionId, _ => new Lazy(() => DisposeConnectionCoreAsync(connectionId, connection))); + return pendingDisposal.Value; + } + + private async Task DisposeConnectionCoreAsync(string connectionId, SseConnection connection) + { + try + { + connection.Abort(); + await connection.DisposeAsync().ConfigureAwait(false); + } + finally + { + _pendingDisposals.TryRemove(connectionId, out _); + } + } + + private async Task ObserveDisposeAsync(string connectionId, Task disposeTask) + { + try + { + await disposeTask.ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "SSE connection cleanup failed for {ConnectionId}", connectionId); + } + } +} diff --git a/src/Exceptionless.Web/Hubs/SseMiddleware.cs b/src/Exceptionless.Web/Hubs/SseMiddleware.cs new file mode 100644 index 0000000000..d3a09b29d4 --- /dev/null +++ b/src/Exceptionless.Web/Hubs/SseMiddleware.cs @@ -0,0 +1,127 @@ +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Utility; + +namespace Exceptionless.Web.Hubs; + +/// +/// Handles SSE connections at /api/v2/push. Replaces MessageBusBrokerMiddleware (WebSocket). +/// Accepts authenticated GET requests, sets SSE response headers, registers the connection +/// with IConnectionMapping, and holds the response open until the client disconnects. +/// +public class SseMiddleware +{ + private static readonly PathString _sseEndpoint = new("/api/v2/push"); + private readonly ILogger _logger; + private readonly SseConnectionManager _connectionManager; + private readonly IConnectionMapping _connectionMapping; + private readonly RequestDelegate _next; + + public SseMiddleware(RequestDelegate next, SseConnectionManager connectionManager, IConnectionMapping connectionMapping, ILogger logger) + { + _next = next; + _connectionManager = connectionManager; + _connectionMapping = connectionMapping; + _logger = logger; + } + + public async Task Invoke(HttpContext context) + { + if (!context.Request.Path.StartsWithSegments(_sseEndpoint, StringComparison.Ordinal) + || !HttpMethods.IsGet(context.Request.Method)) + { + await _next(context); + return; + } + + if (!context.User.IsAuthenticated()) + { + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + return; + } + + string? userId = context.User.GetUserId(); + if (String.IsNullOrEmpty(userId)) + { + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + return; + } + + // Enforce per-user connection limit + var existingConnections = await _connectionMapping.GetUserIdConnectionsAsync(userId); + if (existingConnections.Count >= _connectionManager.MaxConnectionsPerUser) + { + _logger.LogWarning("User {UserId} exceeded max SSE connections ({Max})", userId, _connectionManager.MaxConnectionsPerUser); + context.Response.StatusCode = StatusCodes.Status429TooManyRequests; + return; + } + + // Set SSE response headers + context.Response.Headers.ContentType = "text/event-stream"; + context.Response.Headers.CacheControl = "no-cache, no-store"; + context.Response.Headers["X-Accel-Buffering"] = "no"; // nginx + context.Response.Headers.Connection = "keep-alive"; + + // Disable response buffering + var bufferingFeature = context.Features.Get(); + bufferingFeature?.DisableBuffering(); + + string connectionId = Guid.NewGuid().ToString("N"); + var connection = _connectionManager.AddConnection(connectionId, context.Response, context.RequestAborted); + + await OnConnected(context, connectionId); + + try + { + // Send initial connected event + connection.TryWrite(new { type = "Connected", message = new { connection_id = connectionId } }); + + // Hold the response open until the client disconnects or the connection is aborted + await Task.Delay(Timeout.Infinite, connection.ConnectionAborted); + } + catch (OperationCanceledException) { } + finally + { + await OnDisconnected(context, connectionId); + await _connectionManager.RemoveConnectionAsync(connectionId); + } + } + + private async Task OnConnected(HttpContext context, string connectionId) + { + _logger.LogTrace("SSE connected {ConnectionId}", connectionId); + + try + { + foreach (string organizationId in context.User.GetOrganizationIds()) + await _connectionMapping.GroupAddAsync(organizationId, connectionId); + + string? userId = context.User.GetUserId(); + if (!String.IsNullOrEmpty(userId)) + await _connectionMapping.UserIdAddAsync(userId, connectionId); + } + catch (Exception ex) + { + _logger.LogError(ex, "SSE OnConnected Error: {Message}", ex.Message); + throw; + } + } + + private async Task OnDisconnected(HttpContext context, string connectionId) + { + _logger.LogTrace("SSE disconnected {ConnectionId}", connectionId); + + try + { + foreach (string organizationId in context.User.GetOrganizationIds()) + await _connectionMapping.GroupRemoveAsync(organizationId, connectionId); + + string? userId = context.User.GetUserId(); + if (!String.IsNullOrEmpty(userId)) + await _connectionMapping.UserIdRemoveAsync(userId, connectionId); + } + catch (Exception ex) + { + _logger.LogError(ex, "SSE OnDisconnected Error: {Message}", ex.Message); + } + } +} diff --git a/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs b/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs deleted file mode 100644 index 662aabff39..0000000000 --- a/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs +++ /dev/null @@ -1,193 +0,0 @@ -using System.Collections.Concurrent; -using System.Net.WebSockets; -using System.Text; -using Exceptionless.Core; -using Foundatio.Serializer; - -namespace Exceptionless.Web.Hubs; - -public class WebSocketConnectionManager : IDisposable -{ - private static readonly ArraySegment _keepAliveMessage = new(Encoding.ASCII.GetBytes("{}"), 0, 2); - private readonly ConcurrentDictionary _connections = new(); - private readonly Timer? _timer; - private readonly ITextSerializer _serializer; - private readonly ILogger _logger; - - public WebSocketConnectionManager(AppOptions options, ITextSerializer serializer, ILoggerFactory loggerFactory) - { - _serializer = serializer; - _logger = loggerFactory.CreateLogger(); - - if (!options.EnableWebSockets) - return; - - _timer = new Timer(KeepAlive, null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10)); - } - - private void KeepAlive(object? state) - { - if (_connections is { IsEmpty: true, Count: 0 }) - return; - - Task.Factory.StartNew(async () => - { - var sockets = GetAll(); - var openSockets = sockets.Where(s => s.State == WebSocketState.Open).ToArray(); - _logger.LogTrace("Sending web socket keep alive to {OpenSocketsCount} open connections of {SocketCount} total connections", openSockets.Length, sockets.Count); - - foreach (var socket in openSockets) - { - try - { - await socket.SendAsync(buffer: _keepAliveMessage, - messageType: WebSocketMessageType.Text, - endOfMessage: true, - cancellationToken: CancellationToken.None); - } - catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) - { - // NOTE: This will not remove it from the ConnectionMappings. - await RemoveWebSocketAsync(socket); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error sending keep alive socket message: {Message}", ex.Message); - } - } - }); - } - - public WebSocket? GetWebSocketById(string connectionId) - { - return _connections.TryGetValue(connectionId, out var socket) ? socket : null; - } - - public ICollection GetAll() - { - return _connections.Values; - } - - public string GetConnectionId(WebSocket socket) - { - return _connections.FirstOrDefault(p => p.Value == socket).Key; - } - - public string AddWebSocket(WebSocket socket) - { - string connectionId = Guid.NewGuid().ToString("N"); - _connections.TryAdd(connectionId, socket); - return connectionId; - } - - private Task RemoveWebSocketAsync(WebSocket socket) - { - string id = GetConnectionId(socket); - if (String.IsNullOrEmpty(id) || !_connections.TryRemove(id, out var _)) - return Task.CompletedTask; - - return CloseWebSocketAsync(socket); - } - - public Task RemoveWebSocketAsync(string id) - { - if (!_connections.TryRemove(id, out var socket)) - return Task.CompletedTask; - - return CloseWebSocketAsync(socket); - } - - private async Task CloseWebSocketAsync(WebSocket socket) - { - if (!CanSendWebSocketMessage(socket)) - return; - - try - { - await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Closed by manager", CancellationToken.None); - } - catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) - { - // Ignored - } - catch (Exception ex) - { - _logger.LogError(ex, "Error closing web socket: {Message}", ex.Message); - } - } - - private Task SendMessageAsync(WebSocket socket, object message) - { - if (!CanSendWebSocketMessage(socket)) - return Task.CompletedTask; - - string serializedMessage = _serializer.SerializeToString(message); - Task.Factory.StartNew(async () => - { - if (!CanSendWebSocketMessage(socket)) - return; - - try - { - await socket.SendAsync(buffer: new ArraySegment(Encoding.ASCII.GetBytes(serializedMessage), 0, serializedMessage.Length), - messageType: WebSocketMessageType.Text, - endOfMessage: true, - cancellationToken: CancellationToken.None); - } - catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) - { - // Ignored - } - catch (Exception ex) - { - _logger.LogError(ex, "Error sending socket message: {Message}", ex.Message); - } - }); - - return Task.CompletedTask; - } - - public Task SendMessageAsync(string connectionId, object message) - { - var socket = GetWebSocketById(connectionId); - return socket is not null ? SendMessageAsync(socket, message) : Task.CompletedTask; - } - - public Task SendMessageAsync(IEnumerable connectionIds, object message) - { - return Task.WhenAll(connectionIds.Select(id => - { - var socket = GetWebSocketById(id); - return socket is not null ? SendMessageAsync(socket, message) : Task.CompletedTask; - })); - } - - public async Task SendMessageToAllAsync(object message, bool throwOnError = true) - { - foreach (var socket in GetAll()) - { - if (!CanSendWebSocketMessage(socket)) - continue; - - try - { - await SendMessageAsync(socket, message); - } - catch (Exception) - { - if (throwOnError) - throw; - } - } - } - - private bool CanSendWebSocketMessage(WebSocket socket) - { - return socket.State != WebSocketState.Aborted && socket.State != WebSocketState.Closed && socket.State != WebSocketState.CloseSent; - } - - public void Dispose() - { - _timer?.Dispose(); - } -} diff --git a/src/Exceptionless.Web/Startup.cs b/src/Exceptionless.Web/Startup.cs index 4a37df1963..b94ddcd57b 100644 --- a/src/Exceptionless.Web/Startup.cs +++ b/src/Exceptionless.Web/Startup.cs @@ -305,10 +305,9 @@ ApplicationException applicationException when applicationException.Message.Cont // Reject event posts in organizations over their max event limits. app.UseMiddleware(); - if (options.EnableWebSockets) + if (options.EnablePush) { - app.UseWebSockets(); - app.UseMiddleware(); + app.UseMiddleware(); } app.UseEndpoints(endpoints => diff --git a/src/Exceptionless.Web/Utility/Handlers/ThrottlingMiddleware.cs b/src/Exceptionless.Web/Utility/Handlers/ThrottlingMiddleware.cs index ee7722b814..4c1d69ec1a 100644 --- a/src/Exceptionless.Web/Utility/Handlers/ThrottlingMiddleware.cs +++ b/src/Exceptionless.Web/Utility/Handlers/ThrottlingMiddleware.cs @@ -21,7 +21,7 @@ public class ThrottlingMiddleware private static readonly PathString _v1ProjectConfigPath = new("/api/v1/project/config"); private static readonly PathString _v2ProjectConfigPath = new("/api/v2/projects/config"); private static readonly PathString _heartbeatPath = new("/api/v2/events/session/heartbeat"); - private static readonly PathString _webSocketPath = new("/api/v2/push"); + private static readonly PathString _ssePath = new("/api/v2/push"); public ThrottlingMiddleware(RequestDelegate next, ICacheClient cacheClient, ThrottlingOptions options, TimeProvider timeProvider) @@ -111,7 +111,7 @@ private bool IsUnthrottledRoute(HttpContext context) return context.Request.Path.StartsWithSegments(_v2ProjectConfigPath, StringComparison.Ordinal) || context.Request.Path.StartsWithSegments(_heartbeatPath, StringComparison.Ordinal) - || context.Request.Path.StartsWithSegments(_webSocketPath, StringComparison.Ordinal) + || context.Request.Path.StartsWithSegments(_ssePath, StringComparison.Ordinal) || context.Request.Path.StartsWithSegments(_v1ProjectConfigPath, StringComparison.Ordinal); } } diff --git a/tests/Exceptionless.Tests/Hubs/FakeHttpResponse.cs b/tests/Exceptionless.Tests/Hubs/FakeHttpResponse.cs new file mode 100644 index 0000000000..3a48a8c642 --- /dev/null +++ b/tests/Exceptionless.Tests/Hubs/FakeHttpResponse.cs @@ -0,0 +1,45 @@ +using System.IO.Pipelines; +using System.Text; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Features; + +namespace Exceptionless.Tests.Hubs; + +/// +/// A minimal fake HttpResponse for testing SSE connections. +/// Captures written data in a MemoryStream. +/// The WriteAsync extension on HttpResponse writes to Body directly, +/// so the MemoryStream captures all output. +/// +internal sealed class FakeHttpResponse : HttpResponse, IDisposable +{ + private readonly MemoryStream _body = new(); + private readonly HeaderDictionary _headers = new(); + + public override HttpContext HttpContext => null!; + public override int StatusCode { get; set; } + public override IHeaderDictionary Headers => _headers; + public override Stream Body + { + get => _body; + set { } + } + public override long? ContentLength { get; set; } + public override string? ContentType { get; set; } + public override IResponseCookies Cookies => null!; + public override bool HasStarted => true; + + /// + /// Get all data written to this response as a string. + /// + public string WrittenData => Encoding.UTF8.GetString(_body.ToArray()); + + public override void OnCompleted(Func callback, object state) { } + public override void OnStarting(Func callback, object state) { } + public override void Redirect(string location, bool permanent) { } + + public void Dispose() + { + _body.Dispose(); + } +} diff --git a/tests/Exceptionless.Tests/Hubs/SseIntegrationTests.cs b/tests/Exceptionless.Tests/Hubs/SseIntegrationTests.cs new file mode 100644 index 0000000000..8b772d5f49 --- /dev/null +++ b/tests/Exceptionless.Tests/Hubs/SseIntegrationTests.cs @@ -0,0 +1,196 @@ +using System.Net; +using System.Text; +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Utility; +using Exceptionless.Web.Hubs; +using Foundatio.Messaging; +using Foundatio.Repositories.Models; +using Xunit; + +namespace Exceptionless.Tests.Hubs; + +/// +/// Integration tests for the SSE endpoint (/api/v2/push). +/// These test the full HTTP pipeline including auth, middleware, and message delivery. +/// +public sealed class SseIntegrationTests : IntegrationTestsBase +{ + private readonly IMessagePublisher _messagePublisher; + + public SseIntegrationTests(ITestOutputHelper output, AppWebHostFactory factory) + : base(output, factory) + { + _messagePublisher = GetService(); + } + + [Fact] + public async Task ConnectWithValidToken_ReturnsEventStream() + { + var token = await CreateTokenAsync(); + + using var client = _server.CreateClient(); + using var request = new HttpRequestMessage(HttpMethod.Get, "/api/v2/push"); + request.Headers.Add("Accept", "text/event-stream"); + request.Headers.Add("Authorization", $"Bearer {token.Id}"); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(5)); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cts.Token); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("text/event-stream", response.Content.Headers.ContentType?.MediaType); + } + + [Fact] + public async Task ConnectWithoutAuth_Returns401() + { + using var client = _server.CreateClient(); + using var request = new HttpRequestMessage(HttpMethod.Get, "/api/v2/push"); + request.Headers.Add("Accept", "text/event-stream"); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(5)); + using var response = await client.SendAsync(request, cts.Token); + + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + } + + [Fact] + public async Task ConnectWithInvalidToken_Returns401() + { + using var client = _server.CreateClient(); + using var request = new HttpRequestMessage(HttpMethod.Get, "/api/v2/push"); + request.Headers.Add("Accept", "text/event-stream"); + request.Headers.Add("Authorization", "Bearer invalid-token-xyz"); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(5)); + using var response = await client.SendAsync(request, cts.Token); + + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + } + + [Fact] + public async Task ConnectWithAccessTokenQueryParam_Succeeds() + { + var token = await CreateTokenAsync(); + + using var client = _server.CreateClient(); + using var request = new HttpRequestMessage(HttpMethod.Get, $"/api/v2/push?access_token={token.Id}"); + request.Headers.Add("Accept", "text/event-stream"); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(5)); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cts.Token); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + + [Fact] + public async Task ConnectedClient_ReceivesEntityChangedMessage() + { + var token = await CreateTokenAsync(); + var orgId = token.OrganizationId; + + using var client = _server.CreateClient(); + using var request = new HttpRequestMessage(HttpMethod.Get, "/api/v2/push"); + request.Headers.Add("Accept", "text/event-stream"); + request.Headers.Add("Authorization", $"Bearer {token.Id}"); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cts.Token); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var stream = await response.Content.ReadAsStreamAsync(cts.Token); + using var reader = new StreamReader(stream, Encoding.UTF8); + + // Read the initial "Connected" event + string? connectedEvent = await ReadSseEventAsync(reader, cts.Token); + Assert.NotNull(connectedEvent); + Assert.Contains("Connected", connectedEvent); + + // Publish an EntityChanged message to the organization + var entityChanged = new EntityChanged + { + Id = "stack-123", + Type = "Stack", + ChangeType = ChangeType.Saved + }; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.OrganizationId] = orgId; +#pragma warning disable xUnit1051 + await _messagePublisher.PublishAsync(entityChanged); +#pragma warning restore xUnit1051 + + // Wait for and read the message + string? receivedEvent = await ReadSseEventAsync(reader, cts.Token); + Assert.NotNull(receivedEvent); + Assert.Contains("StackChanged", receivedEvent); + Assert.Contains("stack-123", receivedEvent); + } + + [Fact] + public async Task SseEndpoint_IsExemptFromThrottling() + { + var token = await CreateTokenAsync(); + + using var client = _server.CreateClient(); + + // Make multiple SSE connection attempts - should not be throttled + for (int i = 0; i < 5; i++) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/api/v2/push"); + request.Headers.Add("Accept", "text/event-stream"); + request.Headers.Add("Authorization", $"Bearer {token.Id}"); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(3)); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cts.Token); + + // Should never be 429 + Assert.NotEqual(HttpStatusCode.TooManyRequests, response.StatusCode); + } + } + + /// + /// Read a single SSE event (terminated by double newline) from the stream. + /// + private static async Task ReadSseEventAsync(StreamReader reader, CancellationToken ct) + { + var sb = new StringBuilder(); + int emptyLineCount = 0; + + while (!ct.IsCancellationRequested) + { + string? line = await reader.ReadLineAsync(ct); + if (line is null) + return sb.Length > 0 ? sb.ToString() : null; + + if (line.Length == 0) + { + emptyLineCount++; + if (emptyLineCount >= 1 && sb.Length > 0) + return sb.ToString(); + continue; + } + + // Skip comments (keep-alive) + if (line.StartsWith(':')) + continue; + + emptyLineCount = 0; + sb.AppendLine(line); + } + + return sb.Length > 0 ? sb.ToString() : null; + } + + private async Task CreateTokenAsync() + { + var tokenData = GetService(); + return tokenData.GenerateSampleUserToken(); + } +} diff --git a/tests/Exceptionless.Tests/Hubs/SseTests.cs b/tests/Exceptionless.Tests/Hubs/SseTests.cs new file mode 100644 index 0000000000..396fb172bd --- /dev/null +++ b/tests/Exceptionless.Tests/Hubs/SseTests.cs @@ -0,0 +1,459 @@ +using Exceptionless.Core; +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models; +using Exceptionless.Core.Utility; +using Exceptionless.Web.Hubs; +using Foundatio.Repositories.Models; +using Foundatio.Serializer; +using Xunit; + +namespace Exceptionless.Tests.Hubs; + +public sealed class SseConnectionManagerTests : TestWithServices +{ + public SseConnectionManagerTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void AddConnection_NewConnection_CanLookupAndEnumerate() + { + using var manager = CreateManager(); + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + + string connectionId = "test-conn-1"; + var connection = manager.AddConnection(connectionId, response, cts.Token); + + Assert.NotNull(connection); + Assert.Same(connection, manager.GetConnectionById(connectionId)); + Assert.Equal(1, manager.ConnectionCount); + Assert.Contains(connection, manager.GetAll()); + } + + [Fact] + public async Task RemoveConnectionAsync_ExistingConnection_RemovesAndAborts() + { + using var manager = CreateManager(); + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + + string connectionId = "test-conn-2"; + var connection = manager.AddConnection(connectionId, response, cts.Token); + + await manager.RemoveConnectionAsync(connectionId); + + Assert.Null(manager.GetConnectionById(connectionId)); + Assert.Equal(0, manager.ConnectionCount); + Assert.True(connection.ConnectionAborted.IsCancellationRequested); + } + + [Fact] + public async Task RemoveConnectionAsync_UnknownConnection_DoesNothing() + { + using var manager = CreateManager(); + + await manager.RemoveConnectionAsync("nonexistent"); + + Assert.Equal(0, manager.ConnectionCount); + } + + [Fact] + public void SendMessage_ValidConnection_EnqueuesMessage() + { + using var manager = CreateManager(); + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + + string connectionId = "test-conn-3"; + manager.AddConnection(connectionId, response, cts.Token); + + bool sent = manager.SendMessage(connectionId, new { type = "test", message = "hello" }); + + Assert.True(sent); + } + + [Fact] + public void SendMessage_UnknownConnection_ReturnsFalse() + { + using var manager = CreateManager(); + + bool sent = manager.SendMessage("missing", new { type = "test" }); + + Assert.False(sent); + } + + [Fact] + public async Task SendMessage_AbortedConnection_ReturnsFalseAndRemoves() + { + using var manager = CreateManager(); + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + + string connectionId = "test-conn-4"; + manager.AddConnection(connectionId, response, cts.Token); + + await cts.CancelAsync(); + + bool sent = manager.SendMessage(connectionId, new { type = "test" }); + + Assert.False(sent); + // Connection should be cleaned up + Assert.Null(manager.GetConnectionById(connectionId)); + } + + [Fact] + public void SendMessageToAll_MultipleConnections_SendsToAll() + { + using var manager = CreateManager(); + using var response1 = new FakeHttpResponse(); + using var response2 = new FakeHttpResponse(); + using var cts1 = new CancellationTokenSource(); + using var cts2 = new CancellationTokenSource(); + + manager.AddConnection("conn-1", response1, cts1.Token); + manager.AddConnection("conn-2", response2, cts2.Token); + + manager.SendMessageToAll(new { type = "broadcast" }); + + // Both connections should have received the message (enqueued) + Assert.Equal(2, manager.ConnectionCount); + } + + private SseConnectionManager CreateManager() + { + var options = new AppOptions { EnablePush = true }; + return new SseConnectionManager(options, GetService(), Log); + } +} + +/// +/// Tests for MessageBusBroker using SSE connections. +/// +public sealed class SseBrokerTests : TestWithServices +{ + private readonly MessageBusBroker _broker; + private readonly IConnectionMapping _connectionMapping; + private readonly SseConnectionManager _connectionManager; + + public SseBrokerTests(ITestOutputHelper output) : base(output) + { + _broker = GetService(); + _connectionMapping = GetService(); + _connectionManager = GetService(); + } + + [Fact] + public async Task OnEntityChangedAsync_AuthTokenRemoved_ClosesConnectionsAndClearsMapping() + { + const string userId = "test-user-id"; + const string organizationId = "test-org-id"; + using var response1 = new FakeHttpResponse(); + using var response2 = new FakeHttpResponse(); + using var unrelatedResponse = new FakeHttpResponse(); + using var cts1 = new CancellationTokenSource(); + using var cts2 = new CancellationTokenSource(); + using var ctsu = new CancellationTokenSource(); + + string connId1 = "conn-auth-1"; + string connId2 = "conn-auth-2"; + string unrelatedConnId = "conn-unrelated"; + + _connectionManager.AddConnection(connId1, response1, cts1.Token); + _connectionManager.AddConnection(connId2, response2, cts2.Token); + _connectionManager.AddConnection(unrelatedConnId, unrelatedResponse, ctsu.Token); + + try + { + await _connectionMapping.UserIdAddAsync(userId, connId1); + await _connectionMapping.UserIdAddAsync(userId, connId2); + await _connectionMapping.GroupAddAsync(organizationId, connId1); + await _connectionMapping.GroupAddAsync(organizationId, connId2); + await _connectionMapping.GroupAddAsync(organizationId, unrelatedConnId); + + var entityChanged = new EntityChanged + { + Id = "test-token-id", + Type = nameof(Token), + ChangeType = ChangeType.Removed + }; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.OrganizationId] = organizationId; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.UserId] = userId; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.IsAuthenticationToken] = true; + + await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); + + // Connections should be removed + Assert.Null(_connectionManager.GetConnectionById(connId1)); + Assert.Null(_connectionManager.GetConnectionById(connId2)); + Assert.NotNull(_connectionManager.GetConnectionById(unrelatedConnId)); + + // User mapping cleared + var remaining = await _connectionMapping.GetUserIdConnectionsAsync(userId); + Assert.Empty(remaining); + + // Org mapping only has unrelated connection + var orgConnections = await _connectionMapping.GetGroupConnectionsAsync(organizationId); + Assert.DoesNotContain(connId1, orgConnections); + Assert.DoesNotContain(connId2, orgConnections); + Assert.Contains(unrelatedConnId, orgConnections); + } + finally + { + await _connectionMapping.GroupRemoveAsync(organizationId, unrelatedConnId); + await _connectionManager.RemoveConnectionAsync(unrelatedConnId); + } + } + + [Fact] + public async Task OnEntityChangedAsync_NonAuthTokenRemoved_DoesNotCloseConnections() + { + const string userId = "test-user-id-2"; + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + + string connectionId = "conn-nonauth"; + _connectionManager.AddConnection(connectionId, response, cts.Token); + + try + { + await _connectionMapping.UserIdAddAsync(userId, connectionId); + + var entityChanged = new EntityChanged + { + Id = "test-api-token-id", + Type = nameof(Token), + ChangeType = ChangeType.Removed + }; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.UserId] = userId; + // IsAuthenticationToken intentionally omitted (defaults false) + + await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); + + // Connection should NOT be closed + Assert.NotNull(_connectionManager.GetConnectionById(connectionId)); + } + finally + { + await _connectionMapping.UserIdRemoveAsync(userId, connectionId); + await _connectionManager.RemoveConnectionAsync(connectionId); + } + } + + [Fact] + public async Task OnEntityChangedAsync_OrganizationMessage_SentToGroupOnly() + { + const string orgId = "org-1"; + const string otherOrgId = "org-2"; + using var responseInOrg = new FakeHttpResponse(); + using var responseOutOrg = new FakeHttpResponse(); + using var cts1 = new CancellationTokenSource(); + using var cts2 = new CancellationTokenSource(); + + string inOrgConn = "conn-in-org"; + string outOrgConn = "conn-out-org"; + + _connectionManager.AddConnection(inOrgConn, responseInOrg, cts1.Token); + _connectionManager.AddConnection(outOrgConn, responseOutOrg, cts2.Token); + + try + { + await _connectionMapping.GroupAddAsync(orgId, inOrgConn); + await _connectionMapping.GroupAddAsync(otherOrgId, outOrgConn); + + var entityChanged = new EntityChanged + { + Id = "stack-123", + Type = "Stack", + ChangeType = ChangeType.Saved + }; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.OrganizationId] = orgId; + + await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); + + // Give write loop a moment to process + await Task.Delay(200, TestContext.Current.CancellationToken); + + // In-org connection should receive message, out-org should not + Assert.True(responseInOrg.WrittenData.Length > 0, "In-org connection should receive message"); + Assert.Equal(0, responseOutOrg.WrittenData.Length); + } + finally + { + await _connectionMapping.GroupRemoveAsync(orgId, inOrgConn); + await _connectionMapping.GroupRemoveAsync(otherOrgId, outOrgConn); + await _connectionManager.RemoveConnectionAsync(inOrgConn); + await _connectionManager.RemoveConnectionAsync(outOrgConn); + } + } + + [Fact] + public async Task OnEntityChangedAsync_UserMessage_SentToUserOnly() + { + const string userId = "user-target"; + const string otherUserId = "user-other"; + using var responseTarget = new FakeHttpResponse(); + using var responseOther = new FakeHttpResponse(); + using var cts1 = new CancellationTokenSource(); + using var cts2 = new CancellationTokenSource(); + + string targetConn = "conn-target-user"; + string otherConn = "conn-other-user"; + + _connectionManager.AddConnection(targetConn, responseTarget, cts1.Token); + _connectionManager.AddConnection(otherConn, responseOther, cts2.Token); + + try + { + await _connectionMapping.UserIdAddAsync(userId, targetConn); + await _connectionMapping.UserIdAddAsync(otherUserId, otherConn); + + var entityChanged = new EntityChanged + { + Id = userId, + Type = nameof(User), + ChangeType = ChangeType.Saved + }; + + await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); + + await Task.Delay(200, TestContext.Current.CancellationToken); + + Assert.True(responseTarget.WrittenData.Length > 0, "Target user should receive message"); + Assert.Equal(0, responseOther.WrittenData.Length); + } + finally + { + await _connectionMapping.UserIdRemoveAsync(userId, targetConn); + await _connectionMapping.UserIdRemoveAsync(otherUserId, otherConn); + await _connectionManager.RemoveConnectionAsync(targetConn); + await _connectionManager.RemoveConnectionAsync(otherConn); + } + } +} + +/// +/// Tests for the deduplication behavior of SseConnection. +/// Validates that identical messages queued in quick succession are coalesced. +/// +public sealed class SseDeduplicationTests : TestWithServices +{ + public SseDeduplicationTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public async Task DuplicateMessages_AreDeduped_OnlyOneQueued() + { + var queue = new SseConnection.DedupQueue(8); + var evt = new SseConnection.SseEvent { Data = "{\"type\":\"StackChanged\",\"id\":\"stack-123\",\"change_type\":1}", DedupeKey = "stack-123" }; + int dedupedCount = 0; + + for (int i = 0; i < 5; i++) + { + if (queue.TryEnqueue(evt) == SseConnection.EnqueueResult.Deduped) + dedupedCount++; + } + + using var cts = new CancellationTokenSource(); + var queued = await queue.DequeueAsync(cts.Token); + + Assert.NotNull(queued); + Assert.Equal(evt.Data, queued!.Value.Data); + Assert.Equal(4, dedupedCount); + } + + [Fact] + public async Task DifferentMessages_AreNotDeduped() + { + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + var serializer = GetService(); + + await using var connection = new SseConnection("dedup-test-2", response, serializer, cts.Token, Log.CreateLogger()); + + // Send 3 different messages + connection.TryWrite(new { type = "StackChanged", id = "stack-1" }); + connection.TryWrite(new { type = "StackChanged", id = "stack-2" }); + connection.TryWrite(new { type = "ProjectChanged", id = "proj-1" }); + + await Task.Delay(200, TestContext.Current.CancellationToken); + connection.Abort(); + await Task.Delay(50, TestContext.Current.CancellationToken); + + string output = response.WrittenData; + int dataLineCount = output.Split("data: ").Length - 1; + Assert.Equal(3, dataLineCount); + Assert.Equal(0, connection.DedupedMessages); + } + + [Fact] + public async Task SameMessage_AfterFirstIsConsumed_IsNotDeduped() + { + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + var serializer = GetService(); + + await using var connection = new SseConnection("dedup-test-3", response, serializer, cts.Token, Log.CreateLogger()); + + var message = new { type = "StackChanged", id = "stack-repeat" }; + + // Send first message and wait for it to be consumed + connection.TryWrite(message); + await Task.Delay(200, TestContext.Current.CancellationToken); + + // Send same message again — should NOT be deduped because first was already consumed + connection.TryWrite(message); + await Task.Delay(200, TestContext.Current.CancellationToken); + + connection.Abort(); + await Task.Delay(50, TestContext.Current.CancellationToken); + + string output = response.WrittenData; + int dataLineCount = output.Split("data: ").Length - 1; + Assert.Equal(2, dataLineCount); + Assert.Equal(0, connection.DedupedMessages); + } + + [Fact] + public async Task KeepAlive_IsNeverDeduped() + { + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + var serializer = GetService(); + + await using var connection = new SseConnection("dedup-test-4", response, serializer, cts.Token, Log.CreateLogger()); + + // Send multiple keep-alives — none should be deduped + connection.TryWriteKeepAlive(); + connection.TryWriteKeepAlive(); + connection.TryWriteKeepAlive(); + + await Task.Delay(200, TestContext.Current.CancellationToken); + connection.Abort(); + await Task.Delay(50, TestContext.Current.CancellationToken); + + string output = response.WrittenData; + int keepAliveCount = output.Split(": keepalive").Length - 1; + Assert.Equal(3, keepAliveCount); + } + + [Fact] + public async Task Capacity_Exceeded_DropsOldest() + { + // Test the DedupQueue directly to avoid racing with the write loop + var queue = new SseConnection.DedupQueue(3); + + // Enqueue 5 items with unique keys — first 2 should be dropped + for (int i = 0; i < 5; i++) + { + queue.TryEnqueue(new SseConnection.SseEvent { Data = $"msg-{i}", DedupeKey = $"key-{i}" }); + } + + // Dequeue and verify we get the last 3 items (oldest 2 were dropped) + using var cts = new CancellationTokenSource(); + var item1 = await queue.DequeueAsync(cts.Token); + var item2 = await queue.DequeueAsync(cts.Token); + var item3 = await queue.DequeueAsync(cts.Token); + + Assert.Equal("msg-2", item1!.Value.Data); + Assert.Equal("msg-3", item2!.Value.Data); + Assert.Equal("msg-4", item3!.Value.Data); + } +} diff --git a/tests/Exceptionless.Tests/Hubs/TestWebSocket.cs b/tests/Exceptionless.Tests/Hubs/TestWebSocket.cs deleted file mode 100644 index c8343c7b3a..0000000000 --- a/tests/Exceptionless.Tests/Hubs/TestWebSocket.cs +++ /dev/null @@ -1,53 +0,0 @@ -using System.Net.WebSockets; -using System.Text; - -namespace Exceptionless.Tests.Hubs; - -internal sealed class TestWebSocket : WebSocket -{ - private WebSocketState _state; - - public TestWebSocket(WebSocketState state = WebSocketState.Open) - { - _state = state; - } - - public int CloseCount => _closeCount; - private int _closeCount; - public List SentMessages { get; } = []; - public override WebSocketCloseStatus? CloseStatus { get; } = WebSocketCloseStatus.NormalClosure; - public override string? CloseStatusDescription { get; } = "Closed"; - public override string? SubProtocol { get; } = null; - public override WebSocketState State => _state; - - public override void Abort() - { - _state = WebSocketState.Aborted; - } - - public override Task CloseAsync(WebSocketCloseStatus closeStatus, string? statusDescription, CancellationToken cancellationToken) - { - Interlocked.Increment(ref _closeCount); - _state = WebSocketState.Closed; - return Task.CompletedTask; - } - - public override Task CloseOutputAsync(WebSocketCloseStatus closeStatus, string? statusDescription, CancellationToken cancellationToken) - { - _state = WebSocketState.CloseSent; - return Task.CompletedTask; - } - - public override void Dispose() { } - - public override Task ReceiveAsync(ArraySegment buffer, CancellationToken cancellationToken) - { - return Task.FromResult(new WebSocketReceiveResult(0, WebSocketMessageType.Text, true)); - } - - public override Task SendAsync(ArraySegment buffer, WebSocketMessageType messageType, bool endOfMessage, CancellationToken cancellationToken) - { - SentMessages.Add(Encoding.ASCII.GetString(buffer.Array!, buffer.Offset, buffer.Count)); - return Task.CompletedTask; - } -} diff --git a/tests/Exceptionless.Tests/Hubs/WebSocketConnectionManagerTests.cs b/tests/Exceptionless.Tests/Hubs/WebSocketConnectionManagerTests.cs deleted file mode 100644 index 9018916fdc..0000000000 --- a/tests/Exceptionless.Tests/Hubs/WebSocketConnectionManagerTests.cs +++ /dev/null @@ -1,99 +0,0 @@ -using System.Net.WebSockets; -using System.Text; -using Exceptionless.Core; -using Exceptionless.Web.Hubs; -using Foundatio.Serializer; -using Xunit; - -namespace Exceptionless.Tests.Hubs; - -public sealed class WebSocketConnectionManagerTests : TestWithServices -{ - public WebSocketConnectionManagerTests(ITestOutputHelper output) : base(output) { } - - [Fact] - public void AddWebSocket_NewSocket_CanLookupAndEnumerateConnection() - { - // Arrange - using var manager = CreateManager(); - var socket = new TestWebSocket(); - - // Act - string connectionId = manager.AddWebSocket(socket); - - // Assert - Assert.False(String.IsNullOrEmpty(connectionId)); - Assert.Same(socket, manager.GetWebSocketById(connectionId)); - Assert.Equal(connectionId, manager.GetConnectionId(socket)); - Assert.Same(socket, Assert.Single(manager.GetAll())); - } - - [Fact] - public async Task RemoveWebSocketAsync_ExistingConnection_RemovesAndClosesSocket() - { - // Arrange - using var manager = CreateManager(); - var socket = new TestWebSocket(); - string connectionId = manager.AddWebSocket(socket); - - // Act - await manager.RemoveWebSocketAsync(connectionId); - - // Assert - Assert.Null(manager.GetWebSocketById(connectionId)); - Assert.Empty(manager.GetAll()); - Assert.Equal(1, socket.CloseCount); - Assert.Equal(WebSocketState.Closed, socket.State); - } - - [Fact] - public async Task RemoveWebSocketAsync_ClosedSocket_RemovesWithoutClosingAgain() - { - // Arrange - using var manager = CreateManager(); - var socket = new TestWebSocket(WebSocketState.Closed); - string connectionId = manager.AddWebSocket(socket); - - // Act - await manager.RemoveWebSocketAsync(connectionId); - - // Assert - Assert.Null(manager.GetWebSocketById(connectionId)); - Assert.Empty(manager.GetAll()); - Assert.Equal(0, socket.CloseCount); - } - - [Fact] - public async Task RemoveWebSocketAsync_UnknownConnection_DoesNothing() - { - // Arrange - using var manager = CreateManager(); - - // Act - await manager.RemoveWebSocketAsync("missing"); - - // Assert - Assert.Empty(manager.GetAll()); - } - - [Fact] - public async Task SendMessageToAllAsync_ClosedSockets_DoesNotSend() - { - // Arrange - using var manager = CreateManager(); - var socket = new TestWebSocket(WebSocketState.Closed); - manager.AddWebSocket(socket); - - // Act - await manager.SendMessageToAllAsync(new { type = "test" }); - - // Assert - Assert.Empty(socket.SentMessages); - } - - private WebSocketConnectionManager CreateManager() - { - var options = new AppOptions { EnableWebSockets = false }; - return new WebSocketConnectionManager(options, GetService(), Log); - } -} diff --git a/tests/Exceptionless.Tests/Hubs/WebSocketTests.cs b/tests/Exceptionless.Tests/Hubs/WebSocketTests.cs deleted file mode 100644 index a34a33ddc7..0000000000 --- a/tests/Exceptionless.Tests/Hubs/WebSocketTests.cs +++ /dev/null @@ -1,121 +0,0 @@ -using Exceptionless.Core.Messaging.Models; -using Exceptionless.Core.Models; -using Exceptionless.Core.Utility; -using Exceptionless.Web.Hubs; -using Foundatio.Repositories.Models; -using Xunit; - -namespace Exceptionless.Tests.Hubs; - -/// -/// Tests for WebSocket behavior. Calls -/// directly so they do not depend on -/// message bus wiring or EnableWebSockets in test host configuration. -/// -public sealed class WebSocketTests : TestWithServices -{ - private readonly MessageBusBroker _broker; - private readonly IConnectionMapping _connectionMapping; - private readonly WebSocketConnectionManager _connectionManager; - - public WebSocketTests(ITestOutputHelper output) : base(output) - { - _broker = GetService(); - _connectionMapping = GetService(); - _connectionManager = GetService(); - } - - [Fact] - public async Task OnEntityChangedAsync_AuthTokenRemoved_ClosesWebSocketsAndClearsUserMapping() - { - // Arrange - const string userId = "test-user-id"; - const string organizationId = "test-organization-id"; - var socket1 = new TestWebSocket(); - var socket2 = new TestWebSocket(); - var unrelatedSocket = new TestWebSocket(); - - string connectionId1 = _connectionManager.AddWebSocket(socket1); - string connectionId2 = _connectionManager.AddWebSocket(socket2); - string unrelatedConnectionId = _connectionManager.AddWebSocket(unrelatedSocket); - - try - { - await _connectionMapping.UserIdAddAsync(userId, connectionId1); - await _connectionMapping.UserIdAddAsync(userId, connectionId2); - await _connectionMapping.GroupAddAsync(organizationId, connectionId1); - await _connectionMapping.GroupAddAsync(organizationId, connectionId2); - await _connectionMapping.GroupAddAsync(organizationId, unrelatedConnectionId); - - var entityChanged = new EntityChanged - { - Id = "test-token-id", - Type = nameof(Token), - ChangeType = ChangeType.Removed - }; - entityChanged.Data[ExtendedEntityChanged.KnownKeys.OrganizationId] = organizationId; - entityChanged.Data[ExtendedEntityChanged.KnownKeys.UserId] = userId; - entityChanged.Data[ExtendedEntityChanged.KnownKeys.IsAuthenticationToken] = true; - - // Act — call the broker directly; no message bus or EnableWebSockets dependency - await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); - - // Assert – sockets closed and removed from manager - Assert.Null(_connectionManager.GetWebSocketById(connectionId1)); - Assert.Null(_connectionManager.GetWebSocketById(connectionId2)); - Assert.Same(unrelatedSocket, _connectionManager.GetWebSocketById(unrelatedConnectionId)); - - Assert.Equal(1, socket1.CloseCount); - Assert.Equal(1, socket2.CloseCount); - Assert.Equal(0, unrelatedSocket.CloseCount); - - // Assert – user-id mapping removed by broker - var remaining = await _connectionMapping.GetUserIdConnectionsAsync(userId); - Assert.Empty(remaining); - var organizationConnections = await _connectionMapping.GetGroupConnectionsAsync(organizationId); - Assert.DoesNotContain(connectionId1, organizationConnections); - Assert.DoesNotContain(connectionId2, organizationConnections); - Assert.Contains(unrelatedConnectionId, organizationConnections); - } - finally - { - await _connectionMapping.GroupRemoveAsync(organizationId, unrelatedConnectionId); - await _connectionManager.RemoveWebSocketAsync(unrelatedConnectionId); - } - } - - [Fact] - public async Task OnEntityChangedAsync_NonAuthTokenRemoved_DoesNotCloseWebSockets() - { - // Arrange - const string userId = "test-user-id-2"; - var socket = new TestWebSocket(); - string connectionId = _connectionManager.AddWebSocket(socket); - - try - { - await _connectionMapping.UserIdAddAsync(userId, connectionId); - - var entityChanged = new EntityChanged - { - Id = "test-api-token-id", - Type = nameof(Token), - ChangeType = ChangeType.Removed - }; - entityChanged.Data[ExtendedEntityChanged.KnownKeys.UserId] = userId; - // IsAuthenticationToken intentionally omitted (defaults false) - - // Act - await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); - - // Assert – socket should NOT be closed for a non-auth token removal - Assert.Equal(0, socket.CloseCount); - Assert.Same(socket, _connectionManager.GetWebSocketById(connectionId)); - } - finally - { - await _connectionMapping.UserIdRemoveAsync(userId, connectionId); - await _connectionManager.RemoveWebSocketAsync(connectionId); - } - } -} diff --git a/tests/Exceptionless.Tests/appsettings.yml b/tests/Exceptionless.Tests/appsettings.yml index 5bdb00cb6a..0daab2e20d 100644 --- a/tests/Exceptionless.Tests/appsettings.yml +++ b/tests/Exceptionless.Tests/appsettings.yml @@ -20,7 +20,7 @@ EnableDailySummary: false # Runs the jobs in the current website process RunJobsInProcess: false -EnableWebSockets: false +EnableWebSockets: true Serilog: MinimumLevel: Warning From da83047c6972b1a8a7a1ff1445cb1574549d669e Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 27 May 2026 21:26:22 -0500 Subject: [PATCH 2/7] fix: harden SSE rollout compatibility Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- k8s/exceptionless/templates/api.yaml | 7 +- .../Utility/AppDiagnostics.cs | 5 + src/Exceptionless.Web/Bootstrapper.cs | 1 + .../features/websockets/sse-client.svelte.ts | 39 +++- .../features/websockets/sse-client.test.ts | 16 ++ .../Hubs/MessageBusBroker.cs | 33 +++- src/Exceptionless.Web/Hubs/SseConnection.cs | 47 +++-- .../Hubs/SseConnectionManager.cs | 16 +- src/Exceptionless.Web/Hubs/SseMiddleware.cs | 3 +- .../Hubs/WebSocketConnectionManager.cs | 182 ++++++++++++++++++ .../Hubs/WebSocketPushMiddleware.cs | 109 +++++++++++ src/Exceptionless.Web/Startup.cs | 2 + .../Hubs/SseIntegrationTests.cs | 34 +++- tests/Exceptionless.Tests/Hubs/SseTests.cs | 18 ++ .../Exceptionless.Tests/Hubs/TestWebSocket.cs | 53 +++++ .../Hubs/WebSocketCompatibilityTests.cs | 156 +++++++++++++++ tests/http/push.http | 29 +++ 17 files changed, 703 insertions(+), 47 deletions(-) create mode 100644 src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs create mode 100644 src/Exceptionless.Web/Hubs/WebSocketPushMiddleware.cs create mode 100644 tests/Exceptionless.Tests/Hubs/TestWebSocket.cs create mode 100644 tests/Exceptionless.Tests/Hubs/WebSocketCompatibilityTests.cs create mode 100644 tests/http/push.http diff --git a/k8s/exceptionless/templates/api.yaml b/k8s/exceptionless/templates/api.yaml index f131ce253b..e0ba872569 100644 --- a/k8s/exceptionless/templates/api.yaml +++ b/k8s/exceptionless/templates/api.yaml @@ -82,8 +82,11 @@ spec: {{- include "exceptionless.otel-env" . | indent 12 }} - name: RunJobsInProcess value: 'false' + # SSE rollout prerequisite: Azure Application Gateway for Containers Ingress API + # does not support the routeTimeout=0s override required for long-lived SSE streams. + # Keep push disabled here until this route moves to Gateway API + RoutePolicy. - name: EnablePush - value: 'true' + value: 'false' {{- if (empty .Values.storage.connectionString) }} volumeMounts: - mountPath: "/app/storage" @@ -163,6 +166,8 @@ metadata: alb.networking.azure.io/alb-namespace: {{ .Values.ingress.albNamespace }} alb.networking.azure.io/alb-frontend: {{ template "exceptionless.fullname" . }}-fe cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }} + # SSE is not safe to enable behind the current AGC Ingress API path. + # Migrate to Gateway API and attach a RoutePolicy with routeTimeout: 0s before enabling push. spec: ingressClassName: azure-alb-external tls: diff --git a/src/Exceptionless.Core/Utility/AppDiagnostics.cs b/src/Exceptionless.Core/Utility/AppDiagnostics.cs index 6ef85bc829..145ec6bec1 100644 --- a/src/Exceptionless.Core/Utility/AppDiagnostics.cs +++ b/src/Exceptionless.Core/Utility/AppDiagnostics.cs @@ -124,6 +124,11 @@ public GaugeInfo(Meter meter, string name) internal static readonly Counter SavedViewsSize = Meter.CreateCounter("ex.savedviews.size", description: "Size of user saved views"); internal static readonly Counter SavedViewsViewTypeSize = Meter.CreateCounter("ex.savedviews.viewtype.size", description: "Size of user saved views by view type"); + + internal static readonly Counter PushSseConnectionsOpened = Meter.CreateCounter("ex.push.connections.sse.opened", description: "SSE push connections opened"); + internal static readonly Counter PushSseConnectionsClosed = Meter.CreateCounter("ex.push.connections.sse.closed", description: "SSE push connections closed"); + internal static readonly Counter PushWebSocketConnectionsOpened = Meter.CreateCounter("ex.push.connections.websocket.opened", description: "WebSocket push connections opened"); + internal static readonly Counter PushWebSocketConnectionsClosed = Meter.CreateCounter("ex.push.connections.websocket.closed", description: "WebSocket push connections closed"); } public static class MetricsClientExtensions diff --git a/src/Exceptionless.Web/Bootstrapper.cs b/src/Exceptionless.Web/Bootstrapper.cs index 8004ff3d2e..07d368d522 100644 --- a/src/Exceptionless.Web/Bootstrapper.cs +++ b/src/Exceptionless.Web/Bootstrapper.cs @@ -15,6 +15,7 @@ public class Bootstrapper public static void RegisterServices(IServiceCollection services, AppOptions appOptions, ILoggerFactory loggerFactory) { services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts index b94a609a06..8bd3a014c5 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts @@ -55,6 +55,7 @@ export class SseClient { private hasConnectedBefore: boolean = false; private reconnectAttempts: number = 0; private reconnectTimeoutId: null | ReturnType = null; + private streamGeneration: number = 0; private abortController: AbortController | null = null; @@ -73,11 +74,9 @@ export class SseClient { this.accessToken = accessToken.current; this.reconnectAttempts = 0; this.authFailed = false; - this.close(); - this.forcedClose = false; // Allow reconnect with new token + this.close(false); } else if (!visibility.visible) { - this.close(); - this.forcedClose = false; // Allow reconnect when visible again + this.close(false); } if (this.accessToken && visibility.visible && this.readyState === SSE_CLOSED && this.reconnectTimeoutId === null && !this.authFailed && !this.forcedClose) { @@ -86,14 +85,15 @@ export class SseClient { }); } - public close(): boolean { + public close(isManual: boolean = true): boolean { clearTimeout(this.reconnectTimeoutId!); this.reconnectTimeoutId = null; clearTimeout(this.connectionTimeoutId!); this.connectionTimeoutId = null; + this.forcedClose = isManual; if (this.abortController) { - this.forcedClose = true; + this.streamGeneration++; this.abortController.abort(); this.abortController = null; this.readyState = SSE_CLOSED; @@ -105,6 +105,7 @@ export class SseClient { public connect() { const isReconnect: boolean = this.hasConnectedBefore; + const generation = ++this.streamGeneration; this.readyState = SSE_CONNECTING; this.forcedClose = false; @@ -125,7 +126,7 @@ export class SseClient { } }, timeout); - this.startStream(signal, isReconnect); + this.startStream(signal, isReconnect, generation); } public onClose: () => void = () => {}; @@ -146,7 +147,7 @@ export class SseClient { return Math.min(1000 * Math.pow(2, attempt - 1), 30000); } - private async startStream(signal: AbortSignal, isReconnect: boolean) { + private async startStream(signal: AbortSignal, isReconnect: boolean, generation: number) { try { const token = this.accessToken ?? accessToken.current; const response = await fetch(this.url, { @@ -184,6 +185,11 @@ export class SseClient { throw new Error('SSE response has no body'); } + if (generation !== this.streamGeneration) { + this.readyState = SSE_CLOSED; + return; + } + this.readyState = SSE_OPEN; this.reconnectAttempts = 0; this.hasConnectedBefore = true; @@ -201,6 +207,11 @@ export class SseClient { break; } + if (generation !== this.streamGeneration) { + this.readyState = SSE_CLOSED; + return; + } + buffer += decoder.decode(value, { stream: true }); // Process complete SSE messages (separated by double newline) @@ -236,6 +247,11 @@ export class SseClient { clearTimeout(this.connectionTimeoutId!); this.connectionTimeoutId = null; + if (generation !== this.streamGeneration) { + this.readyState = SSE_CLOSED; + return; + } + if (signal.aborted && this.forcedClose) { // Intentional close - don't reconnect this.readyState = SSE_CLOSED; @@ -254,12 +270,17 @@ export class SseClient { } // Stream ended (server closed connection) - reconnect - if (!this.forcedClose) { + if (generation === this.streamGeneration && !this.forcedClose) { this.scheduleReconnect(); } } private scheduleReconnect() { + if (this.reconnectTimeoutId !== null || this.authFailed || this.forcedClose || !(this.accessToken ?? accessToken.current)) { + this.readyState = SSE_CLOSED; + return; + } + this.reconnectAttempts++; const delay = this.getReconnectDelay(this.reconnectAttempts); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts index 3b8a8907d0..3da2899d70 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts @@ -197,6 +197,22 @@ describe('SseClient', () => { // fetch should only be called once (no reconnect) expect(fetchMock).toHaveBeenCalledTimes(1); }); + + it('should allow reconnect after internal close', async () => { + fetchMock.mockImplementation(() => Promise.resolve(createOpenSseResponse([': keepalive\n\n']))); + + const client = trackedClient(); + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(client.close(false)).toBe(true); + + client.connect(); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(client.readyState).toBe(SSE_OPEN); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); }); describe('Auth Failure Handling', () => { diff --git a/src/Exceptionless.Web/Hubs/MessageBusBroker.cs b/src/Exceptionless.Web/Hubs/MessageBusBroker.cs index 2c0010cdf1..89f21ce680 100644 --- a/src/Exceptionless.Web/Hubs/MessageBusBroker.cs +++ b/src/Exceptionless.Web/Hubs/MessageBusBroker.cs @@ -13,15 +13,17 @@ public sealed class MessageBusBroker : IStartupAction { private static readonly string TokenTypeName = nameof(Token); private static readonly string UserTypeName = nameof(User); - private readonly SseConnectionManager _connectionManager; + private readonly SseConnectionManager _sseConnectionManager; + private readonly WebSocketConnectionManager _webSocketConnectionManager; private readonly IConnectionMapping _connectionMapping; private readonly IMessageSubscriber _subscriber; private readonly AppOptions _options; private readonly ILogger _logger; - public MessageBusBroker(SseConnectionManager connectionManager, IConnectionMapping connectionMapping, IMessageSubscriber subscriber, AppOptions options, ILogger logger) + public MessageBusBroker(SseConnectionManager sseConnectionManager, WebSocketConnectionManager webSocketConnectionManager, IConnectionMapping connectionMapping, IMessageSubscriber subscriber, AppOptions options, ILogger logger) { - _connectionManager = connectionManager; + _sseConnectionManager = sseConnectionManager; + _webSocketConnectionManager = webSocketConnectionManager; _connectionMapping = connectionMapping; _subscriber = subscriber; _options = options; @@ -110,7 +112,7 @@ internal async Task OnEntityChangedAsync(EntityChanged ec, CancellationToken can // there is no point delivering a message to a connection we are about to tear down. if (isAuthToken && entityChanged.ChangeType is ChangeType.Removed) { - _logger.LogTrace("Auth token removed for user {UserId}; closing {ConnectionCount} SSE connection(s)", userId, userConnectionIds.Count); + _logger.LogTrace("Auth token removed for user {UserId}; closing {ConnectionCount} push connection(s)", userId, userConnectionIds.Count); string? organizationId = entityChanged.OrganizationId; foreach (string connectionId in userConnectionIds) { @@ -118,7 +120,8 @@ internal async Task OnEntityChangedAsync(EntityChanged ec, CancellationToken can await _connectionMapping.GroupRemoveAsync(organizationId, connectionId); await _connectionMapping.UserIdRemoveAsync(userId, connectionId); - await _connectionManager.RemoveConnectionAsync(connectionId); + await _sseConnectionManager.RemoveConnectionAsync(connectionId); + await _webSocketConnectionManager.RemoveConnectionAsync(connectionId); } return; @@ -197,17 +200,26 @@ private async Task GroupSendAsync(string group, object value) public void TypedSend(string connectionId, object value) { - _connectionManager.SendMessage(connectionId, new TypedMessage { Type = GetMessageType(value), Message = value }); + var message = new TypedMessage { Type = GetMessageType(value), Message = value }; + bool canDrop = CanDrop(value); + _sseConnectionManager.SendMessage(connectionId, message, canDrop); + _webSocketConnectionManager.SendMessage(connectionId, message); } public void TypedSend(IEnumerable connectionIds, object value) { - _connectionManager.SendMessage(connectionIds, new TypedMessage { Type = GetMessageType(value), Message = value }); + var message = new TypedMessage { Type = GetMessageType(value), Message = value }; + bool canDrop = CanDrop(value); + _sseConnectionManager.SendMessage(connectionIds, message, canDrop); + _webSocketConnectionManager.SendMessage(connectionIds, message); } public void TypedBroadcast(object value) { - _connectionManager.SendMessageToAll(new TypedMessage { Type = GetMessageType(value), Message = value }); + var message = new TypedMessage { Type = GetMessageType(value), Message = value }; + bool canDrop = CanDrop(value); + _sseConnectionManager.SendMessageToAll(message, canDrop); + _webSocketConnectionManager.SendMessageToAll(message); } private static string GetMessageType(object value) @@ -217,6 +229,11 @@ private static string GetMessageType(object value) return value.GetType().Name; } + + private static bool CanDrop(object value) + { + return value is not (PlanOverage or ReleaseNotification or SystemNotification); + } } public record TypedMessage diff --git a/src/Exceptionless.Web/Hubs/SseConnection.cs b/src/Exceptionless.Web/Hubs/SseConnection.cs index 075b4ec2cb..84bc925df6 100644 --- a/src/Exceptionless.Web/Hubs/SseConnection.cs +++ b/src/Exceptionless.Web/Hubs/SseConnection.cs @@ -57,13 +57,13 @@ public SseConnection(string connectionId, HttpResponse response, ITextSerializer /// If an identical message (same serialized payload) is already queued, the new /// one is skipped (deduped) and this returns true. /// - public bool TryWrite(object message) + public bool TryWrite(object message, bool canDrop = true) { if (_cts.IsCancellationRequested) return false; string data = _serializer.SerializeToString(message); - var result = _queue.TryEnqueue(new SseEvent { Data = data, DedupeKey = data }); + var result = _queue.TryEnqueue(new SseEvent { Data = data, DedupeKey = canDrop ? data : null, CanDrop = canDrop }); if (result == EnqueueResult.Deduped) { @@ -71,7 +71,7 @@ public bool TryWrite(object message) return true; } - if (result == EnqueueResult.DroppedOldest) + if (result == EnqueueResult.DroppedQueuedMessage) Interlocked.Increment(ref _droppedMessages); return true; @@ -172,16 +172,17 @@ internal readonly record struct SseEvent /// the same client-side cache invalidation, so coalescing is safe. /// public string? DedupeKey { get; init; } + public bool CanDrop { get; init; } public bool IsKeepAlive { get; init; } - public static SseEvent KeepAlive => new() { IsKeepAlive = true }; + public static SseEvent KeepAlive => new() { IsKeepAlive = true, CanDrop = true }; } internal enum EnqueueResult { Enqueued, Deduped, - DroppedOldest + DroppedQueuedMessage } /// @@ -216,14 +217,13 @@ public EnqueueResult TryEnqueue(SseEvent evt) var result = EnqueueResult.Enqueued; - // Enforce capacity: drop oldest if full + // Enforce capacity: drop the oldest droppable message first so direct user + // notifications do not get crowded out by stale cache invalidations. if (_list.Count >= _capacity) { - var oldest = _list.First!; - _list.RemoveFirst(); - if (oldest.Value.DedupeKey is not null) - _index.Remove(oldest.Value.DedupeKey); - result = EnqueueResult.DroppedOldest; + var queuedToDrop = !evt.CanDrop ? FindFirstDroppableNode() : null; + RemoveNode(queuedToDrop ?? _list.First!); + result = EnqueueResult.DroppedQueuedMessage; } var node = _list.AddLast(evt); @@ -245,9 +245,7 @@ public EnqueueResult TryEnqueue(SseEvent evt) return null; // Completed var node = _list.First!; - _list.RemoveFirst(); - if (node.Value.DedupeKey is not null) - _index.Remove(node.Value.DedupeKey); + RemoveNode(node); return node.Value; } } @@ -267,5 +265,26 @@ public void Dispose() { _signal.Dispose(); } + + private LinkedListNode? FindFirstDroppableNode() + { + var current = _list.First; + while (current is not null) + { + if (current.Value.CanDrop) + return current; + + current = current.Next; + } + + return null; + } + + private void RemoveNode(LinkedListNode node) + { + _list.Remove(node); + if (node.Value.DedupeKey is not null) + _index.Remove(node.Value.DedupeKey); + } } } diff --git a/src/Exceptionless.Web/Hubs/SseConnectionManager.cs b/src/Exceptionless.Web/Hubs/SseConnectionManager.cs index ba0a7f4297..655b5e316f 100644 --- a/src/Exceptionless.Web/Hubs/SseConnectionManager.cs +++ b/src/Exceptionless.Web/Hubs/SseConnectionManager.cs @@ -85,6 +85,8 @@ public SseConnection AddConnection(string connectionId, HttpResponse response, C { var connection = new SseConnection(connectionId, response, _serializer, requestAborted, _logger); _connections.TryAdd(connectionId, connection); + AppDiagnostics.PushSseConnectionsOpened.Add(1); + AppDiagnostics.Gauge("push.connections.sse.active", _connections.Count); return connection; } @@ -106,7 +108,7 @@ private void TryRemove(string connectionId) _ = ObserveDisposeAsync(connectionId, DisposeConnectionAsync(connectionId, connection)); } - public bool SendMessage(string connectionId, object message) + public bool SendMessage(string connectionId, object message, bool canDrop = true) { if (!_connections.TryGetValue(connectionId, out var connection)) return false; @@ -117,16 +119,16 @@ public bool SendMessage(string connectionId, object message) return false; } - return connection.TryWrite(message); + return connection.TryWrite(message, canDrop); } - public void SendMessage(IEnumerable connectionIds, object message) + public void SendMessage(IEnumerable connectionIds, object message, bool canDrop = true) { foreach (string connectionId in connectionIds) - SendMessage(connectionId, message); + SendMessage(connectionId, message, canDrop); } - public void SendMessageToAll(object message) + public void SendMessageToAll(object message, bool canDrop = true) { foreach (var (connectionId, connection) in _connections) { @@ -136,7 +138,7 @@ public void SendMessageToAll(object message) continue; } - connection.TryWrite(message); + connection.TryWrite(message, canDrop); } } @@ -174,6 +176,8 @@ private async Task DisposeConnectionCoreAsync(string connectionId, SseConnection finally { _pendingDisposals.TryRemove(connectionId, out _); + AppDiagnostics.PushSseConnectionsClosed.Add(1); + AppDiagnostics.Gauge("push.connections.sse.active", _connections.Count); } } diff --git a/src/Exceptionless.Web/Hubs/SseMiddleware.cs b/src/Exceptionless.Web/Hubs/SseMiddleware.cs index d3a09b29d4..d50e679119 100644 --- a/src/Exceptionless.Web/Hubs/SseMiddleware.cs +++ b/src/Exceptionless.Web/Hubs/SseMiddleware.cs @@ -27,7 +27,8 @@ public SseMiddleware(RequestDelegate next, SseConnectionManager connectionManage public async Task Invoke(HttpContext context) { if (!context.Request.Path.StartsWithSegments(_sseEndpoint, StringComparison.Ordinal) - || !HttpMethods.IsGet(context.Request.Method)) + || !HttpMethods.IsGet(context.Request.Method) + || context.WebSockets.IsWebSocketRequest) { await _next(context); return; diff --git a/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs b/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs new file mode 100644 index 0000000000..c3b46571da --- /dev/null +++ b/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs @@ -0,0 +1,182 @@ +using System.Collections.Concurrent; +using System.Net.WebSockets; +using System.Text; +using Exceptionless.Core; +using Foundatio.Serializer; + +namespace Exceptionless.Web.Hubs; + +public sealed class WebSocketConnectionManager : IDisposable +{ + private static readonly ArraySegment KeepAliveMessage = new(Encoding.ASCII.GetBytes("{}"), 0, 2); + private readonly ConcurrentDictionary _connections = new(); + private readonly Timer? _timer; + private readonly ITextSerializer _serializer; + private readonly ILogger _logger; + + public int MaxConnectionsPerUser { get; init; } = 10; + public int ConnectionCount => _connections.Count; + + public WebSocketConnectionManager(AppOptions options, ITextSerializer serializer, ILoggerFactory loggerFactory) + { + _serializer = serializer; + _logger = loggerFactory.CreateLogger(); + + if (!options.EnablePush) + return; + + _timer = new Timer(SendKeepAlive, null, TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(15)); + } + + private void SendKeepAlive(object? state) + { + if (_connections.IsEmpty) + return; + + foreach (var (connectionId, socket) in _connections) + { + if (!CanSend(socket)) + { + _ = RemoveConnectionAsync(connectionId); + continue; + } + + _ = SendKeepAliveAsync(connectionId, socket); + } + } + + public WebSocket? GetConnectionById(string connectionId) + { + return _connections.TryGetValue(connectionId, out var socket) ? socket : null; + } + + public ICollection GetAll() + { + return _connections.Values; + } + + public string AddConnection(WebSocket socket) + { + string connectionId = Guid.NewGuid().ToString("N"); + _connections.TryAdd(connectionId, socket); + AppDiagnostics.PushWebSocketConnectionsOpened.Add(1); + AppDiagnostics.Gauge("push.connections.websocket.active", _connections.Count); + return connectionId; + } + + public async Task RemoveConnectionAsync(string connectionId) + { + if (!_connections.TryRemove(connectionId, out var socket)) + return; + + if (!CanSend(socket)) + { + AppDiagnostics.PushWebSocketConnectionsClosed.Add(1); + AppDiagnostics.Gauge("push.connections.websocket.active", _connections.Count); + return; + } + + try + { + await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Closed by manager", CancellationToken.None).ConfigureAwait(false); + } + catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) { } + catch (ObjectDisposedException) { } + catch (Exception ex) + { + _logger.LogDebug(ex, "Error closing websocket {ConnectionId}", connectionId); + } + finally + { + AppDiagnostics.PushWebSocketConnectionsClosed.Add(1); + AppDiagnostics.Gauge("push.connections.websocket.active", _connections.Count); + } + } + + public bool SendMessage(string connectionId, object message) + { + if (!_connections.TryGetValue(connectionId, out var socket)) + return false; + + if (!CanSend(socket)) + { + _ = RemoveConnectionAsync(connectionId); + return false; + } + + _ = SendMessageAsync(connectionId, socket, message); + return true; + } + + public void SendMessage(IEnumerable connectionIds, object message) + { + foreach (var connectionId in connectionIds) + SendMessage(connectionId, message); + } + + public void SendMessageToAll(object message) + { + foreach (var (connectionId, socket) in _connections) + { + if (!CanSend(socket)) + { + _ = RemoveConnectionAsync(connectionId); + continue; + } + + _ = SendMessageAsync(connectionId, socket, message); + } + } + + public void Dispose() + { + _timer?.Dispose(); + } + + private async Task SendKeepAliveAsync(string connectionId, WebSocket socket) + { + try + { + await socket.SendAsync(KeepAliveMessage, WebSocketMessageType.Text, true, CancellationToken.None).ConfigureAwait(false); + } + catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) + { + await RemoveConnectionAsync(connectionId).ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + await RemoveConnectionAsync(connectionId).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Error sending websocket keepalive for {ConnectionId}", connectionId); + } + } + + private async Task SendMessageAsync(string connectionId, WebSocket socket, object message) + { + try + { + string serializedMessage = _serializer.SerializeToString(message); + byte[] bytes = Encoding.UTF8.GetBytes(serializedMessage); + await socket.SendAsync(new ArraySegment(bytes, 0, bytes.Length), WebSocketMessageType.Text, true, CancellationToken.None).ConfigureAwait(false); + } + catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) + { + await RemoveConnectionAsync(connectionId).ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + await RemoveConnectionAsync(connectionId).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Error sending websocket message for {ConnectionId}", connectionId); + } + } + + private static bool CanSend(WebSocket socket) + { + return socket.State is WebSocketState.Open; + } +} diff --git a/src/Exceptionless.Web/Hubs/WebSocketPushMiddleware.cs b/src/Exceptionless.Web/Hubs/WebSocketPushMiddleware.cs new file mode 100644 index 0000000000..467d5062fa --- /dev/null +++ b/src/Exceptionless.Web/Hubs/WebSocketPushMiddleware.cs @@ -0,0 +1,109 @@ +using System.Net.WebSockets; +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Utility; + +namespace Exceptionless.Web.Hubs; + +public sealed class WebSocketPushMiddleware +{ + private static readonly PathString PushEndpoint = new("/api/v2/push"); + private readonly ILogger _logger; + private readonly WebSocketConnectionManager _connectionManager; + private readonly IConnectionMapping _connectionMapping; + private readonly RequestDelegate _next; + + public WebSocketPushMiddleware(RequestDelegate next, WebSocketConnectionManager connectionManager, IConnectionMapping connectionMapping, ILogger logger) + { + _next = next; + _connectionManager = connectionManager; + _connectionMapping = connectionMapping; + _logger = logger; + } + + public async Task Invoke(HttpContext context) + { + if (!context.Request.Path.StartsWithSegments(PushEndpoint, StringComparison.Ordinal) + || !context.WebSockets.IsWebSocketRequest) + { + await _next(context); + return; + } + + if (!context.User.IsAuthenticated()) + { + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + return; + } + + string? userId = context.User.GetUserId(); + if (String.IsNullOrEmpty(userId)) + { + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + return; + } + + var existingConnections = await _connectionMapping.GetUserIdConnectionsAsync(userId); + if (existingConnections.Count >= _connectionManager.MaxConnectionsPerUser) + { + _logger.LogWarning("User {UserId} exceeded max websocket push connections ({Max})", userId, _connectionManager.MaxConnectionsPerUser); + context.Response.StatusCode = StatusCodes.Status429TooManyRequests; + return; + } + + using var socket = await context.WebSockets.AcceptWebSocketAsync(); + string connectionId = _connectionManager.AddConnection(socket); + await OnConnected(context, connectionId).ConfigureAwait(false); + + try + { + await ReceiveUntilCloseAsync(socket, context.RequestAborted).ConfigureAwait(false); + } + catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) { } + catch (OperationCanceledException) { } + finally + { + await OnDisconnected(context, connectionId).ConfigureAwait(false); + await _connectionManager.RemoveConnectionAsync(connectionId).ConfigureAwait(false); + } + } + + private async Task OnConnected(HttpContext context, string connectionId) + { + _logger.LogTrace("WebSocket push connected {ConnectionId}", connectionId); + + foreach (string organizationId in context.User.GetOrganizationIds()) + await _connectionMapping.GroupAddAsync(organizationId, connectionId).ConfigureAwait(false); + + string? userId = context.User.GetUserId(); + if (!String.IsNullOrEmpty(userId)) + await _connectionMapping.UserIdAddAsync(userId, connectionId).ConfigureAwait(false); + } + + private async Task OnDisconnected(HttpContext context, string connectionId) + { + _logger.LogTrace("WebSocket push disconnected {ConnectionId}", connectionId); + + foreach (string organizationId in context.User.GetOrganizationIds()) + await _connectionMapping.GroupRemoveAsync(organizationId, connectionId).ConfigureAwait(false); + + string? userId = context.User.GetUserId(); + if (!String.IsNullOrEmpty(userId)) + await _connectionMapping.UserIdRemoveAsync(userId, connectionId).ConfigureAwait(false); + } + + private static async Task ReceiveUntilCloseAsync(WebSocket socket, CancellationToken cancellationToken) + { + var buffer = new byte[4096]; + + while (socket.State is WebSocketState.Open) + { + WebSocketReceiveResult result; + do + { + result = await socket.ReceiveAsync(new ArraySegment(buffer), cancellationToken).ConfigureAwait(false); + if (result.MessageType is WebSocketMessageType.Close) + return; + } while (!result.EndOfMessage); + } + } +} diff --git a/src/Exceptionless.Web/Startup.cs b/src/Exceptionless.Web/Startup.cs index b94ddcd57b..6cedc2ddb0 100644 --- a/src/Exceptionless.Web/Startup.cs +++ b/src/Exceptionless.Web/Startup.cs @@ -307,6 +307,8 @@ ApplicationException applicationException when applicationException.Message.Cont if (options.EnablePush) { + app.UseWebSockets(); + app.UseMiddleware(); app.UseMiddleware(); } diff --git a/tests/Exceptionless.Tests/Hubs/SseIntegrationTests.cs b/tests/Exceptionless.Tests/Hubs/SseIntegrationTests.cs index 8b772d5f49..55f73f9bdc 100644 --- a/tests/Exceptionless.Tests/Hubs/SseIntegrationTests.cs +++ b/tests/Exceptionless.Tests/Hubs/SseIntegrationTests.cs @@ -3,8 +3,10 @@ using Exceptionless.Core.Messaging.Models; using Exceptionless.Core.Models; using Exceptionless.Core.Utility; +using Exceptionless.Tests.Extensions; using Exceptionless.Tests.Utility; using Exceptionless.Web.Hubs; +using Exceptionless.Web.Models; using Foundatio.Messaging; using Foundatio.Repositories.Models; using Xunit; @@ -25,6 +27,12 @@ public SseIntegrationTests(ITestOutputHelper output, AppWebHostFactory factory) _messagePublisher = GetService(); } + protected override async Task ResetDataAsync() + { + await base.ResetDataAsync(); + await GetService().CreateDataAsync(); + } + [Fact] public async Task ConnectWithValidToken_ReturnsEventStream() { @@ -33,7 +41,7 @@ public async Task ConnectWithValidToken_ReturnsEventStream() using var client = _server.CreateClient(); using var request = new HttpRequestMessage(HttpMethod.Get, "/api/v2/push"); request.Headers.Add("Accept", "text/event-stream"); - request.Headers.Add("Authorization", $"Bearer {token.Id}"); + request.Headers.Add("Authorization", $"Bearer {token}"); using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(5)); @@ -78,7 +86,7 @@ public async Task ConnectWithAccessTokenQueryParam_Succeeds() var token = await CreateTokenAsync(); using var client = _server.CreateClient(); - using var request = new HttpRequestMessage(HttpMethod.Get, $"/api/v2/push?access_token={token.Id}"); + using var request = new HttpRequestMessage(HttpMethod.Get, $"/api/v2/push?access_token={token}"); request.Headers.Add("Accept", "text/event-stream"); using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); @@ -92,12 +100,12 @@ public async Task ConnectWithAccessTokenQueryParam_Succeeds() public async Task ConnectedClient_ReceivesEntityChangedMessage() { var token = await CreateTokenAsync(); - var orgId = token.OrganizationId; + var orgId = SampleDataService.TEST_ORG_ID; using var client = _server.CreateClient(); using var request = new HttpRequestMessage(HttpMethod.Get, "/api/v2/push"); request.Headers.Add("Accept", "text/event-stream"); - request.Headers.Add("Authorization", $"Bearer {token.Id}"); + request.Headers.Add("Authorization", $"Bearer {token}"); using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); @@ -144,7 +152,7 @@ public async Task SseEndpoint_IsExemptFromThrottling() { using var request = new HttpRequestMessage(HttpMethod.Get, "/api/v2/push"); request.Headers.Add("Accept", "text/event-stream"); - request.Headers.Add("Authorization", $"Bearer {token.Id}"); + request.Headers.Add("Authorization", $"Bearer {token}"); using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(3)); @@ -188,9 +196,19 @@ public async Task SseEndpoint_IsExemptFromThrottling() return sb.Length > 0 ? sb.ToString() : null; } - private async Task CreateTokenAsync() + private async Task CreateTokenAsync() { - var tokenData = GetService(); - return tokenData.GenerateSampleUserToken(); + var result = await SendRequestAsAsync(r => r + .Post() + .AppendPath("auth/login") + .Content(new Login + { + Email = SampleDataService.TEST_USER_EMAIL, + Password = SampleDataService.TEST_USER_PASSWORD + }) + .StatusCodeShouldBeOk() + ); + + return result?.Token ?? throw new InvalidOperationException("Login did not return a token."); } } diff --git a/tests/Exceptionless.Tests/Hubs/SseTests.cs b/tests/Exceptionless.Tests/Hubs/SseTests.cs index 396fb172bd..40960def89 100644 --- a/tests/Exceptionless.Tests/Hubs/SseTests.cs +++ b/tests/Exceptionless.Tests/Hubs/SseTests.cs @@ -456,4 +456,22 @@ public async Task Capacity_Exceeded_DropsOldest() Assert.Equal("msg-3", item2!.Value.Data); Assert.Equal("msg-4", item3!.Value.Data); } + + [Fact] + public async Task CriticalMessage_WhenQueueFull_DropsOldestDroppableMessageFirst() + { + var queue = new SseConnection.DedupQueue(2); + queue.TryEnqueue(new SseConnection.SseEvent { Data = "lossy-1", DedupeKey = "lossy-1", CanDrop = true }); + queue.TryEnqueue(new SseConnection.SseEvent { Data = "critical-1", CanDrop = false }); + + var result = queue.TryEnqueue(new SseConnection.SseEvent { Data = "critical-2", CanDrop = false }); + + using var cts = new CancellationTokenSource(); + var item1 = await queue.DequeueAsync(cts.Token); + var item2 = await queue.DequeueAsync(cts.Token); + + Assert.Equal(SseConnection.EnqueueResult.DroppedQueuedMessage, result); + Assert.Equal("critical-1", item1!.Value.Data); + Assert.Equal("critical-2", item2!.Value.Data); + } } diff --git a/tests/Exceptionless.Tests/Hubs/TestWebSocket.cs b/tests/Exceptionless.Tests/Hubs/TestWebSocket.cs new file mode 100644 index 0000000000..54aabbe351 --- /dev/null +++ b/tests/Exceptionless.Tests/Hubs/TestWebSocket.cs @@ -0,0 +1,53 @@ +using System.Net.WebSockets; +using System.Text; + +namespace Exceptionless.Tests.Hubs; + +internal sealed class TestWebSocket : WebSocket +{ + private WebSocketState _state; + private int _closeCount; + + public TestWebSocket(WebSocketState state = WebSocketState.Open) + { + _state = state; + } + + public int CloseCount => _closeCount; + public List SentMessages { get; } = []; + public override WebSocketCloseStatus? CloseStatus { get; } = WebSocketCloseStatus.NormalClosure; + public override string? CloseStatusDescription { get; } = "Closed"; + public override string? SubProtocol { get; } = null; + public override WebSocketState State => _state; + + public override void Abort() + { + _state = WebSocketState.Aborted; + } + + public override Task CloseAsync(WebSocketCloseStatus closeStatus, string? statusDescription, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _closeCount); + _state = WebSocketState.Closed; + return Task.CompletedTask; + } + + public override Task CloseOutputAsync(WebSocketCloseStatus closeStatus, string? statusDescription, CancellationToken cancellationToken) + { + _state = WebSocketState.CloseSent; + return Task.CompletedTask; + } + + public override void Dispose() { } + + public override Task ReceiveAsync(ArraySegment buffer, CancellationToken cancellationToken) + { + return Task.FromResult(new WebSocketReceiveResult(0, WebSocketMessageType.Text, true)); + } + + public override Task SendAsync(ArraySegment buffer, WebSocketMessageType messageType, bool endOfMessage, CancellationToken cancellationToken) + { + SentMessages.Add(Encoding.UTF8.GetString(buffer.Array!, buffer.Offset, buffer.Count)); + return Task.CompletedTask; + } +} diff --git a/tests/Exceptionless.Tests/Hubs/WebSocketCompatibilityTests.cs b/tests/Exceptionless.Tests/Hubs/WebSocketCompatibilityTests.cs new file mode 100644 index 0000000000..755f1b2b70 --- /dev/null +++ b/tests/Exceptionless.Tests/Hubs/WebSocketCompatibilityTests.cs @@ -0,0 +1,156 @@ +using System.Net.WebSockets; +using Exceptionless.Core; +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models; +using Exceptionless.Core.Utility; +using Exceptionless.Web.Hubs; +using Foundatio.Repositories.Models; +using Foundatio.Serializer; +using Xunit; + +namespace Exceptionless.Tests.Hubs; + +public sealed class WebSocketConnectionManagerTests : TestWithServices +{ + public WebSocketConnectionManagerTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void AddConnection_NewSocket_CanLookupAndEnumerateConnection() + { + using var manager = CreateManager(); + var socket = new TestWebSocket(); + + string connectionId = manager.AddConnection(socket); + + Assert.False(String.IsNullOrEmpty(connectionId)); + Assert.Same(socket, manager.GetConnectionById(connectionId)); + Assert.Same(socket, Assert.Single(manager.GetAll())); + } + + [Fact] + public async Task RemoveConnectionAsync_ExistingConnection_RemovesAndClosesSocket() + { + using var manager = CreateManager(); + var socket = new TestWebSocket(); + string connectionId = manager.AddConnection(socket); + + await manager.RemoveConnectionAsync(connectionId); + + Assert.Null(manager.GetConnectionById(connectionId)); + Assert.Empty(manager.GetAll()); + Assert.Equal(1, socket.CloseCount); + Assert.Equal(WebSocketState.Closed, socket.State); + } + + [Fact] + public void SendMessage_ClosedSocket_ReturnsFalseAndRemovesConnection() + { + using var manager = CreateManager(); + var socket = new TestWebSocket(WebSocketState.Closed); + string connectionId = manager.AddConnection(socket); + + bool sent = manager.SendMessage(connectionId, new { type = "test" }); + + Assert.False(sent); + Assert.Null(manager.GetConnectionById(connectionId)); + } + + private WebSocketConnectionManager CreateManager() + { + var options = new AppOptions { EnablePush = true }; + return new WebSocketConnectionManager(options, GetService(), Log); + } +} + +public sealed class PushCompatibilityBrokerTests : TestWithServices +{ + private readonly MessageBusBroker _broker; + private readonly IConnectionMapping _connectionMapping; + private readonly SseConnectionManager _sseConnectionManager; + private readonly WebSocketConnectionManager _webSocketConnectionManager; + + public PushCompatibilityBrokerTests(ITestOutputHelper output) : base(output) + { + _broker = GetService(); + _connectionMapping = GetService(); + _sseConnectionManager = GetService(); + _webSocketConnectionManager = GetService(); + } + + [Fact] + public async Task OnEntityChangedAsync_FansOutToSseAndWebSocketConnections() + { + const string organizationId = "compat-org"; + using var response = new FakeHttpResponse(); + using var cts = new CancellationTokenSource(); + var socket = new TestWebSocket(); + + string sseConnectionId = "compat-sse"; + string webSocketConnectionId = _webSocketConnectionManager.AddConnection(socket); + _sseConnectionManager.AddConnection(sseConnectionId, response, cts.Token); + + try + { + await _connectionMapping.GroupAddAsync(organizationId, sseConnectionId); + await _connectionMapping.GroupAddAsync(organizationId, webSocketConnectionId); + + var entityChanged = new EntityChanged + { + Id = "stack-compat", + Type = "Stack", + ChangeType = ChangeType.Saved + }; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.OrganizationId] = organizationId; + + await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); + await Task.Delay(200, TestContext.Current.CancellationToken); + + Assert.Contains("StackChanged", response.WrittenData); + Assert.Single(socket.SentMessages); + Assert.Contains("StackChanged", socket.SentMessages[0]); + } + finally + { + await _connectionMapping.GroupRemoveAsync(organizationId, sseConnectionId); + await _connectionMapping.GroupRemoveAsync(organizationId, webSocketConnectionId); + await _sseConnectionManager.RemoveConnectionAsync(sseConnectionId); + await _webSocketConnectionManager.RemoveConnectionAsync(webSocketConnectionId); + } + } + + [Fact] + public async Task OnEntityChangedAsync_AuthTokenRemoved_ClosesWebSocketConnectionsAndClearsMapping() + { + const string userId = "compat-user"; + const string organizationId = "compat-org"; + var socket = new TestWebSocket(); + string connectionId = _webSocketConnectionManager.AddConnection(socket); + + try + { + await _connectionMapping.UserIdAddAsync(userId, connectionId); + await _connectionMapping.GroupAddAsync(organizationId, connectionId); + + var entityChanged = new EntityChanged + { + Id = "compat-token", + Type = nameof(Token), + ChangeType = ChangeType.Removed + }; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.OrganizationId] = organizationId; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.UserId] = userId; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.IsAuthenticationToken] = true; + + await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); + + Assert.Null(_webSocketConnectionManager.GetConnectionById(connectionId)); + Assert.Equal(1, socket.CloseCount); + Assert.Empty(await _connectionMapping.GetUserIdConnectionsAsync(userId)); + } + finally + { + await _connectionMapping.GroupRemoveAsync(organizationId, connectionId); + await _connectionMapping.UserIdRemoveAsync(userId, connectionId); + } + } +} diff --git a/tests/http/push.http b/tests/http/push.http new file mode 100644 index 0000000000..db874cbf25 --- /dev/null +++ b/tests/http/push.http @@ -0,0 +1,29 @@ +@url = http://localhost:7110 +@apiUrl = {{url}}/api/v2 +@email = admin@exceptionless.test +@password = tester + +### login to test account +# @name login +POST {{apiUrl}}/auth/login +Content-Type: application/json + +{ + "email": "{{email}}", + "password": "{{password}}" +} + +### + +@token = {{login.response.body.$.token}} + +### SSE push via bearer token +# This request intentionally stays open. Cancel it manually after verifying headers/events. +GET {{apiUrl}}/push +Accept: text/event-stream +Authorization: Bearer {{token}} + +### SSE push via query-string token +# This request intentionally stays open. Cancel it manually after verifying headers/events. +GET {{apiUrl}}/push?access_token={{token}} +Accept: text/event-stream From 86dcf1de925b7e95167428f3f83f5a4fd73898c6 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 28 May 2026 06:48:44 -0500 Subject: [PATCH 3/7] =?UTF-8?q?fix:=20finalize=20SSE=20rollout=20safety=20?= =?UTF-8?q?=E2=80=94=20async=20disposal,=20graceful=20drain,=20test=20conf?= =?UTF-8?q?ig?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - SseConnectionManager: implement IAsyncDisposable so DI container on .NET 8+ calls DisposeAsync() at shutdown, eliminating the blocking GetAwaiter().GetResult() that could deadlock under async test hosts; sync Dispose() delegates to DisposeAsync() - Program.cs: set ShutdownTimeout = 45s to match k8s drain window (terminationGracePeriodSeconds 60s - preStop sleep 15s = 45s for ASP.NET Core drain) - k8s api.yaml: add terminationGracePeriodSeconds: 60 and preStop: sleep 15 lifecycle hook so the ALB/ingress deregisters the pod before SIGTERM cancels active SSE connections - tests/appsettings.yml: EnableWebSockets → EnablePush (canonical new key) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- k8s/exceptionless/templates/api.yaml | 12 ++++++++++++ src/Exceptionless.Web/Hubs/SseConnectionManager.cs | 14 +++++++++++--- src/Exceptionless.Web/Program.cs | 6 ++++++ tests/Exceptionless.Tests/appsettings.yml | 2 +- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/k8s/exceptionless/templates/api.yaml b/k8s/exceptionless/templates/api.yaml index e0ba872569..37016f2779 100644 --- a/k8s/exceptionless/templates/api.yaml +++ b/k8s/exceptionless/templates/api.yaml @@ -28,6 +28,11 @@ spec: annotations: checksum/config: {{ include (print $.Template.BasePath "/config.yaml") . | sha256sum }} spec: + # SSE connections are long-lived; give the pod enough time to drain before SIGTERM. + # The preStop sleep lets the ALB/ingress controller deregister the pod before traffic stops, + # then the remaining window allows ASP.NET Core to cancel RequestAborted tokens and clean up. + # When push is eventually enabled behind a Gateway API RoutePolicy, revisit this value. + terminationGracePeriodSeconds: 60 topologySpreadConstraints: - maxSkew: 1 topologyKey: kubernetes.io/hostname @@ -39,6 +44,13 @@ spec: - name: {{ template "exceptionless.name" . }}-api image: "{{ .Values.api.image.repository }}:{{ .Values.version }}" imagePullPolicy: {{ .Values.api.image.pullPolicy }} + lifecycle: + preStop: + # Give the ALB ~15s to deregister this pod before SIGTERM fires. + # The total graceful window is terminationGracePeriodSeconds (60s) minus + # this sleep, leaving ~45s for ASP.NET Core to drain active SSE connections. + exec: + command: ["sleep", "15"] livenessProbe: httpGet: path: /health diff --git a/src/Exceptionless.Web/Hubs/SseConnectionManager.cs b/src/Exceptionless.Web/Hubs/SseConnectionManager.cs index 655b5e316f..063f503ce9 100644 --- a/src/Exceptionless.Web/Hubs/SseConnectionManager.cs +++ b/src/Exceptionless.Web/Hubs/SseConnectionManager.cs @@ -9,7 +9,7 @@ namespace Exceptionless.Web.Hubs; /// Sends keep-alive comments every 15 seconds to prevent proxy/LB disconnects. /// Proactively prunes dead connections during keep-alive sweeps. /// -public sealed class SseConnectionManager : IDisposable +public sealed class SseConnectionManager : IDisposable, IAsyncDisposable { private readonly ConcurrentDictionary _connections = new(); private readonly ConcurrentDictionary> _pendingDisposals = new(); @@ -143,10 +143,17 @@ public void SendMessageToAll(object message, bool canDrop = true) } public void Dispose() + { + // Synchronous disposal: used by test hosts and non-async disposal paths. + // For production host shutdown, the DI container will prefer DisposeAsync(). + DisposeAsync().AsTask().GetAwaiter().GetResult(); + } + + public async ValueTask DisposeAsync() { _timer?.Dispose(); - var disposeTasks = new HashSet(); + var disposeTasks = new List(); foreach (var (connectionId, connection) in _connections) { @@ -157,7 +164,8 @@ public void Dispose() foreach (var pendingDisposal in _pendingDisposals.Values) disposeTasks.Add(pendingDisposal.Value); - Task.WhenAll(disposeTasks).GetAwaiter().GetResult(); + if (disposeTasks.Count > 0) + await Task.WhenAll(disposeTasks).ConfigureAwait(false); } private Task DisposeConnectionAsync(string connectionId, SseConnection connection) diff --git a/src/Exceptionless.Web/Program.cs b/src/Exceptionless.Web/Program.cs index db9dd79749..5d3a9ed460 100644 --- a/src/Exceptionless.Web/Program.cs +++ b/src/Exceptionless.Web/Program.cs @@ -73,6 +73,12 @@ public static IHostBuilder CreateHostBuilder(IConfigurationRoot config, string e var builder = Host.CreateDefaultBuilder() .UseEnvironment(environment) + .ConfigureHostOptions(o => + { + // Align with k8s terminationGracePeriodSeconds (60s) minus preStop sleep (15s). + // Gives ASP.NET Core 45s to drain active SSE connections before the pod is force-killed. + o.ShutdownTimeout = TimeSpan.FromSeconds(45); + }) .ConfigureLogging(b => b.ClearProviders()) // clears .net providers since we are telling serilog to write to providers we only want it to be the otel provider .UseSerilog((ctx, sp, c) => { diff --git a/tests/Exceptionless.Tests/appsettings.yml b/tests/Exceptionless.Tests/appsettings.yml index 0daab2e20d..f4c6494257 100644 --- a/tests/Exceptionless.Tests/appsettings.yml +++ b/tests/Exceptionless.Tests/appsettings.yml @@ -20,7 +20,7 @@ EnableDailySummary: false # Runs the jobs in the current website process RunJobsInProcess: false -EnableWebSockets: true +EnablePush: true Serilog: MinimumLevel: Warning From 7398308ff99875a17fac6c83b38aa0a7a099c7cf Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 28 May 2026 07:24:11 -0500 Subject: [PATCH 4/7] fix: keep Angular websocket-first during rollout - restore the legacy Angular push service to prefer WebSocket so cached and in-flight Angular clients keep their current release-notification refresh behavior during rollout - add a thin SSE fallback for Angular if WebSocket cannot be established at startup, while leaving the Svelte app on SSE only - preserve the legacy WebSocket manager/test surface with temporary compatibility wrappers and explicit deprecation comments so the backend can dual-write with minimal churn - harden AppWebHostFactory Elasticsearch slice reuse so rebased full-suite runs clean up stale test indices before startup and do not return a slice to the reuse pool until the previous host is fully disposed Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../components/websocket/websocket-service.js | 298 +++++++++++------- .../Hubs/WebSocketConnectionManager.cs | 42 +++ .../Hubs/WebSocketPushMiddleware.cs | 4 + .../Exceptionless.Tests/AppWebHostFactory.cs | 71 ++++- .../Hubs/WebSocketCompatibilityTests.cs | 4 +- .../Hubs/WebSocketConnectionManagerTests.cs | 83 +++++ .../Hubs/WebSocketTests.cs | 113 +++++++ 7 files changed, 485 insertions(+), 130 deletions(-) create mode 100644 tests/Exceptionless.Tests/Hubs/WebSocketConnectionManagerTests.cs create mode 100644 tests/Exceptionless.Tests/Hubs/WebSocketTests.cs diff --git a/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js b/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js index f720bb459a..83d1ad52ab 100644 --- a/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js +++ b/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js @@ -4,150 +4,212 @@ angular .module("exceptionless.websocket", ["app.config", "exceptionless", "exceptionless.auth"]) .factory("websocketService", function ($ExceptionlessClient, $rootScope, $timeout, authService, BASE_URL) { + var ResilientWebSocket = (function () { + function ResilientWebSocket(url, protocols) { + if (protocols === void 0) { + protocols = []; + } + this.reconnectInterval = 1000; + this.timeoutInterval = 2000; + this.forcedClose = false; + this.timedOut = false; + this.hasConnectedOnce = false; + this.protocols = []; + this.onopen = function (event) {}; + this.onclose = function (event) {}; + this.onconnecting = function () {}; + this.onmessage = function (event) {}; + this.onerror = function (event) {}; + this.ontransportfallback = function (event) { + return false; + }; + this.url = url; + this.protocols = protocols; + this.readyState = WebSocket.CONNECTING; + this.connect(false); + } + + ResilientWebSocket.prototype.connect = function (reconnectAttempt) { + var _this = this; + this.ws = new WebSocket(this.url, this.protocols); + this.onconnecting(); + var localWs = this.ws; + var timeout = setTimeout(function () { + _this.timedOut = true; + localWs.close(); + _this.timedOut = false; + }, this.timeoutInterval); + this.ws.onopen = function (event) { + clearTimeout(timeout); + _this.readyState = WebSocket.OPEN; + _this.hasConnectedOnce = true; + reconnectAttempt = false; + _this.onopen(event); + }; + this.ws.onclose = function (event) { + clearTimeout(timeout); + _this.ws = null; + if (_this.forcedClose) { + _this.readyState = WebSocket.CLOSED; + _this.onclose(event); + } else if (!_this.hasConnectedOnce && _this.ontransportfallback(event) === true) { + _this.readyState = WebSocket.CLOSED; + } else { + _this.readyState = WebSocket.CONNECTING; + _this.onconnecting(); + if (!reconnectAttempt && !_this.timedOut) { + _this.onclose(event); + } + setTimeout(function () { + _this.connect(true); + }, _this.reconnectInterval); + } + }; + this.ws.onmessage = function (event) { + _this.onmessage(event); + }; + this.ws.onerror = function (event) { + _this.onerror(event); + }; + }; + ResilientWebSocket.prototype.send = function (data) { + if (this.ws) { + return this.ws.send(data); + } + throw new Error("INVALID_STATE_ERR : Pausing to reconnect websocket"); + }; + ResilientWebSocket.prototype.close = function () { + if (this.ws) { + this.forcedClose = true; + this.ws.close(); + return true; + } + return false; + }; + ResilientWebSocket.prototype.refresh = function () { + if (this.ws) { + this.ws.close(); + return true; + } + + return false; + }; + + return ResilientWebSocket; + })(); + var source = "exceptionless.websocket.websocketService"; - var _abortController; - var _reconnectTimeout; - var _reconnectAttempts = 0; - var _forcedClose = false; + var _connection; + var _websocketTimeout; function start() { startDelayed(1); } function startDelayed(delay) { - if (_abortController || _reconnectTimeout) { + function startImpl() { + if (supportsWebSocket() && startWebSocket()) { + return; + } + + if (!startSse()) { + $ExceptionlessClient.submitLog("No supported push transport is available.", "warn", source); + } + } + + if (_connection || _websocketTimeout) { stop(); } - _reconnectTimeout = $timeout(function () { - _reconnectTimeout = null; - connect(); - }, delay || 1000); + _websocketTimeout = $timeout(startImpl, delay || 1000); } - function connect() { - _forcedClose = false; - _abortController = new AbortController(); - var signal = _abortController.signal; - - var url = BASE_URL + "/api/v2/push"; - var token = authService.getToken(); - - fetch(url, { - headers: { - Accept: "text/event-stream", - Authorization: "Bearer " + token, - }, - signal: signal, - }) - .then(function (response) { - if (!response.ok) { - if (response.status === 401 || response.status === 403) { - // Auth failure - don't reconnect - return; - } - throw new Error("SSE connection failed: " + response.status); - } + function startWebSocket() { + // Keep WebSocket as the preferred Angular transport during rollout so existing + // release notification refresh behavior stays unchanged until SSE fully replaces it. + try { + _connection = new ResilientWebSocket(getWebSocketPushUrl()); + } catch (error) { + _connection = null; + return false; + } - _reconnectAttempts = 0; - var reader = response.body.getReader(); - var decoder = new TextDecoder(); - var buffer = ""; - - function readChunk() { - return reader.read().then(function (result) { - if (result.done) { - if (!_forcedClose) { - scheduleReconnect(); - } - return; - } - - buffer += decoder.decode(result.value, { stream: true }); - - var messages = buffer.split("\n\n"); - buffer = messages.pop() || ""; - - messages.forEach(function (message) { - if (!message.trim()) return; - - var lines = message.split("\n"); - var data = ""; - - lines.forEach(function (line) { - if (line.indexOf("data: ") === 0) { - data += line.slice(6); - } else if (line.indexOf("data:") === 0) { - data += line.slice(5); - } - // Comments (: keepalive) are ignored - }); - - if (data) { - var parsed = JSON.parse(data); - if (!parsed || !parsed.type) { - return; - } - - if (parsed.message && parsed.message.change_type >= 0) { - parsed.message.added = parsed.message.change_type === 0; - parsed.message.updated = parsed.message.change_type === 1; - parsed.message.deleted = parsed.message.change_type === 2; - } - - $rootScope.$emit(parsed.type, parsed.message); - - // This event is fired when a user is added or removed from an organization. - if (parsed.type === "UserMembershipChanged" && parsed.message && parsed.message.organization_id) { - $rootScope.$emit("OrganizationChanged", parsed.message); - $rootScope.$emit("ProjectChanged", parsed.message); - } - } - }); - - return readChunk(); - }); - } + _connection.ontransportfallback = function () { + return startSse(); + }; + _connection.onmessage = function (ev) { + handleMessage(ev.data); + }; - return readChunk(); - }) - .catch(function (error) { - if (signal.aborted && _forcedClose) { - return; - } + return true; + } - if (!_forcedClose) { - scheduleReconnect(); - } - }); + function startSse() { + if (typeof EventSource === "undefined") { + return false; + } + + _connection = new EventSource(getSsePushUrl()); + _connection.onmessage = function (ev) { + handleMessage(ev.data); + }; + + return true; } - function scheduleReconnect() { - var multiplier = 1; + function handleMessage(payload) { + var data = payload ? JSON.parse(payload) : null; + if (!data || !data.type) { + return; + } - _reconnectAttempts++; - for (var attempt = 1; attempt < _reconnectAttempts; attempt++) { - multiplier *= 2; + if (data.message && data.message.change_type >= 0) { + data.message.added = data.message.change_type === 0; + data.message.updated = data.message.change_type === 1; + data.message.deleted = data.message.change_type === 2; } - var delay = Math.min(1000 * multiplier, 30000); - _reconnectTimeout = $timeout(function () { - _reconnectTimeout = null; - connect(); - }, delay); + $rootScope.$emit(data.type, data.message); + + // This event is fired when a user is added or removed from an organization. + if (data.type === "UserMembershipChanged" && data.message && data.message.organization_id) { + $rootScope.$emit("OrganizationChanged", data.message); + $rootScope.$emit("ProjectChanged", data.message); + } } function stop() { - if (_reconnectTimeout) { - $timeout.cancel(_reconnectTimeout); - _reconnectTimeout = null; + if (_websocketTimeout) { + $timeout.cancel(_websocketTimeout); + _websocketTimeout = null; + } + + if (_connection) { + var connection = _connection; + _connection = null; + + if (connection.close) { + connection.close(); + } } + } + + function supportsWebSocket() { + return typeof WebSocket !== "undefined"; + } - if (_abortController) { - _forcedClose = true; - _abortController.abort(); - _abortController = null; + function getWebSocketPushUrl() { + var pushUrl = getSsePushUrl(); + var protoMatch = /^(https?):\/\//; + if (BASE_URL.startsWith("https:")) { + return pushUrl.replace(protoMatch, "wss://"); } + + return pushUrl.replace(protoMatch, "ws://"); + } + + function getSsePushUrl() { + return BASE_URL + "/api/v2/push?access_token=" + authService.getToken(); } var service = { diff --git a/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs b/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs index c3b46571da..4a57f3de0c 100644 --- a/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs +++ b/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs @@ -6,6 +6,10 @@ namespace Exceptionless.Web.Hubs; +/// +/// Temporary WebSocket compatibility layer for the Angular rollout. Remove once the +/// SSE rollout is complete and the websocket active-connection gauge remains at zero. +/// public sealed class WebSocketConnectionManager : IDisposable { private static readonly ArraySegment KeepAliveMessage = new(Encoding.ASCII.GetBytes("{}"), 0, 2); @@ -50,11 +54,21 @@ private void SendKeepAlive(object? state) return _connections.TryGetValue(connectionId, out var socket) ? socket : null; } + public WebSocket? GetWebSocketById(string connectionId) + { + return GetConnectionById(connectionId); + } + public ICollection GetAll() { return _connections.Values; } + public string GetConnectionId(WebSocket socket) + { + return _connections.FirstOrDefault(pair => pair.Value == socket).Key; + } + public string AddConnection(WebSocket socket) { string connectionId = Guid.NewGuid().ToString("N"); @@ -64,6 +78,11 @@ public string AddConnection(WebSocket socket) return connectionId; } + public string AddWebSocket(WebSocket socket) + { + return AddConnection(socket); + } + public async Task RemoveConnectionAsync(string connectionId) { if (!_connections.TryRemove(connectionId, out var socket)) @@ -93,6 +112,11 @@ public async Task RemoveConnectionAsync(string connectionId) } } + public Task RemoveWebSocketAsync(string connectionId) + { + return RemoveConnectionAsync(connectionId); + } + public bool SendMessage(string connectionId, object message) { if (!_connections.TryGetValue(connectionId, out var socket)) @@ -108,12 +132,24 @@ public bool SendMessage(string connectionId, object message) return true; } + public Task SendMessageAsync(string connectionId, object message) + { + SendMessage(connectionId, message); + return Task.CompletedTask; + } + public void SendMessage(IEnumerable connectionIds, object message) { foreach (var connectionId in connectionIds) SendMessage(connectionId, message); } + public Task SendMessageAsync(IEnumerable connectionIds, object message) + { + SendMessage(connectionIds, message); + return Task.CompletedTask; + } + public void SendMessageToAll(object message) { foreach (var (connectionId, socket) in _connections) @@ -128,6 +164,12 @@ public void SendMessageToAll(object message) } } + public Task SendMessageToAllAsync(object message, bool throwOnError = true) + { + SendMessageToAll(message); + return Task.CompletedTask; + } + public void Dispose() { _timer?.Dispose(); diff --git a/src/Exceptionless.Web/Hubs/WebSocketPushMiddleware.cs b/src/Exceptionless.Web/Hubs/WebSocketPushMiddleware.cs index 467d5062fa..b5f3bceacb 100644 --- a/src/Exceptionless.Web/Hubs/WebSocketPushMiddleware.cs +++ b/src/Exceptionless.Web/Hubs/WebSocketPushMiddleware.cs @@ -4,6 +4,10 @@ namespace Exceptionless.Web.Hubs; +/// +/// Temporary WebSocket endpoint compatibility for the Angular rollout. Keep this in place +/// until all clients are on SSE and websocket active connections stay at zero. +/// public sealed class WebSocketPushMiddleware { private static readonly PathString PushEndpoint = new("/api/v2/push"); diff --git a/tests/Exceptionless.Tests/AppWebHostFactory.cs b/tests/Exceptionless.Tests/AppWebHostFactory.cs index 19aa9d17cf..1c90a93d37 100644 --- a/tests/Exceptionless.Tests/AppWebHostFactory.cs +++ b/tests/Exceptionless.Tests/AppWebHostFactory.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using System.Net; +using System.Text.Json; using Aspire.Hosting; using Aspire.Hosting.ApplicationModel; using Exceptionless.Insulation.Configuration; @@ -12,9 +13,10 @@ namespace Exceptionless.Tests; public class AppWebHostFactory : WebApplicationFactory, IAsyncLifetime { + private static readonly string[] s_indexPrefixes = ["events", "migrations", "organizations", "projects", "saved-views", "stacks", "tokens", "users", "webhooks"]; private static int s_counter = -1; private static readonly ConcurrentQueue s_pool = new(); - private static readonly Lazy> s_sharedApplication = new(StartSharedApplicationAsync, LazyThreadSafetyMode.ExecutionAndPublication); + private static readonly Lazy> s_sharedApplication = new(StartSharedApplicationAsync, LazyThreadSafetyMode.ExecutionAndPublication); private bool _sliceReleased; public AppWebHostFactory() @@ -32,10 +34,11 @@ public AppWebHostFactory() public async ValueTask InitializeAsync() { - _ = await s_sharedApplication.Value; + var sharedApplication = await s_sharedApplication.Value; + await CleanupElasticsearchSliceAsync(sharedApplication.ElasticsearchUri); } - private static async Task StartSharedApplicationAsync() + private static async Task StartSharedApplicationAsync() { var options = new DistributedApplicationOptions { AssemblyName = typeof(ElasticsearchResource).Assembly.FullName, DisableDashboard = true }; var builder = DistributedApplication.CreateBuilder(options); @@ -53,9 +56,10 @@ private static async Task StartSharedApplicationAsync() var connectionString = await elasticsearch.Resource.GetConnectionStringAsync() ?? throw new InvalidOperationException("Could not resolve Elasticsearch connection string."); - await WaitForElasticsearchAsync(new Uri(connectionString)); + var elasticsearchUri = new Uri(connectionString); + await WaitForElasticsearchAsync(elasticsearchUri); - return app; + return new SharedApplicationContext(app, elasticsearchUri); } private static async Task WaitForElasticsearchAsync(Uri elasticsearchUri) @@ -84,6 +88,41 @@ private static async Task WaitForElasticsearchAsync(Uri elasticsearchUri) throw new TimeoutException("Timed out waiting for Elasticsearch test container to be ready."); } + private async Task CleanupElasticsearchSliceAsync(Uri elasticsearchUri) + { + await WaitForElasticsearchAsync(elasticsearchUri); + + using var client = new HttpClient + { + BaseAddress = elasticsearchUri, + Timeout = TimeSpan.FromSeconds(10) + }; + + foreach (var prefix in s_indexPrefixes) + { + string pattern = Uri.EscapeDataString($"{AppScope}-{prefix}*"); + using var listResponse = await client.GetAsync($"/_cat/indices/{pattern}?h=index&format=json&expand_wildcards=all"); + if (listResponse.StatusCode == HttpStatusCode.NotFound) + continue; + + listResponse.EnsureSuccessStatusCode(); + + string payloadJson = await listResponse.Content.ReadAsStringAsync(); + var payload = JsonSerializer.Deserialize>(payloadJson, new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true + }) + ?? []; + + foreach (string indexName in payload.Select(record => record.Index).Where(name => !String.IsNullOrEmpty(name)).Distinct()) + { + using var deleteResponse = await client.DeleteAsync($"/{Uri.EscapeDataString(indexName)}?ignore_unavailable=true"); + if (deleteResponse.StatusCode != HttpStatusCode.NotFound) + deleteResponse.EnsureSuccessStatusCode(); + } + } + } + protected override void ConfigureWebHost(IWebHostBuilder builder) { builder.UseSolutionRelativeContentRoot("src/Exceptionless.Web", "*.slnx"); @@ -103,14 +142,26 @@ protected override IHostBuilder CreateHostBuilder() return Web.Program.CreateHostBuilder(config, Environments.Development); } - public override ValueTask DisposeAsync() + public override async ValueTask DisposeAsync() { - if (!_sliceReleased) + try { - s_pool.Enqueue(InstanceId); - _sliceReleased = true; + await base.DisposeAsync(); } + finally + { + if (!_sliceReleased) + { + s_pool.Enqueue(InstanceId); + _sliceReleased = true; + } + } + } - return base.DisposeAsync(); + private sealed record SharedApplicationContext(DistributedApplication Application, Uri ElasticsearchUri); + + private sealed class CatIndexRecord + { + public string Index { get; set; } = String.Empty; } } diff --git a/tests/Exceptionless.Tests/Hubs/WebSocketCompatibilityTests.cs b/tests/Exceptionless.Tests/Hubs/WebSocketCompatibilityTests.cs index 755f1b2b70..09045c5259 100644 --- a/tests/Exceptionless.Tests/Hubs/WebSocketCompatibilityTests.cs +++ b/tests/Exceptionless.Tests/Hubs/WebSocketCompatibilityTests.cs @@ -10,9 +10,9 @@ namespace Exceptionless.Tests.Hubs; -public sealed class WebSocketConnectionManagerTests : TestWithServices +public sealed class WebSocketConnectionCompatibilityTests : TestWithServices { - public WebSocketConnectionManagerTests(ITestOutputHelper output) : base(output) { } + public WebSocketConnectionCompatibilityTests(ITestOutputHelper output) : base(output) { } [Fact] public void AddConnection_NewSocket_CanLookupAndEnumerateConnection() diff --git a/tests/Exceptionless.Tests/Hubs/WebSocketConnectionManagerTests.cs b/tests/Exceptionless.Tests/Hubs/WebSocketConnectionManagerTests.cs new file mode 100644 index 0000000000..0af21c2074 --- /dev/null +++ b/tests/Exceptionless.Tests/Hubs/WebSocketConnectionManagerTests.cs @@ -0,0 +1,83 @@ +using System.Net.WebSockets; +using Exceptionless.Core; +using Exceptionless.Web.Hubs; +using Foundatio.Serializer; +using Xunit; + +namespace Exceptionless.Tests.Hubs; + +public sealed class WebSocketConnectionManagerTests : TestWithServices +{ + public WebSocketConnectionManagerTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void AddWebSocket_NewSocket_CanLookupAndEnumerateConnection() + { + using var manager = CreateManager(); + var socket = new TestWebSocket(); + + string connectionId = manager.AddWebSocket(socket); + + Assert.False(String.IsNullOrEmpty(connectionId)); + Assert.Same(socket, manager.GetWebSocketById(connectionId)); + Assert.Equal(connectionId, manager.GetConnectionId(socket)); + Assert.Same(socket, Assert.Single(manager.GetAll())); + } + + [Fact] + public async Task RemoveWebSocketAsync_ExistingConnection_RemovesAndClosesSocket() + { + using var manager = CreateManager(); + var socket = new TestWebSocket(); + string connectionId = manager.AddWebSocket(socket); + + await manager.RemoveWebSocketAsync(connectionId); + + Assert.Null(manager.GetWebSocketById(connectionId)); + Assert.Empty(manager.GetAll()); + Assert.Equal(1, socket.CloseCount); + Assert.Equal(WebSocketState.Closed, socket.State); + } + + [Fact] + public async Task RemoveWebSocketAsync_ClosedSocket_RemovesWithoutClosingAgain() + { + using var manager = CreateManager(); + var socket = new TestWebSocket(WebSocketState.Closed); + string connectionId = manager.AddWebSocket(socket); + + await manager.RemoveWebSocketAsync(connectionId); + + Assert.Null(manager.GetWebSocketById(connectionId)); + Assert.Empty(manager.GetAll()); + Assert.Equal(0, socket.CloseCount); + } + + [Fact] + public async Task RemoveWebSocketAsync_UnknownConnection_DoesNothing() + { + using var manager = CreateManager(); + + await manager.RemoveWebSocketAsync("missing"); + + Assert.Empty(manager.GetAll()); + } + + [Fact] + public async Task SendMessageToAllAsync_ClosedSockets_DoesNotSend() + { + using var manager = CreateManager(); + var socket = new TestWebSocket(WebSocketState.Closed); + manager.AddWebSocket(socket); + + await manager.SendMessageToAllAsync(new { type = "test" }); + + Assert.Empty(socket.SentMessages); + } + + private WebSocketConnectionManager CreateManager() + { + var options = new AppOptions { EnablePush = false }; + return new WebSocketConnectionManager(options, GetService(), Log); + } +} diff --git a/tests/Exceptionless.Tests/Hubs/WebSocketTests.cs b/tests/Exceptionless.Tests/Hubs/WebSocketTests.cs new file mode 100644 index 0000000000..1387058751 --- /dev/null +++ b/tests/Exceptionless.Tests/Hubs/WebSocketTests.cs @@ -0,0 +1,113 @@ +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models; +using Exceptionless.Core.Utility; +using Exceptionless.Web.Hubs; +using Foundatio.Repositories.Models; +using Xunit; + +namespace Exceptionless.Tests.Hubs; + +/// +/// Tests for WebSocket behavior. Calls +/// directly so they do not depend on +/// message bus wiring or EnablePush in test host configuration. +/// +public sealed class WebSocketTests : TestWithServices +{ + private readonly MessageBusBroker _broker; + private readonly IConnectionMapping _connectionMapping; + private readonly WebSocketConnectionManager _connectionManager; + + public WebSocketTests(ITestOutputHelper output) : base(output) + { + _broker = GetService(); + _connectionMapping = GetService(); + _connectionManager = GetService(); + } + + [Fact] + public async Task OnEntityChangedAsync_AuthTokenRemoved_ClosesWebSocketsAndClearsUserMapping() + { + const string userId = "test-user-id"; + const string organizationId = "test-organization-id"; + var socket1 = new TestWebSocket(); + var socket2 = new TestWebSocket(); + var unrelatedSocket = new TestWebSocket(); + + string connectionId1 = _connectionManager.AddWebSocket(socket1); + string connectionId2 = _connectionManager.AddWebSocket(socket2); + string unrelatedConnectionId = _connectionManager.AddWebSocket(unrelatedSocket); + + try + { + await _connectionMapping.UserIdAddAsync(userId, connectionId1); + await _connectionMapping.UserIdAddAsync(userId, connectionId2); + await _connectionMapping.GroupAddAsync(organizationId, connectionId1); + await _connectionMapping.GroupAddAsync(organizationId, connectionId2); + await _connectionMapping.GroupAddAsync(organizationId, unrelatedConnectionId); + + var entityChanged = new EntityChanged + { + Id = "test-token-id", + Type = nameof(Token), + ChangeType = ChangeType.Removed + }; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.OrganizationId] = organizationId; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.UserId] = userId; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.IsAuthenticationToken] = true; + + await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); + + Assert.Null(_connectionManager.GetWebSocketById(connectionId1)); + Assert.Null(_connectionManager.GetWebSocketById(connectionId2)); + Assert.Same(unrelatedSocket, _connectionManager.GetWebSocketById(unrelatedConnectionId)); + + Assert.Equal(1, socket1.CloseCount); + Assert.Equal(1, socket2.CloseCount); + Assert.Equal(0, unrelatedSocket.CloseCount); + + var remaining = await _connectionMapping.GetUserIdConnectionsAsync(userId); + Assert.Empty(remaining); + var organizationConnections = await _connectionMapping.GetGroupConnectionsAsync(organizationId); + Assert.DoesNotContain(connectionId1, organizationConnections); + Assert.DoesNotContain(connectionId2, organizationConnections); + Assert.Contains(unrelatedConnectionId, organizationConnections); + } + finally + { + await _connectionMapping.GroupRemoveAsync(organizationId, unrelatedConnectionId); + await _connectionManager.RemoveWebSocketAsync(unrelatedConnectionId); + } + } + + [Fact] + public async Task OnEntityChangedAsync_NonAuthTokenRemoved_DoesNotCloseWebSockets() + { + const string userId = "test-user-id-2"; + var socket = new TestWebSocket(); + string connectionId = _connectionManager.AddWebSocket(socket); + + try + { + await _connectionMapping.UserIdAddAsync(userId, connectionId); + + var entityChanged = new EntityChanged + { + Id = "test-api-token-id", + Type = nameof(Token), + ChangeType = ChangeType.Removed + }; + entityChanged.Data[ExtendedEntityChanged.KnownKeys.UserId] = userId; + + await _broker.OnEntityChangedAsync(entityChanged, CancellationToken.None); + + Assert.Equal(0, socket.CloseCount); + Assert.Same(socket, _connectionManager.GetWebSocketById(connectionId)); + } + finally + { + await _connectionMapping.UserIdRemoveAsync(userId, connectionId); + await _connectionManager.RemoveWebSocketAsync(connectionId); + } + } +} From b9f0e4ebf4996d18c1236f53c3477419a71d2b6d Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 28 May 2026 07:59:44 -0500 Subject: [PATCH 5/7] fix: address push rollout review feedback - restore the legacy Angular websocket client and leave only a deprecation note - tighten SSE and websocket lifecycle cleanup plus bounded queue behavior - fix client lint and typecheck failures and harden test-host Elasticsearch isolation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../components/websocket/websocket-service.js | 106 ++++-------------- .../features/websockets/sse-client.svelte.ts | 59 ++++++---- .../features/websockets/sse-client.test.ts | 65 ++++++----- .../ClientApp/src/routes/(app)/+layout.svelte | 24 +++- src/Exceptionless.Web/Hubs/SseConnection.cs | 51 +++++---- .../Hubs/SseConnectionManager.cs | 4 +- src/Exceptionless.Web/Hubs/SseMiddleware.cs | 48 ++++---- .../Hubs/WebSocketConnectionManager.cs | 27 ++++- .../Hubs/WebSocketPushMiddleware.cs | 2 +- .../Exceptionless.Tests/AppWebHostFactory.cs | 6 +- 10 files changed, 191 insertions(+), 201 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js b/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js index 83d1ad52ab..77fc55f16c 100644 --- a/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js +++ b/src/Exceptionless.Web/ClientApp.angular/components/websocket/websocket-service.js @@ -1,6 +1,7 @@ (function () { "use strict"; + // Deprecated: keep the legacy Angular client on WebSocket during the SSE rollout. angular .module("exceptionless.websocket", ["app.config", "exceptionless", "exceptionless.auth"]) .factory("websocketService", function ($ExceptionlessClient, $rootScope, $timeout, authService, BASE_URL) { @@ -13,16 +14,12 @@ this.timeoutInterval = 2000; this.forcedClose = false; this.timedOut = false; - this.hasConnectedOnce = false; this.protocols = []; this.onopen = function (event) {}; this.onclose = function (event) {}; this.onconnecting = function () {}; this.onmessage = function (event) {}; this.onerror = function (event) {}; - this.ontransportfallback = function (event) { - return false; - }; this.url = url; this.protocols = protocols; this.readyState = WebSocket.CONNECTING; @@ -42,7 +39,6 @@ this.ws.onopen = function (event) { clearTimeout(timeout); _this.readyState = WebSocket.OPEN; - _this.hasConnectedOnce = true; reconnectAttempt = false; _this.onopen(event); }; @@ -52,8 +48,6 @@ if (_this.forcedClose) { _this.readyState = WebSocket.CLOSED; _this.onclose(event); - } else if (!_this.hasConnectedOnce && _this.ontransportfallback(event) === true) { - _this.readyState = WebSocket.CLOSED; } else { _this.readyState = WebSocket.CONNECTING; _this.onconnecting(); @@ -108,13 +102,27 @@ function startDelayed(delay) { function startImpl() { - if (supportsWebSocket() && startWebSocket()) { - return; - } + _connection = new ResilientWebSocket(getPushUrl()); + _connection.onmessage = function (ev) { + var data = ev.data ? JSON.parse(ev.data) : null; + if (!data || !data.type) { + return; + } - if (!startSse()) { - $ExceptionlessClient.submitLog("No supported push transport is available.", "warn", source); - } + if (data.message && data.message.change_type >= 0) { + data.message.added = data.message.change_type === 0; + data.message.updated = data.message.change_type === 1; + data.message.deleted = data.message.change_type === 2; + } + + $rootScope.$emit(data.type, data.message); + + // This event is fired when a user is added or removed from an organization. + if (data.type === "UserMembershipChanged" && data.message && data.message.organization_id) { + $rootScope.$emit("OrganizationChanged", data.message); + $rootScope.$emit("ProjectChanged", data.message); + } + }; } if (_connection || _websocketTimeout) { @@ -124,60 +132,6 @@ _websocketTimeout = $timeout(startImpl, delay || 1000); } - function startWebSocket() { - // Keep WebSocket as the preferred Angular transport during rollout so existing - // release notification refresh behavior stays unchanged until SSE fully replaces it. - try { - _connection = new ResilientWebSocket(getWebSocketPushUrl()); - } catch (error) { - _connection = null; - return false; - } - - _connection.ontransportfallback = function () { - return startSse(); - }; - _connection.onmessage = function (ev) { - handleMessage(ev.data); - }; - - return true; - } - - function startSse() { - if (typeof EventSource === "undefined") { - return false; - } - - _connection = new EventSource(getSsePushUrl()); - _connection.onmessage = function (ev) { - handleMessage(ev.data); - }; - - return true; - } - - function handleMessage(payload) { - var data = payload ? JSON.parse(payload) : null; - if (!data || !data.type) { - return; - } - - if (data.message && data.message.change_type >= 0) { - data.message.added = data.message.change_type === 0; - data.message.updated = data.message.change_type === 1; - data.message.deleted = data.message.change_type === 2; - } - - $rootScope.$emit(data.type, data.message); - - // This event is fired when a user is added or removed from an organization. - if (data.type === "UserMembershipChanged" && data.message && data.message.organization_id) { - $rootScope.$emit("OrganizationChanged", data.message); - $rootScope.$emit("ProjectChanged", data.message); - } - } - function stop() { if (_websocketTimeout) { $timeout.cancel(_websocketTimeout); @@ -185,21 +139,13 @@ } if (_connection) { - var connection = _connection; + _connection.close(); _connection = null; - - if (connection.close) { - connection.close(); - } } } - function supportsWebSocket() { - return typeof WebSocket !== "undefined"; - } - - function getWebSocketPushUrl() { - var pushUrl = getSsePushUrl(); + function getPushUrl() { + var pushUrl = BASE_URL + "/api/v2/push?access_token=" + authService.getToken(); var protoMatch = /^(https?):\/\//; if (BASE_URL.startsWith("https:")) { return pushUrl.replace(protoMatch, "wss://"); @@ -208,10 +154,6 @@ return pushUrl.replace(protoMatch, "ws://"); } - function getSsePushUrl() { - return BASE_URL + "/api/v2/push?access_token=" + authService.getToken(); - } - var service = { start: start, startDelayed: startDelayed, diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts index 8bd3a014c5..11adf81186 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts @@ -26,6 +26,8 @@ export const SSE_CONNECTING = 0; export const SSE_OPEN = 1; export const SSE_CLOSED = 2; +// EventSource does not support custom Authorization headers, so the app uses fetch + +// ReadableStream to keep bearer tokens out of the query string. export class SseClient { public readyState = $state(SSE_CLOSED); @@ -48,6 +50,7 @@ export class SseClient { private _options: SseClientOptions; private _path: string; private _url: null | string = null; + private abortController: AbortController | null = null; private accessToken: null | string = null; private authFailed: boolean = false; private connectionTimeoutId: null | ReturnType = null; @@ -55,9 +58,8 @@ export class SseClient { private hasConnectedBefore: boolean = false; private reconnectAttempts: number = 0; private reconnectTimeoutId: null | ReturnType = null; - private streamGeneration: number = 0; - private abortController: AbortController | null = null; + private streamGeneration: number = 0; /** * @param path - SSE endpoint path (default: '/api/v2/push') @@ -79,7 +81,14 @@ export class SseClient { this.close(false); } - if (this.accessToken && visibility.visible && this.readyState === SSE_CLOSED && this.reconnectTimeoutId === null && !this.authFailed && !this.forcedClose) { + if ( + this.accessToken && + visibility.visible && + this.readyState === SSE_CLOSED && + this.reconnectTimeoutId === null && + !this.authFailed && + !this.forcedClose + ) { this.connect(); } }); @@ -147,6 +156,26 @@ export class SseClient { return Math.min(1000 * Math.pow(2, attempt - 1), 30000); } + private scheduleReconnect() { + if (this.reconnectTimeoutId !== null || this.authFailed || this.forcedClose || !(this.accessToken ?? accessToken.current)) { + this.readyState = SSE_CLOSED; + return; + } + + this.reconnectAttempts++; + const delay = this.getReconnectDelay(this.reconnectAttempts); + + this.readyState = SSE_CONNECTING; + this.onConnecting(true); + this.onClose(); + + clearTimeout(this.reconnectTimeoutId!); + this.reconnectTimeoutId = setTimeout(() => { + this.reconnectTimeoutId = null; + this.connect(); + }, delay); + } + private async startStream(signal: AbortSignal, isReconnect: boolean, generation: number) { try { const token = this.accessToken ?? accessToken.current; @@ -219,7 +248,9 @@ export class SseClient { buffer = messages.pop() ?? ''; for (const message of messages) { - if (!message.trim()) continue; + if (!message.trim()) { + continue; + } // Parse SSE format const lines = message.split('\n'); @@ -274,24 +305,4 @@ export class SseClient { this.scheduleReconnect(); } } - - private scheduleReconnect() { - if (this.reconnectTimeoutId !== null || this.authFailed || this.forcedClose || !(this.accessToken ?? accessToken.current)) { - this.readyState = SSE_CLOSED; - return; - } - - this.reconnectAttempts++; - const delay = this.getReconnectDelay(this.reconnectAttempts); - - this.readyState = SSE_CONNECTING; - this.onConnecting(true); - this.onClose(); - - clearTimeout(this.reconnectTimeoutId!); - this.reconnectTimeoutId = setTimeout(() => { - this.reconnectTimeoutId = null; - this.connect(); - }, delay); - } } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts index 3da2899d70..b3e22054e4 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts @@ -18,68 +18,71 @@ vi.mock('$shared/document-visibility.svelte', () => { }; }); -// Helper to create a mock fetch response that streams SSE data -function createSseResponse(events: string[] = [], options: { status?: number; delay?: number } = {}) { - const { status = 200, delay = 0 } = options; +function createClient(options?: SseClientOptions): SseClient { + return new SseClient('/api/v2/push', { + baseUrl: 'http://localhost:5200', + reconnectDelay: () => 50, + ...options + }); +} +// Creates a response whose stream stays open indefinitely (for testing open connections) +function createOpenSseResponse(initialEvents: string[] = []) { return new Response( new ReadableStream({ - async start(controller) { - for (const event of events) { - if (delay > 0) { - await new Promise((resolve) => setTimeout(resolve, delay)); - } + start(controller) { + for (const event of initialEvents) { controller.enqueue(new TextEncoder().encode(event)); } - controller.close(); + // intentionally never close } }), { - status, - headers: { 'Content-Type': 'text/event-stream' } + headers: { 'Content-Type': 'text/event-stream' }, + status: 200 } ); } -// Creates a response whose stream stays open indefinitely (for testing open connections) -function createOpenSseResponse(initialEvents: string[] = []) { +// Helper to create a mock fetch response that streams SSE data +function createSseResponse(events: string[] = [], options: { delay?: number; status?: number } = {}) { + const { delay = 0, status = 200 } = options; + return new Response( new ReadableStream({ - start(controller) { - for (const event of initialEvents) { + async start(controller) { + for (const event of events) { + if (delay > 0) { + await new Promise((resolve) => setTimeout(resolve, delay)); + } + controller.enqueue(new TextEncoder().encode(event)); } - // intentionally never close + + controller.close(); } }), { - status: 200, - headers: { 'Content-Type': 'text/event-stream' } + headers: { 'Content-Type': 'text/event-stream' }, + status } ); } -function createClient(options?: SseClientOptions): SseClient { - return new SseClient('/api/v2/push', { - baseUrl: 'http://localhost:5200', - reconnectDelay: () => 50, - ...options - }); -} - describe('SseClient', () => { - let fetchMock: ReturnType; + let fetchMock: ReturnType>; let activeClients: SseClient[] = []; beforeEach(() => { - fetchMock = vi.fn(); - global.fetch = fetchMock; + fetchMock = vi.fn(); + global.fetch = fetchMock as typeof fetch; }); afterEach(() => { for (const client of activeClients) { client.close(); } + activeClients = []; vi.restoreAllMocks(); }); @@ -152,7 +155,7 @@ describe('SseClient', () => { // Never close - simulate long-lived connection } }), - { status: 200, headers: { 'Content-Type': 'text/event-stream' } } + { headers: { 'Content-Type': 'text/event-stream' }, status: 200 } ) ); @@ -182,7 +185,7 @@ describe('SseClient', () => { // intentionally never close - stream stays open } }), - { status: 200, headers: { 'Content-Type': 'text/event-stream' } } + { headers: { 'Content-Type': 'text/event-stream' }, status: 200 } ) ); diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte index 3c21033a9d..31e7ded1d9 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte @@ -28,7 +28,7 @@ import { getMeQuery, invalidateUserQueries } from '$features/users/api.svelte'; import { getGravatarFromCurrentUser } from '$features/users/gravatar.svelte'; import { invalidateWebhookQueries } from '$features/webhooks/api.svelte'; - import { isEntityChangedType, type WebSocketMessageType } from '$features/websockets/models'; + import { type EntityChanged, isEntityChangedType, type UserMembershipChanged, type WebSocketMessageType } from '$features/websockets/models'; import { SseClient } from '$features/websockets/sse-client.svelte'; import { useMiddleware } from '@exceptionless/fetchclient'; import { useQueryClient } from '@tanstack/svelte-query'; @@ -156,10 +156,24 @@ // When a user is added or removed from an organization, invalidate org/project caches // so the UI reflects the membership change without a manual reload. if (data.type === 'UserMembershipChanged') { - const msg = data.message as { organization_id?: string }; - if (msg?.organization_id) { - await invalidateOrganizationQueries(queryClient, msg); - await invalidateProjectQueries(queryClient, msg); + const membershipMessage = data.message as UserMembershipChanged; + if (membershipMessage.organization_id) { + const organizationChangedMessage: EntityChanged = { + change_type: membershipMessage.change_type, + data: {}, + id: membershipMessage.organization_id, + organization_id: membershipMessage.organization_id, + type: 'Organization' + }; + const projectChangedMessage: EntityChanged = { + change_type: membershipMessage.change_type, + data: {}, + organization_id: membershipMessage.organization_id, + type: 'Project' + }; + + await invalidateOrganizationQueries(queryClient, organizationChangedMessage); + await invalidateProjectQueries(queryClient, projectChangedMessage); } } } diff --git a/src/Exceptionless.Web/Hubs/SseConnection.cs b/src/Exceptionless.Web/Hubs/SseConnection.cs index 84bc925df6..ba3e5c9598 100644 --- a/src/Exceptionless.Web/Hubs/SseConnection.cs +++ b/src/Exceptionless.Web/Hubs/SseConnection.cs @@ -96,7 +96,11 @@ public bool TryWriteKeepAlive() public void Abort() { try { _cts.Cancel(); } - catch (ObjectDisposedException) { } + catch (ObjectDisposedException ex) + { + _logger.LogDebug(ex, "SSE cancellation token source was already disposed for {ConnectionId}", ConnectionId); + } + _queue.Complete(); } @@ -104,17 +108,16 @@ public async ValueTask DisposeAsync() { if (Interlocked.Exchange(ref _disposeState, 1) != 0) return; - Abort(); - try - { - await _writeLoop.ConfigureAwait(false); - } - catch (OperationCanceledException) { } - finally + Abort(); + using (_queue) + using (_cts) { - _queue.Dispose(); - _cts.Dispose(); + try + { + await _writeLoop.ConfigureAwait(false); + } + catch (OperationCanceledException) { } } } @@ -128,15 +131,9 @@ private async Task WriteLoopAsync(CancellationToken ct) if (evt is null) break; // Queue completed - byte[] bytes; - if (evt.Value.IsKeepAlive) - { - bytes = KeepAliveBytes; - } - else - { - bytes = System.Text.Encoding.UTF8.GetBytes($"data: {evt.Value.Data}\n\n"); - } + var bytes = evt.Value.IsKeepAlive + ? KeepAliveBytes + : System.Text.Encoding.UTF8.GetBytes($"data: {evt.Value.Data}\n\n"); await _response.Body.WriteAsync(bytes, ct); await _response.Body.FlushAsync(ct); @@ -145,10 +142,6 @@ private async Task WriteLoopAsync(CancellationToken ct) catch (OperationCanceledException) { } catch (ObjectDisposedException) { } catch (IOException) { } - catch (Exception ex) - { - _logger.LogDebug(ex, "SSE write loop ended for connection {ConnectionId}", ConnectionId); - } finally { // Always signal ConnectionAborted so the middleware's Task.Delay unblocks @@ -156,8 +149,14 @@ private async Task WriteLoopAsync(CancellationToken ct) _queue.Complete(); if (!_cts.IsCancellationRequested) { - try { _cts.Cancel(); } - catch (ObjectDisposedException) { } + try + { + _cts.Cancel(); + } + catch (ObjectDisposedException ex) + { + _logger.LogDebug(ex, "SSE cancellation token source was already disposed for {ConnectionId}", ConnectionId); + } } } } @@ -221,7 +220,7 @@ public EnqueueResult TryEnqueue(SseEvent evt) // notifications do not get crowded out by stale cache invalidations. if (_list.Count >= _capacity) { - var queuedToDrop = !evt.CanDrop ? FindFirstDroppableNode() : null; + var queuedToDrop = FindFirstDroppableNode(); RemoveNode(queuedToDrop ?? _list.First!); result = EnqueueResult.DroppedQueuedMessage; } diff --git a/src/Exceptionless.Web/Hubs/SseConnectionManager.cs b/src/Exceptionless.Web/Hubs/SseConnectionManager.cs index 063f503ce9..1c3dc83e5a 100644 --- a/src/Exceptionless.Web/Hubs/SseConnectionManager.cs +++ b/src/Exceptionless.Web/Hubs/SseConnectionManager.cs @@ -195,7 +195,9 @@ private async Task ObserveDisposeAsync(string connectionId, Task disposeTask) { await disposeTask.ConfigureAwait(false); } - catch (Exception ex) + catch (OperationCanceledException) { } + catch (ObjectDisposedException) { } + catch (InvalidOperationException ex) { _logger.LogDebug(ex, "SSE connection cleanup failed for {ConnectionId}", connectionId); } diff --git a/src/Exceptionless.Web/Hubs/SseMiddleware.cs b/src/Exceptionless.Web/Hubs/SseMiddleware.cs index d50e679119..a6afbf0472 100644 --- a/src/Exceptionless.Web/Hubs/SseMiddleware.cs +++ b/src/Exceptionless.Web/Hubs/SseMiddleware.cs @@ -60,51 +60,45 @@ public async Task Invoke(HttpContext context) context.Response.Headers.ContentType = "text/event-stream"; context.Response.Headers.CacheControl = "no-cache, no-store"; context.Response.Headers["X-Accel-Buffering"] = "no"; // nginx - context.Response.Headers.Connection = "keep-alive"; // Disable response buffering var bufferingFeature = context.Features.Get(); bufferingFeature?.DisableBuffering(); string connectionId = Guid.NewGuid().ToString("N"); - var connection = _connectionManager.AddConnection(connectionId, context.Response, context.RequestAborted); - - await OnConnected(context, connectionId); + SseConnection? connection = null; try { + connection = _connectionManager.AddConnection(connectionId, context.Response, context.RequestAborted); + await OnConnected(context, connectionId).ConfigureAwait(false); + // Send initial connected event connection.TryWrite(new { type = "Connected", message = new { connection_id = connectionId } }); // Hold the response open until the client disconnects or the connection is aborted - await Task.Delay(Timeout.Infinite, connection.ConnectionAborted); + await Task.Delay(Timeout.Infinite, connection.ConnectionAborted).ConfigureAwait(false); } catch (OperationCanceledException) { } finally { - await OnDisconnected(context, connectionId); - await _connectionManager.RemoveConnectionAsync(connectionId); + if (connection is not null) + { + await OnDisconnected(context, connectionId).ConfigureAwait(false); + await _connectionManager.RemoveConnectionAsync(connectionId).ConfigureAwait(false); + } } } private async Task OnConnected(HttpContext context, string connectionId) { _logger.LogTrace("SSE connected {ConnectionId}", connectionId); + foreach (string organizationId in context.User.GetOrganizationIds()) + await _connectionMapping.GroupAddAsync(organizationId, connectionId).ConfigureAwait(false); - try - { - foreach (string organizationId in context.User.GetOrganizationIds()) - await _connectionMapping.GroupAddAsync(organizationId, connectionId); - - string? userId = context.User.GetUserId(); - if (!String.IsNullOrEmpty(userId)) - await _connectionMapping.UserIdAddAsync(userId, connectionId); - } - catch (Exception ex) - { - _logger.LogError(ex, "SSE OnConnected Error: {Message}", ex.Message); - throw; - } + string? userId = context.User.GetUserId(); + if (!String.IsNullOrEmpty(userId)) + await _connectionMapping.UserIdAddAsync(userId, connectionId).ConfigureAwait(false); } private async Task OnDisconnected(HttpContext context, string connectionId) @@ -114,15 +108,19 @@ private async Task OnDisconnected(HttpContext context, string connectionId) try { foreach (string organizationId in context.User.GetOrganizationIds()) - await _connectionMapping.GroupRemoveAsync(organizationId, connectionId); + await _connectionMapping.GroupRemoveAsync(organizationId, connectionId).ConfigureAwait(false); string? userId = context.User.GetUserId(); if (!String.IsNullOrEmpty(userId)) - await _connectionMapping.UserIdRemoveAsync(userId, connectionId); + await _connectionMapping.UserIdRemoveAsync(userId, connectionId).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _logger.LogDebug(ex, "SSE disconnect was canceled for {ConnectionId}", connectionId); } - catch (Exception ex) + catch (ObjectDisposedException ex) { - _logger.LogError(ex, "SSE OnDisconnected Error: {Message}", ex.Message); + _logger.LogDebug(ex, "SSE disconnect raced with disposal for {ConnectionId}", connectionId); } } } diff --git a/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs b/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs index 4a57f3de0c..57456956f5 100644 --- a/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs +++ b/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs @@ -88,7 +88,7 @@ public async Task RemoveConnectionAsync(string connectionId) if (!_connections.TryRemove(connectionId, out var socket)) return; - if (!CanSend(socket)) + if (!CanClose(socket)) { AppDiagnostics.PushWebSocketConnectionsClosed.Add(1); AppDiagnostics.Gauge("push.connections.websocket.active", _connections.Count); @@ -189,7 +189,7 @@ private async Task SendKeepAliveAsync(string connectionId, WebSocket socket) { await RemoveConnectionAsync(connectionId).ConfigureAwait(false); } - catch (Exception ex) + catch (WebSocketException ex) { _logger.LogDebug(ex, "Error sending websocket keepalive for {ConnectionId}", connectionId); } @@ -211,7 +211,23 @@ private async Task SendMessageAsync(string connectionId, WebSocket socket, objec { await RemoveConnectionAsync(connectionId).ConfigureAwait(false); } - catch (Exception ex) + catch (WebSocketException ex) + { + _logger.LogDebug(ex, "Error sending websocket message for {ConnectionId}", connectionId); + } + catch (InvalidOperationException ex) + { + _logger.LogDebug(ex, "Error sending websocket message for {ConnectionId}", connectionId); + } + catch (OperationCanceledException ex) + { + _logger.LogDebug(ex, "Error sending websocket message for {ConnectionId}", connectionId); + } + catch (NotSupportedException ex) + { + _logger.LogDebug(ex, "Error sending websocket message for {ConnectionId}", connectionId); + } + catch (EncoderFallbackException ex) { _logger.LogDebug(ex, "Error sending websocket message for {ConnectionId}", connectionId); } @@ -221,4 +237,9 @@ private static bool CanSend(WebSocket socket) { return socket.State is WebSocketState.Open; } + + private static bool CanClose(WebSocket socket) + { + return socket.State is WebSocketState.Open or WebSocketState.CloseReceived; + } } diff --git a/src/Exceptionless.Web/Hubs/WebSocketPushMiddleware.cs b/src/Exceptionless.Web/Hubs/WebSocketPushMiddleware.cs index b5f3bceacb..024a200c74 100644 --- a/src/Exceptionless.Web/Hubs/WebSocketPushMiddleware.cs +++ b/src/Exceptionless.Web/Hubs/WebSocketPushMiddleware.cs @@ -56,10 +56,10 @@ public async Task Invoke(HttpContext context) using var socket = await context.WebSockets.AcceptWebSocketAsync(); string connectionId = _connectionManager.AddConnection(socket); - await OnConnected(context, connectionId).ConfigureAwait(false); try { + await OnConnected(context, connectionId).ConfigureAwait(false); await ReceiveUntilCloseAsync(socket, context.RequestAborted).ConfigureAwait(false); } catch (WebSocketException ex) when (ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely) { } diff --git a/tests/Exceptionless.Tests/AppWebHostFactory.cs b/tests/Exceptionless.Tests/AppWebHostFactory.cs index 1c90a93d37..50210a8863 100644 --- a/tests/Exceptionless.Tests/AppWebHostFactory.cs +++ b/tests/Exceptionless.Tests/AppWebHostFactory.cs @@ -14,6 +14,7 @@ namespace Exceptionless.Tests; public class AppWebHostFactory : WebApplicationFactory, IAsyncLifetime { private static readonly string[] s_indexPrefixes = ["events", "migrations", "organizations", "projects", "saved-views", "stacks", "tokens", "users", "webhooks"]; + private static readonly string s_runScope = $"test-{Guid.NewGuid().ToString("N")[..8]}"; private static int s_counter = -1; private static readonly ConcurrentQueue s_pool = new(); private static readonly Lazy> s_sharedApplication = new(StartSharedApplicationAsync, LazyThreadSafetyMode.ExecutionAndPublication); @@ -25,7 +26,7 @@ public AppWebHostFactory() instanceId = Interlocked.Increment(ref s_counter); InstanceId = instanceId; - AppScope = instanceId == 0 ? "test" : $"test-{instanceId}"; + AppScope = instanceId == 0 ? s_runScope : $"{s_runScope}-{instanceId}"; } public string AppScope { get; } @@ -98,9 +99,8 @@ private async Task CleanupElasticsearchSliceAsync(Uri elasticsearchUri) Timeout = TimeSpan.FromSeconds(10) }; - foreach (var prefix in s_indexPrefixes) + foreach (string pattern in s_indexPrefixes.Select(prefix => Uri.EscapeDataString($"{AppScope}-{prefix}*"))) { - string pattern = Uri.EscapeDataString($"{AppScope}-{prefix}*"); using var listResponse = await client.GetAsync($"/_cat/indices/{pattern}?h=index&format=json&expand_wildcards=all"); if (listResponse.StatusCode == HttpStatusCode.NotFound) continue; From 33c2c3fe32edd97f7d5298a9613f024b489dccd6 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 28 May 2026 08:34:33 -0500 Subject: [PATCH 6/7] fix: harden SSE keepalive backpressure - skip keepalive enqueue when the bounded SSE queue is full so critical notifications are never evicted - pause hidden Svelte tabs without scheduling reconnect churn - add queue regression coverage for critical-message preservation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../lib/features/websockets/sse-client.svelte.ts | 11 ++++++++--- src/Exceptionless.Web/Hubs/SseConnection.cs | 7 +++++-- tests/Exceptionless.Tests/Hubs/SseTests.cs | 16 ++++++++++++++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts index 11adf81186..f8d1de2dfc 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts @@ -56,6 +56,7 @@ export class SseClient { private connectionTimeoutId: null | ReturnType = null; private forcedClose: boolean = false; private hasConnectedBefore: boolean = false; + private pausedForVisibility: boolean = false; private reconnectAttempts: number = 0; private reconnectTimeoutId: null | ReturnType = null; @@ -76,9 +77,13 @@ export class SseClient { this.accessToken = accessToken.current; this.reconnectAttempts = 0; this.authFailed = false; + this.pausedForVisibility = false; this.close(false); } else if (!visibility.visible) { + this.pausedForVisibility = true; this.close(false); + } else { + this.pausedForVisibility = false; } if ( @@ -157,7 +162,7 @@ export class SseClient { } private scheduleReconnect() { - if (this.reconnectTimeoutId !== null || this.authFailed || this.forcedClose || !(this.accessToken ?? accessToken.current)) { + if (this.reconnectTimeoutId !== null || this.authFailed || this.forcedClose || this.pausedForVisibility || !(this.accessToken ?? accessToken.current)) { this.readyState = SSE_CLOSED; return; } @@ -283,7 +288,7 @@ export class SseClient { return; } - if (signal.aborted && this.forcedClose) { + if (signal.aborted && (this.forcedClose || this.pausedForVisibility)) { // Intentional close - don't reconnect this.readyState = SSE_CLOSED; this.onClose(); @@ -301,7 +306,7 @@ export class SseClient { } // Stream ended (server closed connection) - reconnect - if (generation === this.streamGeneration && !this.forcedClose) { + if (generation === this.streamGeneration && !this.forcedClose && !this.pausedForVisibility) { this.scheduleReconnect(); } } diff --git a/src/Exceptionless.Web/Hubs/SseConnection.cs b/src/Exceptionless.Web/Hubs/SseConnection.cs index ba3e5c9598..6d4eb05cb0 100644 --- a/src/Exceptionless.Web/Hubs/SseConnection.cs +++ b/src/Exceptionless.Web/Hubs/SseConnection.cs @@ -109,7 +109,6 @@ public async ValueTask DisposeAsync() if (Interlocked.Exchange(ref _disposeState, 1) != 0) return; Abort(); - Abort(); using (_queue) using (_cts) { @@ -181,7 +180,8 @@ internal enum EnqueueResult { Enqueued, Deduped, - DroppedQueuedMessage + DroppedQueuedMessage, + Skipped } /// @@ -220,6 +220,9 @@ public EnqueueResult TryEnqueue(SseEvent evt) // notifications do not get crowded out by stale cache invalidations. if (_list.Count >= _capacity) { + if (evt.IsKeepAlive) + return EnqueueResult.Skipped; + var queuedToDrop = FindFirstDroppableNode(); RemoveNode(queuedToDrop ?? _list.First!); result = EnqueueResult.DroppedQueuedMessage; diff --git a/tests/Exceptionless.Tests/Hubs/SseTests.cs b/tests/Exceptionless.Tests/Hubs/SseTests.cs index 40960def89..8c190ae712 100644 --- a/tests/Exceptionless.Tests/Hubs/SseTests.cs +++ b/tests/Exceptionless.Tests/Hubs/SseTests.cs @@ -474,4 +474,20 @@ public async Task CriticalMessage_WhenQueueFull_DropsOldestDroppableMessageFirst Assert.Equal("critical-1", item1!.Value.Data); Assert.Equal("critical-2", item2!.Value.Data); } + + [Fact] + public async Task KeepAlive_WhenQueueFull_DoesNotEvictCriticalMessage() + { + var queue = new SseConnection.DedupQueue(1); + queue.TryEnqueue(new SseConnection.SseEvent { Data = "critical-1", CanDrop = false }); + + var result = queue.TryEnqueue(SseConnection.SseEvent.KeepAlive); + + using var cts = new CancellationTokenSource(); + var item = await queue.DequeueAsync(cts.Token); + + Assert.Equal(SseConnection.EnqueueResult.Skipped, result); + Assert.Equal("critical-1", item!.Value.Data); + Assert.False(item.Value.IsKeepAlive); + } } From 0675ae8ea9ab9012f3757f43fd4d4568859e54ae Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 28 May 2026 08:51:48 -0500 Subject: [PATCH 7/7] fix: stop retrying when push is disabled - treat a missing /api/v2/push endpoint as unavailable instead of reconnectable - reset the unavailable state when the auth token changes - cover the rollout-off path with a no-reconnect client test Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../features/websockets/sse-client.svelte.ts | 20 ++++++++++++++++++- .../features/websockets/sse-client.test.ts | 15 ++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts index f8d1de2dfc..eb054e630e 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.svelte.ts @@ -54,6 +54,7 @@ export class SseClient { private accessToken: null | string = null; private authFailed: boolean = false; private connectionTimeoutId: null | ReturnType = null; + private endpointUnavailable: boolean = false; private forcedClose: boolean = false; private hasConnectedBefore: boolean = false; private pausedForVisibility: boolean = false; @@ -77,6 +78,7 @@ export class SseClient { this.accessToken = accessToken.current; this.reconnectAttempts = 0; this.authFailed = false; + this.endpointUnavailable = false; this.pausedForVisibility = false; this.close(false); } else if (!visibility.visible) { @@ -92,6 +94,7 @@ export class SseClient { this.readyState === SSE_CLOSED && this.reconnectTimeoutId === null && !this.authFailed && + !this.endpointUnavailable && !this.forcedClose ) { this.connect(); @@ -162,7 +165,14 @@ export class SseClient { } private scheduleReconnect() { - if (this.reconnectTimeoutId !== null || this.authFailed || this.forcedClose || this.pausedForVisibility || !(this.accessToken ?? accessToken.current)) { + if ( + this.reconnectTimeoutId !== null || + this.authFailed || + this.endpointUnavailable || + this.forcedClose || + this.pausedForVisibility || + !(this.accessToken ?? accessToken.current) + ) { this.readyState = SSE_CLOSED; return; } @@ -205,6 +215,14 @@ export class SseClient { return; } + if (response.status === 404) { + console.info('[SseClient] Push endpoint unavailable, not reconnecting'); + this.endpointUnavailable = true; + this.readyState = SSE_CLOSED; + this.onClose(); + return; + } + // Rate limited if (response.status === 429) { console.warn('[SseClient] Rate limited, will retry'); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts index b3e22054e4..13ac777930 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/sse-client.test.ts @@ -245,6 +245,21 @@ describe('SseClient', () => { expect(client.readyState).toBe(SSE_CLOSED); expect(fetchMock).toHaveBeenCalledTimes(1); }); + + it('should NOT reconnect when push endpoint is unavailable', async () => { + const onClose = vi.fn(); + fetchMock.mockImplementation(() => Promise.resolve(new Response(null, { status: 404 }))); + + const client = trackedClient(); + client.onClose = onClose; + client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(onClose).toHaveBeenCalledTimes(1); + expect(client.readyState).toBe(SSE_CLOSED); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); }); describe('Reconnection Logic', () => {