Add browser heartbeat and auto-restart

Introduce proactive detection and recovery for crashed or unresponsive browsers.

- Add IBrowser.IsHealthy to expose connection+process liveness.
- Implement BrowserHeartbeat to periodically ping Chrome (Browser.getVersion) and call back when the browser becomes unresponsive.
- Update Browser to start/stop the heartbeat for launched processes, handle process exit, fault pending CDP commands, and avoid duplicate disconnect notifications via a flag.
- Enhance BrowserFixture to automatically restart the browser on disconnect, serialize restarts with a semaphore, and retry launches to reduce flaky tests.
- Update BrowserPool to track disconnected browsers, dispose crashed instances, release capacity and proactively warm up replacements toward MinInstances.
- Add necessary using and small helper methods (e.g. process exit handler unregistration).

These changes improve reliability by detecting frozen browsers, failing pending commands fast, and automatically recovering pool/fixture state for subsequent operations.

Author: TorreyBetts

src/Motus.Abstractions/IBrowser.cs

@@ -10,6 +10,12 @@ public interface IBrowser : IAsyncDisposable
     /// </summary>
     bool IsConnected { get; }
 
+    /// <summary>
+    /// Gets whether the browser is connected and its process is alive.
+    /// For browsers connected via ConnectAsync (no process ownership), equivalent to IsConnected.
+    /// </summary>
+    bool IsHealthy => IsConnected;
+
     /// <summary>
     /// Gets all browser contexts.
     /// </summary>

src/Motus.Testing/BrowserFixture.cs

@@ -1,60 +1,137 @@
-using Motus.Abstractions;
-
-namespace Motus.Testing;
-
-/// <summary>
-/// Manages a single browser instance for use in test fixtures.
-/// Call <see cref="InitializeAsync"/> once (typically in assembly/collection setup),
-/// then create isolated contexts per test via <see cref="NewContextAsync"/>.
-/// </summary>
-public sealed class BrowserFixture : IAsyncDisposable
-{
-    private IBrowser? _browser;
-
-    /// <summary>
-    /// Maximum number of launch attempts before giving up.
-    /// Browser startup can fail transiently on CI runners or when antivirus
-    /// delays process creation, so retrying avoids flaky test runs.
-    /// </summary>
-    private const int MaxLaunchAttempts = 3;
-
-    /// <summary>
-    /// Launches a browser instance with the given options, retrying on transient failures.
-    /// </summary>
-    public async Task InitializeAsync(LaunchOptions? options = null)
-    {
-        for (int attempt = 1; attempt <= MaxLaunchAttempts; attempt++)
-        {
-            try
-            {
-                _browser = await MotusLauncher.LaunchAsync(options).ConfigureAwait(false);
-                return;
-            }
-            catch when (attempt < MaxLaunchAttempts)
-            {
-                await Task.Delay(1000 * attempt).ConfigureAwait(false);
-            }
-        }
-    }
-
-    /// <summary>
-    /// The launched browser instance.
-    /// </summary>
-    public IBrowser Browser => _browser ?? throw new InvalidOperationException(
-        "Browser not initialized. Call InitializeAsync first.");
-
-    /// <summary>
-    /// Creates a new isolated browser context.
-    /// </summary>
-    public async Task<IBrowserContext> NewContextAsync(ContextOptions? options = null)
-        => await Browser.NewContextAsync(options);
-
-    public async ValueTask DisposeAsync()
-    {
-        if (_browser is not null)
-        {
-            await _browser.DisposeAsync();
-            _browser = null;
-        }
-    }
-}
+using Motus.Abstractions;
+
+namespace Motus.Testing;
+
+/// <summary>
+/// Manages a single browser instance for use in test fixtures.
+/// Call <see cref="InitializeAsync"/> once (typically in assembly/collection setup),
+/// then create isolated contexts per test via <see cref="NewContextAsync"/>.
+/// If the browser crashes or becomes unresponsive, the fixture automatically
+/// restarts Chrome so subsequent tests proceed against a fresh browser.
+/// </summary>
+public sealed class BrowserFixture : IAsyncDisposable
+{
+    private IBrowser? _browser;
+    private LaunchOptions? _launchOptions;
+    private readonly SemaphoreSlim _restartGate = new(1, 1);
+    private int _disposed;
+
+    /// <summary>
+    /// Maximum number of launch attempts before giving up.
+    /// Browser startup can fail transiently on CI runners or when antivirus
+    /// delays process creation, so retrying avoids flaky test runs.
+    /// </summary>
+    private const int MaxLaunchAttempts = 3;
+
+    /// <summary>
+    /// Launches a browser instance with the given options, retrying on transient failures.
+    /// </summary>
+    public async Task InitializeAsync(LaunchOptions? options = null)
+    {
+        _launchOptions = options;
+        await LaunchWithRetryAsync(options).ConfigureAwait(false);
+        SubscribeDisconnected(_browser!);
+    }
+
+    /// <summary>
+    /// The launched browser instance.
+    /// </summary>
+    public IBrowser Browser => _browser ?? throw new InvalidOperationException(
+        "Browser not initialized. Call InitializeAsync first.");
+
+    /// <summary>
+    /// Creates a new isolated browser context. If a restart is in progress
+    /// (due to a browser crash), callers block until the new browser is ready.
+    /// </summary>
+    public async Task<IBrowserContext> NewContextAsync(ContextOptions? options = null)
+    {
+        await _restartGate.WaitAsync().ConfigureAwait(false);
+        try
+        {
+            return await Browser.NewContextAsync(options).ConfigureAwait(false);
+        }
+        finally
+        {
+            _restartGate.Release();
+        }
+    }
+
+    public async ValueTask DisposeAsync()
+    {
+        if (Interlocked.CompareExchange(ref _disposed, 1, 0) != 0)
+            return;
+
+        await _restartGate.WaitAsync().ConfigureAwait(false);
+        try
+        {
+            if (_browser is not null)
+            {
+                _browser.Disconnected -= OnBrowserDisconnected;
+                await _browser.DisposeAsync().ConfigureAwait(false);
+                _browser = null;
+            }
+        }
+        finally
+        {
+            _restartGate.Release();
+            _restartGate.Dispose();
+        }
+    }
+
+    private void OnBrowserDisconnected(object? sender, EventArgs e)
+    {
+        if (sender is IBrowser old)
+            old.Disconnected -= OnBrowserDisconnected;
+
+        _ = Task.Run(RestartAsync);
+    }
+
+    private async Task RestartAsync()
+    {
+        if (Volatile.Read(ref _disposed) != 0)
+            return;
+
+        await _restartGate.WaitAsync().ConfigureAwait(false);
+        try
+        {
+            if (Volatile.Read(ref _disposed) != 0)
+                return;
+
+            // Dispose the dead browser (best-effort)
+            var old = Interlocked.Exchange(ref _browser, null);
+            if (old is not null)
+            {
+                try { await old.DisposeAsync().ConfigureAwait(false); }
+                catch { /* already dead */ }
+            }
+
+            await LaunchWithRetryAsync(_launchOptions).ConfigureAwait(false);
+
+            if (_browser is not null)
+                SubscribeDisconnected(_browser);
+        }
+        finally
+        {
+            _restartGate.Release();
+        }
+    }
+
+    private async Task LaunchWithRetryAsync(LaunchOptions? options)
+    {
+        for (int attempt = 1; attempt <= MaxLaunchAttempts; attempt++)
+        {
+            try
+            {
+                _browser = await MotusLauncher.LaunchAsync(options).ConfigureAwait(false);
+                return;
+            }
+            catch when (attempt < MaxLaunchAttempts)
+            {
+                await Task.Delay(1000 * attempt).ConfigureAwait(false);
+            }
+        }
+    }
+
+    private void SubscribeDisconnected(IBrowser browser)
+        => browser.Disconnected += OnBrowserDisconnected;
+}

src/Motus/Browser/Browser.cs

@@ -19,6 +19,8 @@ internal sealed class Browser : IBrowser
     private readonly List<BrowserContext> _contexts = [];
 
     private volatile bool _isConnected;
+    private int _disconnectedFlag;
+    private BrowserHeartbeat? _heartbeat;
     private ConsoleCancelEventHandler? _cancelHandler;
     private EventHandler? _processExitHandler;
 
@@ -40,10 +42,18 @@ internal Browser(
         _launchOptions = launchOptions ?? new LaunchOptions();
 
         _transport.Disconnected += OnTransportDisconnected;
+
+        if (_process is not null)
+        {
+            _process.EnableRaisingEvents = true;
+            _process.Exited += OnProcessExited;
+        }
     }
 
     public bool IsConnected => _isConnected;
 
+    public bool IsHealthy => _isConnected && (_process is null || !_process.HasExited);
+
     public string Version { get; private set; } = string.Empty;
 
     public IReadOnlyList<IBrowserContext> Contexts
@@ -68,6 +78,12 @@ internal async Task InitializeAsync(CancellationToken ct)
         _isConnected = true;
 
         RegisterSignalHandlers();
+
+        if (_process is not null)
+        {
+            _heartbeat = new BrowserHeartbeat(_registry.BrowserSession, OnHeartbeatFailed);
+            _heartbeat.Start();
+        }
     }
 
     public async Task CloseAsync()
@@ -77,7 +93,11 @@ public async Task CloseAsync()
 
         _isConnected = false;
 
+        if (_heartbeat is not null)
+            await _heartbeat.DisposeAsync().ConfigureAwait(false);
+
         UnregisterSignalHandlers();
+        UnregisterProcessExitHandler();
 
         // Close all contexts first
         List<BrowserContext> contextsToClose;
@@ -123,7 +143,11 @@ await _registry.BrowserSession.SendAsync(
 
     public async ValueTask DisposeAsync()
     {
+        if (_heartbeat is not null)
+            await _heartbeat.DisposeAsync().ConfigureAwait(false);
+
         UnregisterSignalHandlers();
+        UnregisterProcessExitHandler();
 
         _isConnected = false;
 
@@ -201,10 +225,45 @@ internal void RemoveContext(BrowserContext context)
 
     private void OnTransportDisconnected(Exception? ex)
     {
+        if (Interlocked.CompareExchange(ref _disconnectedFlag, 1, 0) != 0)
+            return;
+
         _isConnected = false;
         Disconnected?.Invoke(this, EventArgs.Empty);
     }
 
+    private void OnProcessExited(object? sender, EventArgs e)
+    {
+        if (Interlocked.CompareExchange(ref _disconnectedFlag, 1, 0) != 0)
+            return;
+
+        _isConnected = false;
+
+        // Dispose transport to fault all pending CDP commands immediately
+        _ = _transport.DisposeAsync().AsTask();
+
+        Disconnected?.Invoke(this, EventArgs.Empty);
+    }
+
+    private void OnHeartbeatFailed(Exception? ex)
+    {
+        if (Interlocked.CompareExchange(ref _disconnectedFlag, 1, 0) != 0)
+            return;
+
+        _isConnected = false;
+
+        // Dispose transport to fault all pending CDP commands (browser is frozen)
+        _ = _transport.DisposeAsync().AsTask();
+
+        Disconnected?.Invoke(this, EventArgs.Empty);
+    }
+
+    private void UnregisterProcessExitHandler()
+    {
+        if (_process is not null)
+            _process.Exited -= OnProcessExited;
+    }
+
     private void RegisterSignalHandlers()
     {
         if (_process is null)