fix(SDK-7061): resilient TRA build-stop — retry + success-gated flag + synchronous shutdown stop

.changeset/pr-99.md

@@ -0,0 +1,5 @@
+---
+"@wdio/browserstack-service": patch
+---
+
+- Fixed WebdriverIO test results sometimes not appearing (builds staying "in progress") when the build-completion signal failed or the test runner was interrupted.

packages/browserstack-service/src/exitHandler.ts

@@ -1,18 +1,155 @@
-import { spawn } from 'node:child_process'
+import { spawn, spawnSync } from 'node:child_process'
 import path from 'node:path'
 import BrowserStackConfig from './config.js'
 import { saveFunnelData } from './instrumentation/funnelInstrumentation.js'
 import { fileURLToPath } from 'node:url'
-import { BROWSERSTACK_TESTHUB_JWT } from './constants.js'
+import { BROWSERSTACK_TESTHUB_JWT, BROWSERSTACK_TESTHUB_UUID } from './constants.js'
 import PerformanceTester from './instrumentation/performance/performance-tester.js'
 import TestOpsConfig from './testOps/testOpsConfig.js'
 import { BStackLogger } from './bstackLogger.js'
 import { BrowserstackCLI } from './cli/index.js'
+import APIUtils from './cli/apiUtils.js'
 
 const __filename = fileURLToPath(import.meta.url)
 const __dirname = path.dirname(__filename)
 
+// SDK-7061: a termination signal (SIGTERM / SIGINT / …) delivered mid-run kills the WDIO
+// launcher before onComplete or the 'exit' hook can send the TRA build-stop, so the build
+// hangs 'running' until the ~60-min server-side inactivity timeout. An async stop from a
+// signal handler does NOT help: WDIO's launcher calls process.exit ~70ms into onComplete,
+// the event loop stops, and the in-flight async PUT never lands. Java avoids this because
+// Runtime.getRuntime().addShutdownHook runs synchronously and the JVM blocks exit until the
+// hook returns. Node has no native sync HTTP, so replicate that blocking semantics with a
+// spawnSync child: the parent process cannot exit until the stop PUT has completed.
+// Guarded so we never double-send with onComplete or the 'exit' cleanup path.
+const STOP_ENDPOINT_ENV = '__BROWSERSTACK_STOP_ENDPOINT'
+
+// Inline script run by the blocking spawnSync child. Node `-e` defaults to CommonJS, so
+// require() is available. Reads UUID/JWT/endpoint from the inherited env, PUTs the build
+// stop, treats non-2xx as failure, retries up to 3x with backoff, exits 0 only on success.
+const SYNC_STOP_SCRIPT = `
+const uuid = process.env['${BROWSERSTACK_TESTHUB_UUID}'];
+const jwt = process.env['${BROWSERSTACK_TESTHUB_JWT}'];
+const endpoint = process.env['${STOP_ENDPOINT_ENV}'];
+if (!uuid || !jwt || !endpoint) { process.exit(0); }
+const target = endpoint + '/api/v1/builds/' + uuid + '/stop';
+const body = JSON.stringify({ stop_time: new Date().toISOString() });
+const lib = target.indexOf('https') === 0 ? require('https') : require('http');
+function attempt(n) {
+  const req = lib.request(target, {
+    method: 'PUT',
+    timeout: 4000,
+    headers: {
+      'Content-Type': 'application/json',
+      'X-BSTACK-OBS': 'true',
+      'Authorization': 'Bearer ' + jwt,
+      'Content-Length': Buffer.byteLength(body)
+    }
+  }, function (res) {
+    let d = '';
+    res.on('data', function (c) { d += c; });
+    res.on('end', function () {
+      if (res.statusCode >= 200 && res.statusCode < 300) {
+        process.stdout.write('[STOP_BUILD] sync stop success ' + res.statusCode + '\\n');
+        process.exit(0);
+      } else {
+        retry(n, 'HTTP ' + res.statusCode);
+      }
+    });
+  });
+  req.on('error', function (e) { retry(n, e && e.message); });
+  req.on('timeout', function () { req.destroy(new Error('timeout')); });
+  req.write(body);
+  req.end();
+}
+function retry(n, msg) {
+  process.stdout.write('[STOP_BUILD] sync attempt ' + n + ' failed: ' + msg + '\\n');
+  if (n < 3) { setTimeout(function () { attempt(n + 1); }, 500 * n); }
+  else { process.exit(1); }
+}
+attempt(1);
+`
+
+let signalHandlersRegistered = false
+let buildStopSentOnSignal = false
+
+// SDK-7061: synchronous, blocking build-stop mirroring java's addShutdownHook. The parent
+// BLOCKS on spawnSync until the child's stop PUT completes, guaranteeing it lands before
+// process.exit. Only the direct-HTTP (non-CLI) observability path with a created build is
+// handled here — the CLI manages its own build lifecycle. Best-effort, never throws.
+function stopBuildSyncBlocking(reason: string): void {
+    // SDK-7061: setupExitHandlers() (which arms this handler) runs in the launcher
+    // constructor BEFORE BrowserStackConfig is initialized, so getInstance() can return
+    // undefined during an early uncaughtException. Guard so this never throws — a throw
+    // from inside the uncaughtException listener would be fatal and mask the original error.
+    const config = BrowserStackConfig.getInstance()
+    if (!config) {
+        return
+    }
+    if (buildStopSentOnSignal || config.testObservability?.buildStopped) {
+        return
+    }
+    if (!process.env[BROWSERSTACK_TESTHUB_UUID] || !process.env[BROWSERSTACK_TESTHUB_JWT]) {
+        return
+    }
+    if (BrowserstackCLI.getInstance().isRunning()) {
+        return
+    }
+    buildStopSentOnSignal = true
+    try {
+        BStackLogger.debug(`Shutdown (${reason}) — stopping TestHub build synchronously (blocking)`)
+        const result = spawnSync(process.execPath, ['-e', SYNC_STOP_SCRIPT], {
+            timeout: 15000,
+            encoding: 'utf-8',
+            env: { ...process.env, [STOP_ENDPOINT_ENV]: APIUtils.DATA_ENDPOINT }
+        })
+        if (result.stdout) {
+            BStackLogger.debug(`[STOP_BUILD] sync child stdout: ${String(result.stdout).trim()}`)
+        }
+        if (result.stderr) {
+            BStackLogger.debug(`[STOP_BUILD] sync child stderr: ${String(result.stderr).trim()}`)
+        }
+        if (result.status === 0) {
+            config.testObservability.buildStopped = true
+            BStackLogger.debug('[STOP_BUILD] sync stop completed successfully')
+        } else {
+            BStackLogger.debug(`[STOP_BUILD] sync stop did not confirm success (status=${result.status}, signal=${result.signal})`)
+        }
+    } catch (err) {
+        BStackLogger.debug(`Error stopping build synchronously on ${reason}: ${err}`)
+    }
+}
+
+function registerSignalHandlers(): void {
+    if (signalHandlersRegistered) {
+        return
+    }
+    signalHandlersRegistered = true
+
+    const onSignal = (signal: NodeJS.Signals) => {
+        stopBuildSyncBlocking(signal)
+        process.exit(0)
+    }
+
+    process.on('SIGTERM', onSignal)
+    process.on('SIGINT', onSignal)
+    process.on('SIGHUP', onSignal)
+    if (process.platform === 'win32') {
+        process.on('SIGBREAK', onSignal)
+    } else {
+        process.on('SIGABRT', onSignal)
+        process.on('SIGQUIT', onSignal)
+    }
+
+    process.on('uncaughtException', (err) => {
+        BStackLogger.debug(`Uncaught exception — stopping TestHub build: ${err?.stack || err}`)
+        stopBuildSyncBlocking('uncaughtException')
+        process.exit(1)
+    })
+}
+
 export function setupExitHandlers() {
+    registerSignalHandlers()
     const handleCLICleanup = () => {
         BStackLogger.debug('Handling CLI cleanup in exit handler')
         try {
@@ -45,6 +182,10 @@ export function setupExitHandlers() {
         }
     }
     process.on('exit', () => {
+        // 'exit' can only do sync work; the blocking spawnSync stop fits. This is the
+        // guaranteed catch-all — even when WDIO's own process.exit pre-empts our signal
+        // handler, this still runs synchronously before the process dies (SDK-7061).
+        stopBuildSyncBlocking('exit')
         const isCLIEnabled = BrowserstackCLI.getInstance().isRunning()
         handleCLICleanup()
         const args = shouldCallCleanup(BrowserStackConfig.getInstance(), isCLIEnabled)

packages/browserstack-service/src/launcher.ts

@@ -629,8 +629,12 @@ export default class BrowserstackLauncherService implements Services.ServiceInst
             // SDK-4671: before stopping the build, synthesize TestRunFinished for any
             // test runs whose worker died mid-test, else they stay 'in progress' on TRA.
             await finalizeOrphanedRuns()
+            // SDK-7061: capture the stop result so we only mark the build stopped when it
+            // actually succeeded. A failed stop must leave buildStopped=false so the
+            // process-exit cleanup re-stop path can still close the build.
+            let stopResult: { status?: string } | undefined
             try {
-                await (isCLIEnabled ? BrowserstackCLI.getInstance().stop() : stopBuildUpstream())
+                stopResult = await (isCLIEnabled ? BrowserstackCLI.getInstance().stop() : stopBuildUpstream()) as { status?: string } | undefined
                 PerformanceTester.end(PERFORMANCE_SDK_EVENTS.FRAMEWORK_EVENTS.STOP)
             } catch (err) {
                 BStackLogger.error(`Error while stopping CLI ${err}`)
@@ -639,7 +643,11 @@ export default class BrowserstackLauncherService implements Services.ServiceInst
             if (process.env[BROWSERSTACK_OBSERVABILITY] && process.env[BROWSERSTACK_TESTHUB_UUID]) {
                 console.log(`\nVisit https://automation.browserstack.com/builds/${process.env[BROWSERSTACK_TESTHUB_UUID]} to view build report, insights, and many more debugging information all at one place!\n`)
             }
-            this.browserStackConfig.testObservability.buildStopped = true
+            // CLI path manages its own build lifecycle; for the direct-HTTP path only
+            // mark stopped when stopBuildUpstream returned success (SDK-7061).
+            if (isCLIEnabled || stopResult?.status === 'success') {
+                this.browserStackConfig.testObservability.buildStopped = true
+            }
 
             await PerformanceTester.stopAndGenerate('performance-launcher.html')
             if (process.env[PERF_MEASUREMENT_ENV]) {

packages/browserstack-service/src/util.ts

@@ -736,30 +736,46 @@ export const stopBuildUpstream = PerformanceTester.measureWrapper(PERFORMANCE_SD
         'stop_time': (new Date()).toISOString()
     }
 
-    try {
-        const url = `${APIUtils.DATA_ENDPOINT}/api/v1/builds/${process.env[BROWSERSTACK_TESTHUB_UUID]}/stop`
-        const response = await fetch(url, {
-            method: 'PUT',
-            headers: {
-                ...DEFAULT_REQUEST_CONFIG.headers,
-                'Authorization': `Bearer ${process.env[BROWSERSTACK_TESTHUB_JWT]}`
-            },
-            body: JSON.stringify(data)
-        })
-        BStackLogger.debug(`[STOP_BUILD] Success response: ${await response.text()}`)
-        stopBuildUsage.success()
-        return {
-            status: 'success',
-            message: ''
-        }
-    } catch (error: unknown) {
-        stopBuildUsage.failed(error)
-        BStackLogger.debug(`[STOP_BUILD] Failed. Error: ${error}`)
-        return {
-            status: 'error',
-            message: (error as Error).message
+    const url = `${APIUtils.DATA_ENDPOINT}/api/v1/builds/${process.env[BROWSERSTACK_TESTHUB_UUID]}/stop`
+    // SDK-7061: the build-stop PUT is the signal that closes the TRA build. A single
+    // best-effort request means a transient failure/timeout leaves the build running
+    // until the ~60-min server-side inactivity timeout. Retry with backoff and treat a
+    // non-2xx response as a failure so the build reliably reaches a terminal state.
+    const maxAttempts = 3
+    let lastError: unknown
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        try {
+            const response = await fetch(url, {
+                method: 'PUT',
+                headers: {
+                    ...DEFAULT_REQUEST_CONFIG.headers,
+                    'Authorization': `Bearer ${process.env[BROWSERSTACK_TESTHUB_JWT]}`
+                },
+                body: JSON.stringify(data)
+            })
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status} ${response.statusText}`)
+            }
+            BStackLogger.debug(`[STOP_BUILD] Success response: ${await response.text()}`)
+            stopBuildUsage.success()
+            return {
+                status: 'success',
+                message: ''
+            }
+        } catch (error: unknown) {
+            lastError = error
+            BStackLogger.debug(`[STOP_BUILD] Attempt ${attempt}/${maxAttempts} failed. Error: ${error}`)
+            if (attempt < maxAttempts) {
+                await new Promise((resolve) => setTimeout(resolve, 500 * attempt))
+            }
         }
     }
+    stopBuildUsage.failed(lastError)
+    BStackLogger.debug(`[STOP_BUILD] Failed after ${maxAttempts} attempts. Error: ${lastError}`)
+    return {
+        status: 'error',
+        message: (lastError as Error)?.message ?? 'stop build failed'
+    }
 }))
 
 export function getCiInfo () {