From 8ae0091f9c165a407743275634e2a6a472ce5afb Mon Sep 17 00:00:00 2001 From: Marc Diaz Date: Fri, 23 Oct 2026 11:52:27 -0400 Subject: [PATCH] fix(audio): start the macOS system-audio tap after the microphone On an idle Mac the Core Audio tap's aggregate device does not run until another app plays a sound. A picker take started the tap before its SCStream, and while the tap sat stalled ScreenCaptureKit delivered no microphone audio either, so the microphone was silent until the first sound (#1173). A take with a microphone now starts its tap on the first microphone buffer, with a one-second fallback for a microphone that never delivers; a take without one starts it right after the stream. The tap is owned by the sample queue, so its start and stop cannot race. --- .../ScreenCaptureRecorder.swift | 41 ++++++++++++++++--- .../testing/manual-e2e-checklist.md | 2 + 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift index 1450f130d7..80fc7f80f2 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift @@ -165,7 +165,10 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { private let hostClock = CMClockGetHostTimeClock() private let picked: PickedSource? /// A `SystemAudioTap` (macOS 14.2+), held untyped so this class keeps its macOS 13 floor. + /// Sample queue only, like the microphone buffers whose arrival starts it. private var systemAudioTap: AnyObject? + /// Sample queue only. When the tap starts anyway if the microphone has not delivered yet. + private var tapFallbackDeadline: Date? /// Sample queue only. Cleared before the writer is finalised: a tap stops on its own /// queue, and a buffer it had in flight must not reach a finished writer input. private var acceptsTappedAudio = true @@ -255,7 +258,6 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { try stream.addStreamOutput(self, type: microphoneOutputType, sampleHandlerQueue: sampleQueue) } try setupWriter() - startSystemAudioTapIfNeeded() self.stream = stream systemAudioStream = audioStream @@ -264,6 +266,19 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { "schemaVersion": 1, ]) try await stream.startCapture() + // On an idle Mac the tap's device does not run until something plays, and a tap + // started before the stream's microphone held the microphone back with it until then + // (#1173). So a take with a microphone starts its tap on the first microphone buffer; + // the deadline is the fallback for a microphone that never delivers. + if tapsSystemAudio { + sampleQueue.sync { + if nativeMicrophoneEnabled { + tapFallbackDeadline = Date().addingTimeInterval(1) + } else { + startSystemAudioTapIfNeeded() + } + } + } if let audioStream { do { try await audioStream.startCapture() @@ -314,8 +329,19 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { await finishWriter() } + /// Sample queue. A microphone that never delivers must not cost the take its system audio. + private func startSystemAudioTapIfMicrophoneSilent() { + guard let deadline = tapFallbackDeadline, Date() >= deadline else { + return + } + tapFallbackDeadline = nil + startSystemAudioTapIfNeeded() + } + + /// Sample queue. Starts the tap once; a no-op after that, and after the take has stopped. private func startSystemAudioTapIfNeeded() { - guard tapsSystemAudio, #available(macOS 14.2, *) else { + guard tapsSystemAudio, systemAudioTap == nil, acceptsTappedAudio, #available(macOS 14.2, *) + else { return } let tap = SystemAudioTap { [weak self] sampleBuffer in @@ -348,13 +374,14 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { /// prompt left unanswered cannot hold the recording's stop hostage; what the tap still /// delivers is dropped by `acceptsTappedAudio`. private func stopSystemAudioTap() { - sampleQueue.sync { + let tap = sampleQueue.sync { () -> AnyObject? in acceptsTappedAudio = false + defer { systemAudioTap = nil } + return systemAudioTap } - guard let tap = systemAudioTap else { + guard let tap else { return } - systemAudioTap = nil tapControlQueue.async { if #available(macOS 14.2, *), let tap = tap as? SystemAudioTap { tap.stop() @@ -441,6 +468,9 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { } func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) { + if type.rawValue == microphoneOutputTypeRawValue { + startSystemAudioTapIfNeeded() + } guard CMSampleBufferDataIsReady(sampleBuffer) else { return } @@ -964,6 +994,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { return } self.audioMixer?.tick() + self.startSystemAudioTapIfMicrophoneSilent() } audioTicker = timer timer.resume() diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 340df14803..92bc6d3eff 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -881,6 +881,7 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov - [ ] After picking a display, change *Hide desktop icons* in Record mode and confirm the pick is dropped, so the next take asks for a source again. - [ ] With system audio never asked for (not from the permissions window either), turn system audio on and confirm macOS asks for it at once, not at the countdown; allow it and confirm the next take carries system audio. Refused, confirm the permissions window's System audio row offers *Open Settings*, and that turning OpenScreen on under "System Audio Recording Only" there brings system audio back. - [ ] Launch with `OPENSCREEN_MAC_SOURCE_PICKER=legacy` and confirm the app's own selector opens instead, with Screen Recording required. +- [ ] With nothing playing on the Mac (no music, video or call app, so the output device is idle), pick a source, turn on the microphone and system audio, record while talking and play a sound halfway (`afplay /System/Library/Sounds/Ping.aiff`). Confirm the microphone is heard from the first second, not only from the sound on, and that the sound is in the take. Repeat with the microphone off: the sound is still in the take (#1173). #### Microphone on macOS 13 and 14 — v2.0.0 @@ -994,3 +995,4 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-17 | PR #1319 (`2236580`), artifact `openscreen-linux` of build run 39046512213, AppImage run directly. Reference: the `v2.1.0` AppImage from the GitHub release | Ubuntu 24.04.4, GNOME 46 Wayland, AMD Radeon 610M (RADV); `[d3d] … -> backend Hardware`, no CPU fallback | **Partial — fix slice only, no defect** | Same MCP pass and project as the rows above, on the hardware Vulkan backend. `orbit` (12 frames) and `left` (4 s, 6 s): the whole card with four rounded corners, no flat cut. `v2.1.0` reproduces the bug (top and bottom edges flat, corners cut from 1.5 s). `none` and the flat frames at 1.0 and 9.5 s are byte-identical to `v2.1.0` (equal md5). **Not covered:** the editor preview; `right`; a full MP4 export. | | 2026-10-17 | `v2.1.1-rc.1`, `release/v2.1.1` @ `6ec38bf` (= `v2.1.0` + #1319), assets of the GitHub pre-release (build run 39069024444). Windows: `Openscreen.Setup.2.1.1-rc.1-x64.exe`, sha512 matched `latest.yml`, unpacked with 7-Zip and run in place, version 2.1.1. macOS: `Openscreen-macOS-Apple-Silicon-2.1.1-rc.1.dmg`, sha256 matched the asset digest; `spctl` accepted, Notarized Developer ID, on the DMG and the app; ticket stapled to the DMG only, as on 2.1.0-rc.6; `CFBundleShortVersionString` 2.1.1-rc.1. Linux: `Openscreen-Linux-2.1.1-rc.1.AppImage`, sha512 matched `latest-linux.yml` | Windows 11 Home 26200, RTX 4070 Ti; macOS 27.0.1 (26A434), Mac mini M1 8 GB; Ubuntu 24.04.4, GNOME 46 Wayland, Radeon 610M, hardware Vulkan backend | **Partial — patch pass toward stable: no defect on any of the three** | Scoped to what the patch changes (the compositor geometry of a tilted filled card), in MCP over stdio with an isolated `--user-data-dir`, on the project of the PR rows above. On all three: `orbit` (12 frames) and `left` show the whole card with four rounded corners and no flat cut; `none` is a flat zoom in a rounded card (byte-identical to `v2.1.0` on Windows and Linux; on Windows every checked frame is byte-identical to the PR build). `exportVideo` at 1080p30: 10.00 s, 1920×1080 H.264 with an AAC track, decodes without error, and its frame at 5 s shows the tilted card with rounded corners. Before the cut, on a dev build of the PR: the four device frames, dual-frame and vertical-stack, under `orbit`, `left` and `none`, rendered byte-identical to `v2.1.0` (49 of 49 frames). **Not covered:** recording, playback and the editor preview, which the patch does not touch; the rest of the checklist (the `v2.1.0-rc.6` rows). | | 2026-10-19 | `v2.1.2-rc.1`, `release/v2.1.2` @ `c4ef8fd` (= `v2.1.1` + #1344), assets of the GitHub pre-release (build run 39224311283). Windows: `Openscreen.Setup.2.1.2-rc.1-x64.exe` unpacked with 7-Zip and run in place. macOS: `Openscreen-Mac-arm64-2.1.2-rc.1.zip`, run in place. Linux: the AppImage, run directly with `--no-sandbox` | Windows 11 Home 26200; macOS 27.0.1, Mac mini M1; Ubuntu 24.04 GNOME Wayland (Xwayland `:0`) | **Partial: fix slice only, no defect** | `openscreen mcp` over stdio, scripted, with a fresh `--user-data-dir` per run. On all three platforms it answered `initialize`, and `usage-ping.json` held today's day, which is saved only once PostHog accepted the event. With `DO_NOT_TRACK=1` no state file was written. stdout held only JSON-RPC on macOS and Linux. On Windows it starts with one `\r\n` that Electron 41 itself writes during browser-process startup, before `main.js` loads: a bare `app.exit(0)` script prints it too, `ELECTRON_RUN_AS_NODE` does not. It predates this change, and the MCP SDK clients skip it with one parse error. Not covered: everything else in this file, including recording, editing, export, the GUI ping and the tray toggle. These runs sent eight real `surface: mcp` events to PostHog on 19/10. | +| 2026-10-23 | `fix/mac-mic-silent-with-system-audio` on `main` @ `264975d`, dev build (`npm run dev`), capture helper built locally with Xcode 27 and copied from `out/Products/Release` by hand | macOS 26.6.2 (25G83), MacBook Pro M2 Pro; built-in microphone and a USB Audio CODEC (08bb:2902) | **Partial: fix slice only, pass** | #1173, Apple's picker, system audio on, output device idle (after `sudo killall coreaudiod`, or checked idle). Before the fix: the microphone delivered nothing until another app played a sound, 56 s (built-in) and 14 s (USB) into the take; without system audio it started at 0.17 s. With the tap merely started after the stream, four takes (built-in twice, USB twice) recorded the microphone from 0.17–0.18 s, but only 12–30 ms ahead of the tap. With this branch's version, the tap waiting for the first microphone buffer: one USB take recorded the microphone from 0.18 s and the tap started 27 ms after it; one take with the microphone off. In both, a sound played mid-take was in system audio. Measured with temporary per-source buffer counters in the helper, not shipped. **Not covered:** this branch's version with the built-in microphone, a legacy-picker take, macOS 15, an Intel Mac, a microphone that never delivers (the 1 s fallback), pause before the first microphone buffer, a CI-built artifact. |