Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,10 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
private let hostClock = CMClockGetHostTimeClock()
private let picked: PickedSource?
/// A `SystemAudioTap` (macOS 14.2+), held untyped so this class keeps its macOS 13 floor.
/// Sample queue only, like the microphone buffers whose arrival starts it.
private var systemAudioTap: AnyObject?
/// Sample queue only. When the tap starts anyway if the microphone has not delivered yet.
private var tapFallbackDeadline: Date?
/// Sample queue only. Cleared before the writer is finalised: a tap stops on its own
/// queue, and a buffer it had in flight must not reach a finished writer input.
private var acceptsTappedAudio = true
Expand Down Expand Up @@ -255,7 +258,6 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
try stream.addStreamOutput(self, type: microphoneOutputType, sampleHandlerQueue: sampleQueue)
}
try setupWriter()
startSystemAudioTapIfNeeded()

self.stream = stream
systemAudioStream = audioStream
Expand All @@ -264,6 +266,19 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
"schemaVersion": 1,
])
try await stream.startCapture()
// On an idle Mac the tap's device does not run until something plays, and a tap
// started before the stream's microphone held the microphone back with it until then
// (#1173). So a take with a microphone starts its tap on the first microphone buffer;
// the deadline is the fallback for a microphone that never delivers.
if tapsSystemAudio {
sampleQueue.sync {
if nativeMicrophoneEnabled {
tapFallbackDeadline = Date().addingTimeInterval(1)
} else {
startSystemAudioTapIfNeeded()
}
}
}
if let audioStream {
do {
try await audioStream.startCapture()
Expand Down Expand Up @@ -314,8 +329,19 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
await finishWriter()
}

/// Sample queue. A microphone that never delivers must not cost the take its system audio.
private func startSystemAudioTapIfMicrophoneSilent() {
guard let deadline = tapFallbackDeadline, Date() >= deadline else {
return
}
tapFallbackDeadline = nil
startSystemAudioTapIfNeeded()
}

/// Sample queue. Starts the tap once; a no-op after that, and after the take has stopped.
private func startSystemAudioTapIfNeeded() {
guard tapsSystemAudio, #available(macOS 14.2, *) else {
guard tapsSystemAudio, systemAudioTap == nil, acceptsTappedAudio, #available(macOS 14.2, *)
else {
return
}
let tap = SystemAudioTap { [weak self] sampleBuffer in
Expand Down Expand Up @@ -348,13 +374,14 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
/// prompt left unanswered cannot hold the recording's stop hostage; what the tap still
/// delivers is dropped by `acceptsTappedAudio`.
private func stopSystemAudioTap() {
sampleQueue.sync {
let tap = sampleQueue.sync { () -> AnyObject? in
acceptsTappedAudio = false
defer { systemAudioTap = nil }
return systemAudioTap
}
guard let tap = systemAudioTap else {
guard let tap else {
return
}
systemAudioTap = nil
tapControlQueue.async {
if #available(macOS 14.2, *), let tap = tap as? SystemAudioTap {
tap.stop()
Expand Down Expand Up @@ -441,6 +468,9 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
}

func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
if type.rawValue == microphoneOutputTypeRawValue {
startSystemAudioTapIfNeeded()
}
guard CMSampleBufferDataIsReady(sampleBuffer) else {
return
}
Expand Down Expand Up @@ -964,6 +994,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
return
}
self.audioMixer?.tick()
self.startSystemAudioTapIfMicrophoneSilent()
}
audioTicker = timer
timer.resume()
Expand Down
2 changes: 2 additions & 0 deletions technical-documentation/testing/manual-e2e-checklist.md
Original file line number Diff line number Diff line change
Expand Up @@ -881,6 +881,7 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov
- [ ] After picking a display, change *Hide desktop icons* in Record mode and confirm the pick is dropped, so the next take asks for a source again.
- [ ] With system audio never asked for (not from the permissions window either), turn system audio on and confirm macOS asks for it at once, not at the countdown; allow it and confirm the next take carries system audio. Refused, confirm the permissions window's System audio row offers *Open Settings*, and that turning OpenScreen on under "System Audio Recording Only" there brings system audio back.
- [ ] Launch with `OPENSCREEN_MAC_SOURCE_PICKER=legacy` and confirm the app's own selector opens instead, with Screen Recording required.
- [ ] With nothing playing on the Mac (no music, video or call app, so the output device is idle), pick a source, turn on the microphone and system audio, record while talking and play a sound halfway (`afplay /System/Library/Sounds/Ping.aiff`). Confirm the microphone is heard from the first second, not only from the sound on, and that the sound is in the take. Repeat with the microphone off: the sound is still in the take (#1173).

#### Microphone on macOS 13 and 14 — v2.0.0

Expand Down Expand Up @@ -994,3 +995,4 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov
| 2026-10-17 | PR #1319 (`2236580`), artifact `openscreen-linux` of build run 39046512213, AppImage run directly. Reference: the `v2.1.0` AppImage from the GitHub release | Ubuntu 24.04.4, GNOME 46 Wayland, AMD Radeon 610M (RADV); `[d3d] … -> backend Hardware`, no CPU fallback | **Partial — fix slice only, no defect** | Same MCP pass and project as the rows above, on the hardware Vulkan backend. `orbit` (12 frames) and `left` (4 s, 6 s): the whole card with four rounded corners, no flat cut. `v2.1.0` reproduces the bug (top and bottom edges flat, corners cut from 1.5 s). `none` and the flat frames at 1.0 and 9.5 s are byte-identical to `v2.1.0` (equal md5). **Not covered:** the editor preview; `right`; a full MP4 export. |
| 2026-10-17 | `v2.1.1-rc.1`, `release/v2.1.1` @ `6ec38bf` (= `v2.1.0` + #1319), assets of the GitHub pre-release (build run 39069024444). Windows: `Openscreen.Setup.2.1.1-rc.1-x64.exe`, sha512 matched `latest.yml`, unpacked with 7-Zip and run in place, version 2.1.1. macOS: `Openscreen-macOS-Apple-Silicon-2.1.1-rc.1.dmg`, sha256 matched the asset digest; `spctl` accepted, Notarized Developer ID, on the DMG and the app; ticket stapled to the DMG only, as on 2.1.0-rc.6; `CFBundleShortVersionString` 2.1.1-rc.1. Linux: `Openscreen-Linux-2.1.1-rc.1.AppImage`, sha512 matched `latest-linux.yml` | Windows 11 Home 26200, RTX 4070 Ti; macOS 27.0.1 (26A434), Mac mini M1 8 GB; Ubuntu 24.04.4, GNOME 46 Wayland, Radeon 610M, hardware Vulkan backend | **Partial — patch pass toward stable: no defect on any of the three** | Scoped to what the patch changes (the compositor geometry of a tilted filled card), in MCP over stdio with an isolated `--user-data-dir`, on the project of the PR rows above. On all three: `orbit` (12 frames) and `left` show the whole card with four rounded corners and no flat cut; `none` is a flat zoom in a rounded card (byte-identical to `v2.1.0` on Windows and Linux; on Windows every checked frame is byte-identical to the PR build). `exportVideo` at 1080p30: 10.00 s, 1920×1080 H.264 with an AAC track, decodes without error, and its frame at 5 s shows the tilted card with rounded corners. Before the cut, on a dev build of the PR: the four device frames, dual-frame and vertical-stack, under `orbit`, `left` and `none`, rendered byte-identical to `v2.1.0` (49 of 49 frames). **Not covered:** recording, playback and the editor preview, which the patch does not touch; the rest of the checklist (the `v2.1.0-rc.6` rows). |
| 2026-10-19 | `v2.1.2-rc.1`, `release/v2.1.2` @ `c4ef8fd` (= `v2.1.1` + #1344), assets of the GitHub pre-release (build run 39224311283). Windows: `Openscreen.Setup.2.1.2-rc.1-x64.exe` unpacked with 7-Zip and run in place. macOS: `Openscreen-Mac-arm64-2.1.2-rc.1.zip`, run in place. Linux: the AppImage, run directly with `--no-sandbox` | Windows 11 Home 26200; macOS 27.0.1, Mac mini M1; Ubuntu 24.04 GNOME Wayland (Xwayland `:0`) | **Partial: fix slice only, no defect** | `openscreen mcp` over stdio, scripted, with a fresh `--user-data-dir` per run. On all three platforms it answered `initialize`, and `usage-ping.json` held today's day, which is saved only once PostHog accepted the event. With `DO_NOT_TRACK=1` no state file was written. stdout held only JSON-RPC on macOS and Linux. On Windows it starts with one `\r\n` that Electron 41 itself writes during browser-process startup, before `main.js` loads: a bare `app.exit(0)` script prints it too, `ELECTRON_RUN_AS_NODE` does not. It predates this change, and the MCP SDK clients skip it with one parse error. Not covered: everything else in this file, including recording, editing, export, the GUI ping and the tray toggle. These runs sent eight real `surface: mcp` events to PostHog on 19/10. |
| 2026-10-23 | `fix/mac-mic-silent-with-system-audio` on `main` @ `264975d`, dev build (`npm run dev`), capture helper built locally with Xcode 27 and copied from `out/Products/Release` by hand | macOS 26.6.2 (25G83), MacBook Pro M2 Pro; built-in microphone and a USB Audio CODEC (08bb:2902) | **Partial: fix slice only, pass** | #1173, Apple's picker, system audio on, output device idle (after `sudo killall coreaudiod`, or checked idle). Before the fix: the microphone delivered nothing until another app played a sound, 56 s (built-in) and 14 s (USB) into the take; without system audio it started at 0.17 s. With the tap merely started after the stream, four takes (built-in twice, USB twice) recorded the microphone from 0.17–0.18 s, but only 12–30 ms ahead of the tap. With this branch's version, the tap waiting for the first microphone buffer: one USB take recorded the microphone from 0.18 s and the tap started 27 ms after it; one take with the microphone off. In both, a sound played mid-take was in system audio. Measured with temporary per-source buffer counters in the helper, not shipped. **Not covered:** this branch's version with the built-in microphone, a legacy-picker take, macOS 15, an Intel Mac, a microphone that never delivers (the 1 s fallback), pause before the first microphone buffer, a CI-built artifact. |
Loading