diff --git a/README.md b/README.md index 6895f0371..ca02bb965 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,13 @@ Browse and install community extensions from the [Recordly Marketplace](https:// - Feedback and issue links from the editor - Project persistence for editor preferences - Faster preview recovery after export + +### Performance and Reliability + +- Stream browser-captured recording chunks to disk to avoid large in-memory video buffers +- Keep editor playback responsive during long recordings with throttled timeline state updates +- Cache caption, zoom, and cursor lookups used by preview rendering + --- # Screenshots diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 32ed313e0..ef00bda44 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -291,6 +291,72 @@ interface Window { videoData: ArrayBuffer, fileName: string, ) => Promise<{ success: boolean; path?: string; message?: string }>; + openRecordingStream: (fileName: string) => Promise<{ + success: boolean; + streamId?: string; + path?: string; + error?: string; + }>; + writeRecordingStreamChunk: ( + streamId: string, + position: number, + chunk: Uint8Array, + ) => Promise<{ success: boolean; error?: string }>; + closeRecordingStream: ( + streamId: string, + options?: { abort?: boolean; mimeType?: string }, + ) => Promise<{ + success: boolean; + path?: string; + message?: string; + error?: string; + bytesWritten?: number; + }>; + openMicrophoneSidecarStream: () => Promise<{ + success: boolean; + streamId?: string; + error?: string; + }>; + writeMicrophoneSidecarStreamChunk: ( + streamId: string, + position: number, + chunk: Uint8Array, + ) => Promise<{ success: boolean; error?: string }>; + closeMicrophoneSidecarStream: ( + streamId: string, + videoPath: string, + options?: { + abort?: boolean; + startDelayMs?: number; + browserMicrophoneProfile?: string; + requestedBrowserMicrophoneProfile?: string | null; + requestedConstraints?: unknown; + mediaTrackSettings?: Record; + audioInputDevices?: Array<{ + deviceId: string; + groupId?: string; + label: string; + }>; + mediaRecorder?: { + mimeType?: string; + audioBitsPerSecond?: number; + timesliceMs?: number; + }; + chunkEvents?: Array<{ + index: number; + size: number; + elapsedMs: number; + deltaMs: number | null; + recordedElapsedMs?: number; + recordedDeltaMs?: number | null; + }>; + pauseIntervals?: Array<{ + startElapsedMs: number; + endElapsedMs?: number; + durationMs?: number; + }>; + }, + ) => Promise<{ success: boolean; path?: string; error?: string }>; storeMicrophoneSidecar: ( audioData: ArrayBuffer, videoPath: string, diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index e88f6f17e..b3202838b 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -1,370 +1,138 @@ -import type { ChildProcessWithoutNullStreams } from "node:child_process"; -import { execFile, spawn } from "node:child_process"; -import fs from "node:fs/promises"; -import path from "node:path"; -import { promisify } from "node:util"; -import { - app, - BrowserWindow, - desktopCapturer, - dialog, - ipcMain, - shell, - systemPreferences, -} from "electron"; -import { showCursor } from "../../cursorHider"; -import { getMonitorHandles } from "../monitorResolver"; -import { ALLOW_RECORDLY_WINDOW_CAPTURE } from "../constants"; -import { startWindowBoundsCapture, stopWindowBoundsCapture } from "../cursor/bounds"; -import { startInteractionCapture, stopInteractionCapture } from "../cursor/interaction"; -import { startNativeCursorMonitor, stopNativeCursorMonitor } from "../cursor/monitor"; -import { - normalizeCursorTelemetrySamples, - pauseCursorCaptureAtBoundary, - persistPendingCursorTelemetry, - resetCursorCaptureClock, - resumeCursorCapture, - sampleCursorPoint, - snapshotCursorTelemetryForPersistence, - startCursorSampling, - stopCursorCapture, - writeCursorTelemetry, -} from "../cursor/telemetry"; -import { getFfmpegBinaryPath } from "../ffmpeg/binary"; -import { - ensureNativeCaptureHelperBinary, - ensureSwiftHelperBinary, - getNativeCaptureHelperBinaryPath, - getSystemCursorHelperBinaryPath, - getSystemCursorHelperSourcePath, - getWindowsCaptureExePath, -} from "../paths/binaries"; -import { rememberApprovedLocalReadPath } from "../project/manager"; -import { - getBrowserMicSidecarFilters, - shouldKeepRecordingAudioSidecars, -} from "../recording/audioFilters"; -import { - getCompanionAudioFallbackInfo, - getFileSizeIfPresent, - type MicrophoneChunkTimingEvent, - type MicrophonePauseInterval, - type RecordingDiagnosticsSnapshot, - recordNativeCaptureDiagnostics, - summarizeMicrophoneChunkTiming, - validateRecordedVideo, - writeRecordingDiagnosticsSnapshot, -} from "../recording/diagnostics"; -import { - buildFfmpegCaptureArgs, - waitForFfmpegCaptureStart, - waitForFfmpegCaptureStop, -} from "../recording/ffmpeg"; -import { - attachNativeCaptureLifecycle, - finalizeStoredVideo, - muxNativeMacRecordingWithAudio, - recoverNativeMacCaptureOutput, - waitForNativeCaptureStart, - waitForNativeCaptureStop, -} from "../recording/mac"; -import { - attachWindowsCaptureLifecycle, - isNativeWindowsCaptureAvailable, - muxNativeWindowsVideoWithAudio, - waitForWindowsCaptureStart, - waitForWindowsCaptureStop, -} from "../recording/windows"; -import { - shouldStartWindowsBrowserMicrophoneFallback, - shouldUseWindowsBrowserMicrophoneFallback, -} from "../recording/windowsFallbacks"; -import { - cachedSystemCursorAssets, - cachedSystemCursorAssetsSourceMtimeMs, - currentVideoPath, - ffmpegCaptureOutputBuffer, - ffmpegCaptureProcess, - ffmpegCaptureTargetPath, - ffmpegScreenRecordingActive, - lastNativeCaptureDiagnostics, - nativeCaptureMicrophonePath, - nativeCaptureOutputBuffer, - nativeCapturePaused, - nativeCaptureProcess, - nativeCaptureSystemAudioPath, - nativeCaptureTargetPath, - nativeScreenRecordingActive, - selectedSource, - setActiveCursorSamples, - setCachedSystemCursorAssets, - setCachedSystemCursorAssetsSourceMtimeMs, - setCursorCaptureStartTimeMs, - setFfmpegCaptureOutputBuffer, - setFfmpegCaptureProcess, - setFfmpegCaptureTargetPath, - setFfmpegScreenRecordingActive, - setIsCursorCaptureActive, - setLastLeftClick, - setLinuxCursorScreenPoint, - setNativeCaptureMicrophonePath, - setNativeCaptureOutputBuffer, - setNativeCapturePaused, - setNativeCaptureProcess, - setNativeCaptureStopRequested, - setNativeCaptureSystemAudioPath, - setNativeCaptureTargetPath, - setNativeScreenRecordingActive, - setPendingCursorSamples, - setWindowsCaptureOutputBuffer, - setWindowsCapturePaused, - setWindowsCaptureProcess, - setWindowsCaptureStopRequested, - setWindowsCaptureTargetPath, - setWindowsMicAudioPath, - setWindowsNativeCaptureActive, - setWindowsOrphanedMicAudioPath, - setWindowsPendingVideoPath, - setWindowsSystemAudioPath, - windowsCaptureOutputBuffer, - windowsCapturePaused, - windowsCaptureProcess, - windowsCaptureTargetPath, - windowsMicAudioPath, - windowsNativeCaptureActive, - windowsOrphanedMicAudioPath, - windowsPendingVideoPath, - windowsSystemAudioPath, -} from "../state"; -import type { CursorTelemetryPoint, NativeMacRecordingOptions, SelectedSource } from "../types"; -import { - getMacPrivacySettingsUrl, - getRecordingsDir, - getScreen, - getTelemetryPathForVideo, - moveFileWithOverwrite, - normalizeVideoSourcePath, - parseJsonWithByteOrderMark, - parseWindowId, -} from "../utils"; -import { resolveWindowsCaptureDisplay } from "../windowsCaptureSelection"; - -const execFileAsync = promisify(execFile); - -async function writeWindowsRecordingDiagnostics( - videoPath: string | null | undefined, - snapshot: Omit, +async function finalizeMicrophoneSidecarFromWebm( + tempWebmPath: string, + videoPath: string, + options?: BrowserMicrophoneSidecarOptions, ) { - if (!videoPath) { - return null; - } + const baseName = videoPath.replace(/\.[^.]+$/, ""); + const sidecarPath = `${baseName}.mic.wav`; + const sourceWebmPath = `${baseName}.mic.source.webm`; + const sourceBytes = await getFileSizeIfPresent(tempWebmPath); try { - return await writeRecordingDiagnosticsSnapshot(videoPath, { - backend: "windows-wgc", - ...snapshot, - }); - } catch (error) { - console.warn("Failed to write Windows recording diagnostics:", error); - return null; - } -} - -function isRecord(value: unknown): value is Record { - return Boolean(value && typeof value === "object" && !Array.isArray(value)); -} - -function pickPrimitiveRecord(value: unknown) { - if (!isRecord(value)) { - return null; - } - - const entries = Object.entries(value).filter( - (entry): entry is [string, boolean | number | string] => { - const primitive = entry[1]; - return ( - typeof primitive === "boolean" || - typeof primitive === "number" || - typeof primitive === "string" - ); - }, - ); - - return entries.length > 0 ? Object.fromEntries(entries) : null; -} - -function normalizeRendererTimestampMs(value: unknown) { - const nowMs = Date.now(); - if (typeof value !== "number" || !Number.isFinite(value)) { - return nowMs; - } - - return Math.min(Math.max(0, Math.round(value)), nowMs); -} - -function pickMicrophoneChunkEvents(value: unknown): MicrophoneChunkTimingEvent[] | null { - if (!Array.isArray(value)) { - return null; - } - - const events = value - .map((event) => { - if (!isRecord(event)) { - return null; - } - - const { index, size, elapsedMs, deltaMs, recordedElapsedMs, recordedDeltaMs } = event; - if ( - typeof index !== "number" || - !Number.isFinite(index) || - index < 0 || - typeof size !== "number" || - !Number.isFinite(size) || - size < 0 || - typeof elapsedMs !== "number" || - !Number.isFinite(elapsedMs) || - elapsedMs < 0 - ) { - return null; - } - - return { - index: Math.round(index), - size: Math.round(size), - elapsedMs: Math.round(elapsedMs), - deltaMs: - typeof deltaMs === "number" && Number.isFinite(deltaMs) - ? Math.max(0, Math.round(deltaMs)) - : null, - ...(typeof recordedElapsedMs === "number" && - Number.isFinite(recordedElapsedMs) && - recordedElapsedMs >= 0 - ? { recordedElapsedMs: Math.round(recordedElapsedMs) } - : {}), - recordedDeltaMs: - typeof recordedDeltaMs === "number" && Number.isFinite(recordedDeltaMs) - ? Math.max(0, Math.round(recordedDeltaMs)) - : null, - }; - }) - .filter((event): event is NonNullable => event !== null); - - return events.length > 0 ? events : null; -} - -function pickMicrophonePauseIntervals(value: unknown): MicrophonePauseInterval[] | null { - if (!Array.isArray(value)) { - return null; - } - - const intervals = value - .map((interval) => { - if ( - !isRecord(interval) || - typeof interval.startElapsedMs !== "number" || - !Number.isFinite(interval.startElapsedMs) || - interval.startElapsedMs < 0 - ) { - return null; - } - - const startElapsedMs = Math.max(0, Math.round(interval.startElapsedMs)); - return { - startElapsedMs, - ...(typeof interval.endElapsedMs === "number" && - Number.isFinite(interval.endElapsedMs) && - interval.endElapsedMs >= startElapsedMs - ? { endElapsedMs: Math.round(interval.endElapsedMs) } - : {}), - ...(typeof interval.durationMs === "number" && - Number.isFinite(interval.durationMs) && - interval.durationMs >= 0 - ? { durationMs: Math.round(interval.durationMs) } - : {}), - }; - }) - .filter((interval): interval is NonNullable => interval !== null); + await execFileAsync( + getFfmpegBinaryPath(), + [ + "-y", + "-hide_banner", + "-nostdin", + "-nostats", + "-i", + tempWebmPath, + "-vn", + "-ac", + "1", + "-ar", + "48000", + "-af", + [ + ...getBrowserMicSidecarFilters(options?.browserMicrophoneProfile), + "aresample=async=1:first_pts=0", + ].join(","), + "-c:a", + "pcm_s16le", + sidecarPath, + ], + { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, + ); + + if (shouldKeepRecordingAudioSidecars()) { + await fs.rename(tempWebmPath, sourceWebmPath).catch(async () => { + await fs.copyFile(tempWebmPath, sourceWebmPath); + await fs.rm(tempWebmPath, { force: true }); + }); + } else { + await fs.rm(tempWebmPath, { force: true }); + } - return intervals.length > 0 ? intervals : null; -} + const startDelayMs = options?.startDelayMs; + const mediaTrackSettings = pickPrimitiveRecord(options?.mediaTrackSettings); + const audioInputDevices = pickAudioInputDevices(options?.audioInputDevices); + const mediaRecorder = isRecord(options?.mediaRecorder) + ? { + ...(typeof options.mediaRecorder.mimeType === "string" + ? { mimeType: options.mediaRecorder.mimeType } + : {}), + ...(typeof options.mediaRecorder.audioBitsPerSecond === "number" + ? { + audioBitsPerSecond: Math.round( + options.mediaRecorder.audioBitsPerSecond, + ), + } + : {}), + ...(typeof options.mediaRecorder.timesliceMs === "number" + ? { timesliceMs: Math.round(options.mediaRecorder.timesliceMs) } + : {}), + } + : null; -function pickAudioInputDevices(value: unknown) { - if (!Array.isArray(value)) { - return null; - } + const chunkEvents = pickMicrophoneChunkEvents(options?.chunkEvents); + const pauseIntervals = pickMicrophonePauseIntervals(options?.pauseIntervals); + const chunkTiming = + chunkEvents || pauseIntervals + ? summarizeMicrophoneChunkTiming( + chunkEvents, + pauseIntervals, + mediaRecorder?.timesliceMs, + ) + : null; + + const metadata = { + ...(Number.isFinite(startDelayMs) && (startDelayMs ?? 0) >= 0 + ? { startDelayMs: Math.round(startDelayMs ?? 0) } + : {}), + ...(typeof options?.browserMicrophoneProfile === "string" + ? { browserMicrophoneProfile: options.browserMicrophoneProfile } + : {}), + ...(typeof options?.requestedBrowserMicrophoneProfile === "string" + ? { + requestedBrowserMicrophoneProfile: + options.requestedBrowserMicrophoneProfile, + } + : {}), + ...(isRecord(options?.requestedConstraints) + ? { requestedConstraints: options.requestedConstraints } + : {}), + ...(mediaTrackSettings ? { mediaTrackSettings } : {}), + ...(audioInputDevices ? { audioInputDevices } : {}), + ...(mediaRecorder && Object.keys(mediaRecorder).length > 0 + ? { mediaRecorder } + : {}), + ...(chunkEvents ? { chunkEvents } : {}), + ...(pauseIntervals ? { pauseIntervals } : {}), + ...(chunkTiming ? { chunkTiming } : {}), + }; - const devices = value - .map((device) => { - if (!isRecord(device) || typeof device.deviceId !== "string") { - return null; + if (Object.keys(metadata).length > 0) { + try { + await fs.writeFile(`${sidecarPath}.json`, JSON.stringify(metadata)); + } catch (metadataError) { + console.warn("Failed to store microphone sidecar timing metadata:", metadataError); } + } - return { - deviceId: device.deviceId, - ...(typeof device.groupId === "string" ? { groupId: device.groupId } : {}), - label: typeof device.label === "string" ? device.label : "", - }; - }) - .filter((device): device is NonNullable => device !== null); - - return devices.length > 0 ? devices : null; -} - -async function getSystemCursorAssets() { - if (process.platform !== "darwin") { - setCachedSystemCursorAssets({}); - setCachedSystemCursorAssetsSourceMtimeMs(null); - return cachedSystemCursorAssets ?? {}; - } - const sourcePath = getSystemCursorHelperSourcePath(); - const sourceStat = await fs.stat(sourcePath); - if (cachedSystemCursorAssets && cachedSystemCursorAssetsSourceMtimeMs === sourceStat.mtimeMs) { - return cachedSystemCursorAssets; - } - const binaryPath = await ensureSwiftHelperBinary( - sourcePath, - getSystemCursorHelperBinaryPath(), - "system cursor helper", - "recordly-system-cursors", - ); - const { stdout } = await execFileAsync(binaryPath, [], { - timeout: 15000, - maxBuffer: 20 * 1024 * 1024, - }); - const parsed = JSON.parse(stdout) as Record< - string, - Partial - >; - const result = Object.fromEntries( - Object.entries(parsed).filter( - ([, asset]) => - typeof asset?.dataUrl === "string" && - typeof asset?.hotspotX === "number" && - typeof asset?.hotspotY === "number" && - typeof asset?.width === "number" && - typeof asset?.height === "number", - ), - ) as Record; - setCachedSystemCursorAssets(result); - setCachedSystemCursorAssetsSourceMtimeMs(sourceStat.mtimeMs); - return result; -} - -function normalizeDesktopSourceName(value: string) { - return value.trim().replace(/\s+/g, " ").toLowerCase(); -} - -async function cleanupWindowsOrphanedMicAudioPath(filePath: string | null) { - if (!filePath) { - return; - } + await writeRecordingDiagnosticsSnapshot(videoPath, { + backend: "browser-store", + phase: "mic-sidecar", + outputPath: videoPath, + microphonePath: sidecarPath, + details: { + sourceBytes, + sourceWebmPath: shouldKeepRecordingAudioSidecars() ? sourceWebmPath : null, + metadata, + }, + }).catch((diagnosticsError) => { + console.warn("Failed to write microphone sidecar diagnostics:", diagnosticsError); + }); - if (shouldKeepRecordingAudioSidecars()) { - console.log(`[recording] Keeping orphaned native mic sidecar for diagnostics: ${filePath}`); - return; + return { success: true, path: sidecarPath }; + } catch (error) { + await Promise.all([ + fs.rm(tempWebmPath, { force: true }).catch(() => undefined), + fs.rm(sidecarPath, { force: true }).catch(() => undefined), + ]); + console.error("Failed to store microphone sidecar:", error); + return { success: false, error: String(error) }; } - - await fs.rm(filePath, { force: true }).catch(() => undefined); } async function pathExists(filePath: string | null | undefined) { @@ -388,1530 +156,4 @@ async function resolveExistingPath(...candidates: Array void, -) { - ipcMain.handle( - "start-native-screen-recording", - async (_, source: SelectedSource, options?: NativeMacRecordingOptions) => { - // Windows native capture path - if (process.platform === "win32") { - const windowsCaptureAvailable = await isNativeWindowsCaptureAvailable(); - if (!windowsCaptureAvailable) { - return { - success: false, - message: "Native Windows capture is not available on this system.", - }; - } - - if (windowsCaptureProcess && !windowsNativeCaptureActive) { - try { - windowsCaptureProcess.kill(); - } catch { - /* ignore */ - } - setWindowsCaptureProcess(null); - setWindowsCaptureTargetPath(null); - setWindowsCaptureStopRequested(false); - } - - if (windowsCaptureProcess) { - return { - success: false, - message: "A native Windows screen recording is already active.", - }; - } - - let wcProc: ChildProcessWithoutNullStreams | null = null; - let tempVideoPath: string | null = null; - let tempSystemAudioPath: string | null = null; - let tempMicPath: string | null = null; - try { - const exePath = getWindowsCaptureExePath(); - const recordingsDir = await getRecordingsDir(); - const timestamp = Date.now(); - const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`); - tempVideoPath = path.join(app.getPath("temp"), `recordly-native-${timestamp}.mp4`); - - let captureOutput = ""; - let systemAudioPath: string | null = null; - let microphonePath: string | null = null; - let orphanedMicAudioPath: string | null = null; - - const browserMicFallbackRequested = - shouldStartWindowsBrowserMicrophoneFallback(options); - const windowId = parseWindowId(source?.id); - const isWindowCapture = Boolean(windowId && source?.id?.startsWith("window:")); - - const resolvedDisplay = resolveWindowsCaptureDisplay( - source, - getScreen().getAllDisplays(), - getScreen().getPrimaryDisplay(), - ); - const displayBounds = resolvedDisplay.bounds; - setWindowsOrphanedMicAudioPath(null); - - const config: Record = { - outputPath: tempVideoPath, - fps: 60, - }; - - if (isWindowCapture) { - config.windowHandle = windowId; - } else { - // Windows Graphics Capture (WGC) requires a raw HMONITOR handle. - // We attempt to resolve the handle by matching the physical coordinates of the target display. - const monitors = getMonitorHandles(); - const matchedMonitor = monitors.find( - (monitor) => - monitor.x === Math.round(displayBounds.x) && - monitor.y === Math.round(displayBounds.y), - ); - - if (matchedMonitor) { - config.displayId = matchedMonitor.handle; - } else { - // Fallback to coordinate-based matching if handle resolution fails - config.displayId = resolvedDisplay.displayId; - } - - config.displayX = Math.round(resolvedDisplay.bounds.x); - config.displayY = Math.round(resolvedDisplay.bounds.y); - config.displayW = Math.round(resolvedDisplay.bounds.width); - config.displayH = Math.round(resolvedDisplay.bounds.height); - } - - if (options?.capturesSystemAudio) { - systemAudioPath = path.join( - recordingsDir, - `recording-${timestamp}.system.wav`, - ); - tempSystemAudioPath = path.join( - app.getPath("temp"), - `recordly-native-${timestamp}.system.wav`, - ); - config.captureSystemAudio = true; - config.audioOutputPath = tempSystemAudioPath; - setWindowsSystemAudioPath(systemAudioPath); - } else { - setWindowsSystemAudioPath(null); - } - - if (options?.capturesMicrophone && !browserMicFallbackRequested) { - microphonePath = path.join(recordingsDir, `recording-${timestamp}.mic.wav`); - tempMicPath = path.join(app.getPath("temp"), `recordly-native-${timestamp}.mic.wav`); - config.captureMic = true; - config.micOutputPath = tempMicPath; - if (options.microphoneLabel) { - config.micDeviceName = options.microphoneLabel; - } - setWindowsMicAudioPath(microphonePath); - } else if (browserMicFallbackRequested) { - config.captureMic = false; - setWindowsMicAudioPath(null); - } else { - setWindowsMicAudioPath(null); - } - - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "start", - sourceId: source?.id ?? null, - sourceType: source?.sourceType ?? "unknown", - displayId: typeof config.displayId === "number" ? config.displayId : null, - displayBounds, - windowHandle: - typeof config.windowHandle === "number" ? config.windowHandle : null, - helperPath: exePath, - outputPath, - systemAudioPath, - microphonePath, - }); - - setWindowsCaptureOutputBuffer(""); - setWindowsCaptureTargetPath(outputPath); - setWindowsCaptureStopRequested(false); - setWindowsCapturePaused(false); - - // The native helper currently does not declare DPI awareness in its own - // manifest or process setup, so we keep the compatibility flag here until - // scaled-display capture is verified without it on Windows. - wcProc = spawn(exePath, [JSON.stringify(config)], { - cwd: recordingsDir, - stdio: ["pipe", "pipe", "pipe"], - env: { ...process.env, __COMPAT_LAYER: "HighDpiAware" }, - }); - setWindowsCaptureProcess(wcProc); - attachWindowsCaptureLifecycle(wcProc); - - wcProc.stdout.on("data", (chunk: Buffer) => { - const msg = chunk.toString(); - captureOutput += msg; - setWindowsCaptureOutputBuffer(captureOutput); - }); - wcProc.stderr.on("data", (chunk: Buffer) => { - const msg = chunk.toString(); - captureOutput += msg; - setWindowsCaptureOutputBuffer(captureOutput); - }); - - await waitForWindowsCaptureStart(wcProc); - const microphoneFallbackRequired = - browserMicFallbackRequested || - shouldUseWindowsBrowserMicrophoneFallback(captureOutput, options); - if (microphoneFallbackRequired) { - orphanedMicAudioPath = tempMicPath ?? microphonePath; - setWindowsOrphanedMicAudioPath(orphanedMicAudioPath); - microphonePath = null; - setWindowsMicAudioPath(null); - } - setWindowsNativeCaptureActive(true); - setNativeScreenRecordingActive(true); - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "start", - sourceId: source?.id ?? null, - sourceType: source?.sourceType ?? "unknown", - displayId: typeof config.displayId === "number" ? config.displayId : null, - displayBounds, - windowHandle: - typeof config.windowHandle === "number" ? config.windowHandle : null, - helperPath: exePath, - outputPath, - systemAudioPath, - microphonePath, - processOutput: captureOutput.trim() || undefined, - }); - return { success: true, microphoneFallbackRequired }; - } catch (error) { - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "start", - sourceId: source?.id ?? null, - sourceType: source?.sourceType ?? "unknown", - helperPath: windowsCaptureTargetPath ? getWindowsCaptureExePath() : null, - outputPath: windowsCaptureTargetPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - error: String(error), - }); - console.error("Failed to start native Windows capture:", error); - try { - if (wcProc) wcProc.kill(); - } catch { - /* ignore */ - } - await Promise.allSettled([ - tempVideoPath - ? fs.rm(tempVideoPath, { force: true }).catch(() => undefined) - : Promise.resolve(), - tempSystemAudioPath - ? fs.rm(tempSystemAudioPath, { force: true }).catch(() => undefined) - : Promise.resolve(), - tempMicPath - ? fs.rm(tempMicPath, { force: true }).catch(() => undefined) - : Promise.resolve(), - ]); - setWindowsNativeCaptureActive(false); - setNativeScreenRecordingActive(false); - setWindowsCaptureProcess(null); - setWindowsCaptureTargetPath(null); - setWindowsSystemAudioPath(null); - setWindowsMicAudioPath(null); - setWindowsOrphanedMicAudioPath(null); - setWindowsCaptureStopRequested(false); - setWindowsCapturePaused(false); - return { - success: false, - message: "Failed to start native Windows capture", - error: String(error), - }; - } - } - - if (process.platform !== "darwin") { - return { - success: false, - message: "Native screen recording is only available on macOS.", - }; - } - - if (nativeCaptureProcess && !nativeScreenRecordingActive) { - try { - nativeCaptureProcess.kill(); - } catch { - // ignore stale helper cleanup failures - } - setNativeCaptureProcess(null); - setNativeCaptureTargetPath(null); - setNativeCaptureStopRequested(false); - } - - if (nativeCaptureProcess) { - return { success: false, message: "A native screen recording is already active." }; - } - - let captProc: ChildProcessWithoutNullStreams | null = null; - try { - const recordingsDir = await getRecordingsDir(); - - // Warm up TCC: trigger an Electron-level screen capture API call so macOS - // activates the screen-recording grant for this process tree before the - // native helper binary spawns and calls SCStream.startCapture(). - try { - await desktopCapturer.getSources({ - types: ["screen"], - thumbnailSize: { width: 1, height: 1 }, - }); - } catch { - // non-fatal – the helper will report its own TCC status - } - - // Ensure microphone TCC is granted for this process tree when mic capture - // is requested, so the child helper inherits the grant. - if (options?.capturesMicrophone) { - const micStatus = systemPreferences.getMediaAccessStatus("microphone"); - if (micStatus !== "granted") { - await systemPreferences.askForMediaAccess("microphone"); - } - } - - const appName = normalizeDesktopSourceName(String(source?.appName ?? "")); - const ownAppName = normalizeDesktopSourceName(app.getName()); - if ( - !ALLOW_RECORDLY_WINDOW_CAPTURE && - source?.id?.startsWith("window:") && - appName && - (appName === ownAppName || appName === "recordly") - ) { - return { - success: false, - message: - "Cannot record Recordly windows. Please select another app window.", - }; - } - - const helperPath = await ensureNativeCaptureHelperBinary(); - const timestamp = Date.now(); - const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`); - const capturesSystemAudio = Boolean(options?.capturesSystemAudio); - const capturesMicrophone = Boolean(options?.capturesMicrophone); - const systemAudioOutputPath = capturesSystemAudio - ? path.join(recordingsDir, `recording-${timestamp}.system.m4a`) - : null; - const microphoneOutputPath = capturesMicrophone - ? path.join(recordingsDir, `recording-${timestamp}.mic.m4a`) - : null; - const config: Record = { - fps: 60, - outputPath, - capturesSystemAudio, - capturesMicrophone, - }; - - if (options?.microphoneDeviceId) { - config.microphoneDeviceId = options.microphoneDeviceId; - } - - if (options?.microphoneLabel) { - config.microphoneLabel = options.microphoneLabel; - } - - if (systemAudioOutputPath) { - config.systemAudioOutputPath = systemAudioOutputPath; - } - - if (microphoneOutputPath) { - config.microphoneOutputPath = microphoneOutputPath; - } - - const windowId = parseWindowId(source?.id); - const screenId = Number(source?.display_id); - - if (Number.isFinite(windowId) && windowId && source?.id?.startsWith("window:")) { - config.windowId = windowId; - } else if (Number.isFinite(screenId) && screenId > 0) { - config.displayId = screenId; - } else { - config.displayId = Number(getScreen().getPrimaryDisplay().id); - } - - setNativeCaptureOutputBuffer(""); - setNativeCaptureTargetPath(outputPath); - setNativeCaptureSystemAudioPath(systemAudioOutputPath); - setNativeCaptureMicrophonePath(microphoneOutputPath); - setNativeCaptureStopRequested(false); - setNativeCapturePaused(false); - captProc = spawn(helperPath, [JSON.stringify(config)], { - cwd: recordingsDir, - stdio: ["pipe", "pipe", "pipe"], - }); - setNativeCaptureProcess(captProc); - attachNativeCaptureLifecycle(captProc); - - captProc.stdout.on("data", (chunk: Buffer) => { - setNativeCaptureOutputBuffer(nativeCaptureOutputBuffer + chunk.toString()); - }); - captProc.stderr.on("data", (chunk: Buffer) => { - setNativeCaptureOutputBuffer(nativeCaptureOutputBuffer + chunk.toString()); - }); - - await waitForNativeCaptureStart(captProc); - setNativeScreenRecordingActive(true); - - // If the native helper reported MICROPHONE_CAPTURE_UNAVAILABLE, it started - // capture without microphone. Clear the mic path so the renderer can fall - // back to a browser-side sidecar recording for the microphone track. - const micUnavailableNatively = nativeCaptureOutputBuffer.includes( - "MICROPHONE_CAPTURE_UNAVAILABLE", - ); - if (micUnavailableNatively) { - setNativeCaptureMicrophonePath(null); - } - - recordNativeCaptureDiagnostics({ - backend: "mac-screencapturekit", - phase: "start", - sourceId: source?.id ?? null, - sourceType: source?.sourceType ?? "unknown", - displayId: typeof config.displayId === "number" ? config.displayId : null, - helperPath, - outputPath, - systemAudioPath: systemAudioOutputPath, - microphonePath: nativeCaptureMicrophonePath, - processOutput: nativeCaptureOutputBuffer.trim() || undefined, - }); - return { success: true, microphoneFallbackRequired: micUnavailableNatively }; - } catch (error) { - console.error("Failed to start native ScreenCaptureKit recording:", error); - const errorStr = String(error); - - // Detect TCC (screen recording permission) errors and show a helpful dialog - if ( - errorStr.includes("declined TCC") || - errorStr.includes("declined TCCs") || - errorStr.includes("SCREEN_RECORDING_PERMISSION_DENIED") - ) { - const { response } = await dialog.showMessageBox({ - type: "warning", - title: "Screen Recording Permission Required", - message: - "Recordly needs screen recording permission to capture your screen.", - detail: "Please open System Settings > Privacy & Security > Screen Recording, make sure Recordly is toggled ON, then try recording again.", - buttons: ["Open System Settings", "Cancel"], - defaultId: 0, - cancelId: 1, - }); - if (response === 0) { - await shell.openExternal(getMacPrivacySettingsUrl("screen")); - } - try { - if (captProc) captProc.kill(); - } catch { - /* ignore */ - } - setNativeScreenRecordingActive(false); - setNativeCaptureProcess(null); - setNativeCaptureTargetPath(null); - setNativeCaptureSystemAudioPath(null); - setNativeCaptureMicrophonePath(null); - setNativeCaptureStopRequested(false); - setNativeCapturePaused(false); - return { - success: false, - message: - "Screen recording permission not granted. Please allow access in System Settings and restart the app.", - userNotified: true, - }; - } - - if (errorStr.includes("MICROPHONE_PERMISSION_DENIED")) { - const { response } = await dialog.showMessageBox({ - type: "warning", - title: "Microphone Permission Required", - message: "Recordly needs microphone permission to record audio.", - detail: "Please open System Settings > Privacy & Security > Microphone, make sure Recordly is toggled ON, then try recording again.", - buttons: ["Open System Settings", "Cancel"], - defaultId: 0, - cancelId: 1, - }); - if (response === 0) { - await shell.openExternal(getMacPrivacySettingsUrl("microphone")); - } - try { - if (captProc) captProc.kill(); - } catch { - /* ignore */ - } - setNativeScreenRecordingActive(false); - setNativeCaptureProcess(null); - setNativeCaptureTargetPath(null); - setNativeCaptureSystemAudioPath(null); - setNativeCaptureMicrophonePath(null); - setNativeCaptureStopRequested(false); - setNativeCapturePaused(false); - return { - success: false, - message: - "Microphone permission not granted. Please allow access in System Settings.", - userNotified: true, - }; - } - - recordNativeCaptureDiagnostics({ - backend: "mac-screencapturekit", - phase: "start", - sourceId: source?.id ?? null, - sourceType: source?.sourceType ?? "unknown", - helperPath: getNativeCaptureHelperBinaryPath(), - outputPath: nativeCaptureTargetPath, - systemAudioPath: nativeCaptureSystemAudioPath, - microphonePath: nativeCaptureMicrophonePath, - processOutput: nativeCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: await getFileSizeIfPresent(nativeCaptureTargetPath), - error: String(error), - }); - try { - if (captProc) captProc.kill(); - } catch { - // ignore cleanup failures - } - setNativeScreenRecordingActive(false); - setNativeCaptureProcess(null); - setNativeCaptureTargetPath(null); - setNativeCaptureSystemAudioPath(null); - setNativeCaptureMicrophonePath(null); - setNativeCaptureStopRequested(false); - setNativeCapturePaused(false); - return { - success: false, - message: "Failed to start native ScreenCaptureKit recording", - error: String(error), - }; - } - }, - ); - - ipcMain.handle("stop-native-screen-recording", async () => { - const start = Date.now(); - console.log("[PERF:MAIN] Handler: stop-native-screen-recording: STARTED"); - try { - // Windows native capture stop path - if (process.platform === "win32" && windowsNativeCaptureActive) { - let stagedTempVideoPath: string | null = null; - let stagedTempSystemAudioPath: string | null = null; - let stagedTempMicAudioPath: string | null = null; - try { - if (!windowsCaptureProcess) { - throw new Error("Native Windows capture process is not running"); - } - - const proc = windowsCaptureProcess; - const preferredVideoPath = windowsCaptureTargetPath; - const preferredOrphanedMicAudioPath = windowsOrphanedMicAudioPath; - const diagnosticsSystemAudioPath = windowsSystemAudioPath; - const diagnosticsMicAudioPath = windowsMicAudioPath; - setWindowsCaptureStopRequested(true); - proc.stdin.write("stop\n"); - const tempVideoPath = await waitForWindowsCaptureStop(proc); - stagedTempVideoPath = tempVideoPath; - const finalVideoPath = preferredVideoPath ?? tempVideoPath; - - // Native Windows capture results are initially written to a safe temporary path - // (to avoid encoding failures with non-ASCII characters). We move them to the final - // destination now using Node.js, which handles Unicode paths correctly. - if (tempVideoPath !== finalVideoPath) { - await moveFileWithOverwrite(tempVideoPath, finalVideoPath); - } - - if (windowsSystemAudioPath && tempVideoPath.endsWith(".mp4")) { - const tempAudioPath = tempVideoPath.replace(".mp4", ".system.wav"); - stagedTempSystemAudioPath = tempAudioPath; - const finalAudioPath = windowsSystemAudioPath; - if (await pathExists(tempAudioPath)) { - await moveFileWithOverwrite(tempAudioPath, finalAudioPath); - const tempJson = tempAudioPath + ".json"; - if (await pathExists(tempJson)) { - await moveFileWithOverwrite(tempJson, finalAudioPath + ".json"); - } - } - } - - if (windowsMicAudioPath && tempVideoPath.endsWith(".mp4")) { - const tempMicPath = tempVideoPath.replace(".mp4", ".mic.wav"); - stagedTempMicAudioPath = tempMicPath; - const finalMicPath = windowsMicAudioPath; - if (await pathExists(tempMicPath)) { - await moveFileWithOverwrite(tempMicPath, finalMicPath); - const tempJson = tempMicPath + ".json"; - if (await pathExists(tempJson)) { - await moveFileWithOverwrite(tempJson, finalMicPath + ".json"); - } - } - } - const validation = await validateRecordedVideo(finalVideoPath); - - setWindowsCaptureProcess(null); - setWindowsNativeCaptureActive(false); - setNativeScreenRecordingActive(false); - setWindowsCaptureTargetPath(null); - setWindowsCaptureStopRequested(false); - setWindowsCapturePaused(false); - setWindowsOrphanedMicAudioPath(null); - await cleanupWindowsOrphanedMicAudioPath(preferredOrphanedMicAudioPath); - setWindowsPendingVideoPath(finalVideoPath); - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "stop", - outputPath: finalVideoPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: validation.fileSizeBytes, - }); - await writeWindowsRecordingDiagnostics(finalVideoPath, { - phase: "stop", - outputPath: finalVideoPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - details: { - fileSizeBytes: validation.fileSizeBytes, - durationSeconds: validation.durationSeconds, - }, - }); - - // Persist cursor telemetry before returning so the editor can find it immediately - snapshotCursorTelemetryForPersistence(); - try { - await persistPendingCursorTelemetry(finalVideoPath); - } catch (error) { - console.warn("Failed to persist cursor telemetry during native stop:", error); - } - - return { success: true, path: finalVideoPath }; - } catch (error) { - console.error("Failed to stop native Windows capture:", error); - const fallbackPath = await resolveExistingPath( - windowsCaptureTargetPath, - stagedTempVideoPath, - ); - const recoveredSystemAudioPath = await resolveExistingPath( - windowsSystemAudioPath, - stagedTempSystemAudioPath, - ); - const recoveredMicAudioPath = await resolveExistingPath( - windowsMicAudioPath, - stagedTempMicAudioPath, - ); - const fallbackOrphanedMicAudioPath = windowsOrphanedMicAudioPath; - const diagnosticsSystemAudioPath = recoveredSystemAudioPath ?? windowsSystemAudioPath; - const diagnosticsMicAudioPath = recoveredMicAudioPath ?? windowsMicAudioPath; - setWindowsNativeCaptureActive(false); - setNativeScreenRecordingActive(false); - setWindowsCaptureProcess(null); - setWindowsCaptureTargetPath(null); - setWindowsCaptureStopRequested(false); - setWindowsCapturePaused(false); - setWindowsOrphanedMicAudioPath(null); - - if (fallbackPath) { - try { - const validation = await validateRecordedVideo(fallbackPath); - setWindowsPendingVideoPath(fallbackPath); - setWindowsSystemAudioPath(recoveredSystemAudioPath); - setWindowsMicAudioPath(recoveredMicAudioPath); - await cleanupWindowsOrphanedMicAudioPath(fallbackOrphanedMicAudioPath); - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: validation.fileSizeBytes, - error: String(error), - }); - await writeWindowsRecordingDiagnostics(fallbackPath, { - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - error: String(error), - details: { - fileSizeBytes: validation.fileSizeBytes, - durationSeconds: validation.durationSeconds, - recoveredAfterStopFailure: true, - }, - }); - return { success: true, path: fallbackPath }; - } catch { - // File is absent or failed validation. - } - } - - setWindowsSystemAudioPath(null); - setWindowsMicAudioPath(null); - setWindowsPendingVideoPath(null); - await cleanupWindowsOrphanedMicAudioPath(fallbackOrphanedMicAudioPath); - - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: await getFileSizeIfPresent(fallbackPath), - error: String(error), - }); - await writeWindowsRecordingDiagnostics(fallbackPath, { - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - error: String(error), - details: { - fileSizeBytes: await getFileSizeIfPresent(fallbackPath), - }, - }); - - return { - success: false, - message: "Failed to stop native Windows capture", - error: String(error), - }; - } - } - - if (process.platform !== "darwin") { - return { - success: false, - message: "Native screen recording is only available on macOS.", - }; - } - - if (!nativeScreenRecordingActive) { - const recovered = await recoverNativeMacCaptureOutput(); - if (recovered) { - return recovered; - } - - return { success: false, message: "No native screen recording is active." }; - } - - try { - if (!nativeCaptureProcess) { - throw new Error("Native capture helper process is not running"); - } - - const process = nativeCaptureProcess; - const preferredVideoPath = nativeCaptureTargetPath; - const preferredSystemAudioPath = nativeCaptureSystemAudioPath; - const preferredMicrophonePath = nativeCaptureMicrophonePath; - console.log( - "[stop-native] Audio paths — system:", - preferredSystemAudioPath, - "mic:", - preferredMicrophonePath, - ); - setNativeCaptureStopRequested(true); - process.stdin.write("stop\n"); - const tempVideoPath = await waitForNativeCaptureStop(process); - console.log("[stop-native] Helper stopped, tempVideoPath:", tempVideoPath); - setNativeCaptureProcess(null); - setNativeScreenRecordingActive(false); - setNativeCaptureTargetPath(null); - setNativeCaptureSystemAudioPath(null); - setNativeCaptureMicrophonePath(null); - setNativeCaptureStopRequested(false); - setNativeCapturePaused(false); - - const finalVideoPath = preferredVideoPath ?? tempVideoPath; - if (tempVideoPath !== finalVideoPath) { - await moveFileWithOverwrite(tempVideoPath, finalVideoPath); - } - - if (preferredSystemAudioPath || preferredMicrophonePath) { - console.log( - "[stop-native] Attempting audio mux (merging separate tracks) into:", - finalVideoPath, - ); - try { - await muxNativeMacRecordingWithAudio( - finalVideoPath, - preferredSystemAudioPath, - preferredMicrophonePath, - ); - console.log("[stop-native] Audio mux completed successfully"); - } catch (error) { - console.warn( - "[stop-native] Audio mux failed (video still has inline audio):", - error, - ); - } - } else { - console.log("[stop-native] No separate audio tracks to mux"); - } - - return await finalizeStoredVideo(finalVideoPath); - } catch (error) { - console.error("Failed to stop native ScreenCaptureKit recording:", error); - const fallbackPath = nativeCaptureTargetPath; - const fallbackSystemAudioPath = nativeCaptureSystemAudioPath; - const fallbackMicrophonePath = nativeCaptureMicrophonePath; - const fallbackFileSizeBytes = await getFileSizeIfPresent(fallbackPath); - setNativeScreenRecordingActive(false); - setNativeCaptureProcess(null); - setNativeCaptureTargetPath(null); - setNativeCaptureSystemAudioPath(null); - setNativeCaptureMicrophonePath(null); - setNativeCaptureStopRequested(false); - setNativeCapturePaused(false); - - recordNativeCaptureDiagnostics({ - backend: "mac-screencapturekit", - phase: "stop", - sourceId: lastNativeCaptureDiagnostics?.sourceId ?? null, - sourceType: lastNativeCaptureDiagnostics?.sourceType ?? "unknown", - displayId: lastNativeCaptureDiagnostics?.displayId ?? null, - displayBounds: lastNativeCaptureDiagnostics?.displayBounds ?? null, - windowHandle: lastNativeCaptureDiagnostics?.windowHandle ?? null, - helperPath: lastNativeCaptureDiagnostics?.helperPath ?? null, - outputPath: fallbackPath, - systemAudioPath: fallbackSystemAudioPath, - microphonePath: fallbackMicrophonePath, - osRelease: lastNativeCaptureDiagnostics?.osRelease, - supported: lastNativeCaptureDiagnostics?.supported, - helperExists: lastNativeCaptureDiagnostics?.helperExists, - processOutput: nativeCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: fallbackFileSizeBytes, - error: String(error), - }); - - // Try to recover: if the target file exists on disk, finalize with it - if (fallbackPath) { - try { - await fs.access(fallbackPath); - console.log( - "[stop-native-screen-recording] Recovering with fallback path:", - fallbackPath, - ); - if (fallbackSystemAudioPath || fallbackMicrophonePath) { - try { - await muxNativeMacRecordingWithAudio( - fallbackPath, - fallbackSystemAudioPath, - fallbackMicrophonePath, - ); - } catch (muxError) { - console.warn( - "Failed to mux recovered native macOS audio into capture:", - muxError, - ); - } - } - return await finalizeStoredVideo(fallbackPath); - } catch { - // File doesn't exist or isn't accessible - } - } - - const recovered = await recoverNativeMacCaptureOutput(); - if (recovered) { - return recovered; - } - - return { - success: false, - message: "Failed to stop native ScreenCaptureKit recording", - error: String(error), - }; - } - } finally { - console.log( - `[PERF:MAIN] Handler: stop-native-screen-recording: COMPLETED in ${Date.now() - start}ms`, - ); - } - }); - - ipcMain.handle("recover-native-screen-recording", async () => { - if (process.platform !== "darwin") { - return { - success: false, - message: "Native screen recording recovery is only available on macOS.", - }; - } - - const recovered = await recoverNativeMacCaptureOutput(); - if (recovered) { - return recovered; - } - - return { - success: false, - message: "No recoverable native macOS recording output was found.", - }; - }); - - ipcMain.handle("pause-native-screen-recording", async () => { - if (process.platform === "win32") { - if (!windowsNativeCaptureActive || !windowsCaptureProcess) { - return { success: false, message: "No native Windows screen recording is active." }; - } - - if (windowsCapturePaused) { - return { success: true }; - } - - try { - windowsCaptureProcess.stdin.write("pause\n"); - setWindowsCapturePaused(true); - return { success: true }; - } catch (error) { - return { - success: false, - message: "Failed to pause native Windows capture", - error: String(error), - }; - } - } - - if (process.platform !== "darwin") { - return { - success: false, - message: "Native screen recording is only available on macOS.", - }; - } - - if (!nativeScreenRecordingActive || !nativeCaptureProcess) { - return { success: false, message: "No native screen recording is active." }; - } - - if (nativeCapturePaused) { - return { success: true }; - } - - try { - nativeCaptureProcess.stdin.write("pause\n"); - setNativeCapturePaused(true); - return { success: true }; - } catch (error) { - return { - success: false, - message: "Failed to pause native screen recording", - error: String(error), - }; - } - }); - - ipcMain.handle("resume-native-screen-recording", async () => { - if (process.platform === "win32") { - if (!windowsNativeCaptureActive || !windowsCaptureProcess) { - return { success: false, message: "No native Windows screen recording is active." }; - } - - if (!windowsCapturePaused) { - return { success: true }; - } - - try { - windowsCaptureProcess.stdin.write("resume\n"); - setWindowsCapturePaused(false); - return { success: true }; - } catch (error) { - return { - success: false, - message: "Failed to resume native Windows capture", - error: String(error), - }; - } - } - - if (process.platform !== "darwin") { - return { - success: false, - message: "Native screen recording is only available on macOS.", - }; - } - - if (!nativeScreenRecordingActive || !nativeCaptureProcess) { - return { success: false, message: "No native screen recording is active." }; - } - - if (!nativeCapturePaused) { - return { success: true }; - } - - try { - nativeCaptureProcess.stdin.write("resume\n"); - setNativeCapturePaused(false); - return { success: true }; - } catch (error) { - return { - success: false, - message: "Failed to resume native screen recording", - error: String(error), - }; - } - }); - - ipcMain.handle("get-system-cursor-assets", async () => { - try { - return { success: true, cursors: await getSystemCursorAssets() }; - } catch (error) { - console.error("Failed to load system cursor assets:", error); - return { success: false, cursors: {}, error: String(error) }; - } - }); - - ipcMain.handle("is-native-windows-capture-available", async () => { - return { available: await isNativeWindowsCaptureAvailable() }; - }); - - ipcMain.handle("get-last-native-capture-diagnostics", async () => { - return { success: true, diagnostics: lastNativeCaptureDiagnostics }; - }); - - ipcMain.handle("get-video-audio-fallback-paths", async (_event, videoPath: string) => { - if (!videoPath) { - return { success: true, paths: [], startDelayMsByPath: {} }; - } - - try { - const { paths, startDelayMsByPath } = await getCompanionAudioFallbackInfo(videoPath); - await Promise.all([ - rememberApprovedLocalReadPath(videoPath), - ...paths.map((fallbackPath) => rememberApprovedLocalReadPath(fallbackPath)), - ]); - return { success: true, paths, startDelayMsByPath }; - } catch (error) { - console.error("Failed to resolve companion audio fallback paths:", error); - return { success: false, paths: [], startDelayMsByPath: {}, error: String(error) }; - } - }); - - ipcMain.handle("mux-native-windows-recording", async (_event, expectedDurationMs?: number) => { - const start = Date.now(); - console.log("[PERF:MAIN] Handler: mux-native-windows-recording: STARTED"); - try { - const videoPath = windowsPendingVideoPath; - const orphanedMicAudioPath = windowsOrphanedMicAudioPath; - const diagnosticsSystemAudioPath = windowsSystemAudioPath; - const diagnosticsMicAudioPath = windowsMicAudioPath; - setWindowsPendingVideoPath(null); - setWindowsOrphanedMicAudioPath(null); - - if (!videoPath) { - return { success: false, message: "No native Windows video pending for mux" }; - } - - try { - await writeWindowsRecordingDiagnostics(videoPath, { - phase: "mux-start", - expectedDurationMs, - outputPath: videoPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - details: { - hasSystemAudio: Boolean(diagnosticsSystemAudioPath), - hasMicrophone: Boolean(diagnosticsMicAudioPath), - hasOrphanedMicrophone: Boolean(orphanedMicAudioPath), - }, - }); - console.log("[mux-win] Optimization active: skipping video padding."); - - let muxDetails: unknown = null; - if (diagnosticsSystemAudioPath || diagnosticsMicAudioPath) { - muxDetails = await muxNativeWindowsVideoWithAudio( - videoPath, - diagnosticsSystemAudioPath, - diagnosticsMicAudioPath, - ); - setWindowsSystemAudioPath(null); - setWindowsMicAudioPath(null); - } - - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "mux", - outputPath: videoPath, - fileSizeBytes: await getFileSizeIfPresent(videoPath), - }); - await writeWindowsRecordingDiagnostics(videoPath, { - phase: "mux-complete", - expectedDurationMs, - outputPath: videoPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - details: { - fileSizeBytes: await getFileSizeIfPresent(videoPath), - mux: muxDetails, - }, - }); - await cleanupWindowsOrphanedMicAudioPath(orphanedMicAudioPath); - return await finalizeStoredVideo(videoPath); - } catch (error) { - console.error("Failed to mux native Windows recording:", error); - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "mux", - outputPath: videoPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - fileSizeBytes: await getFileSizeIfPresent(videoPath), - error: String(error), - }); - await writeWindowsRecordingDiagnostics(videoPath, { - phase: "mux-error", - expectedDurationMs, - outputPath: videoPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - error: String(error), - details: { - fileSizeBytes: await getFileSizeIfPresent(videoPath), - }, - }); - setWindowsSystemAudioPath(null); - setWindowsMicAudioPath(null); - await cleanupWindowsOrphanedMicAudioPath(orphanedMicAudioPath); - try { - return await finalizeStoredVideo(videoPath); - } catch { - try { - await validateRecordedVideo(videoPath); - return { - success: false, - path: videoPath, - message: "Failed to mux native Windows recording", - error: String(error), - }; - } catch { - // The fallback path is not safely playable; surface the original mux error. - } - - return { - success: false, - message: "Failed to mux native Windows recording", - error: String(error), - }; - } - } - } finally { - console.log( - `[PERF:MAIN] Handler: mux-native-windows-recording: COMPLETED in ${Date.now() - start}ms`, - ); - } - }); - - ipcMain.handle("start-ffmpeg-recording", async (_, source: SelectedSource) => { - if (ffmpegCaptureProcess) { - return { success: false, message: "An FFmpeg recording is already active." }; - } - - try { - const recordingsDir = await getRecordingsDir(); - const ffmpegPath = getFfmpegBinaryPath(); - const outputPath = path.join(recordingsDir, `recording-${Date.now()}.mp4`); - const args = await buildFfmpegCaptureArgs(source, outputPath); - - setFfmpegCaptureOutputBuffer(""); - setFfmpegCaptureTargetPath(outputPath); - const ffProc = spawn(ffmpegPath, args, { - cwd: recordingsDir, - stdio: ["pipe", "pipe", "pipe"], - }); - setFfmpegCaptureProcess(ffProc); - - ffProc.stdout.on("data", (chunk: Buffer) => { - setFfmpegCaptureOutputBuffer(ffmpegCaptureOutputBuffer + chunk.toString()); - }); - ffProc.stderr.on("data", (chunk: Buffer) => { - setFfmpegCaptureOutputBuffer(ffmpegCaptureOutputBuffer + chunk.toString()); - }); - - await waitForFfmpegCaptureStart(ffProc); - setFfmpegScreenRecordingActive(true); - return { success: true }; - } catch (error) { - console.error("Failed to start FFmpeg recording:", error); - setFfmpegScreenRecordingActive(false); - setFfmpegCaptureProcess(null); - setFfmpegCaptureTargetPath(null); - return { - success: false, - message: "Failed to start FFmpeg recording", - error: String(error), - }; - } - }); - - ipcMain.handle("stop-ffmpeg-recording", async () => { - if (!ffmpegScreenRecordingActive) { - return { success: false, message: "No FFmpeg recording is active." }; - } - - try { - if (!ffmpegCaptureProcess || !ffmpegCaptureTargetPath) { - throw new Error("FFmpeg process is not running"); - } - - const process = ffmpegCaptureProcess; - const outputPath = ffmpegCaptureTargetPath; - process.stdin.write("q\n"); - const finalVideoPath = await waitForFfmpegCaptureStop(process, outputPath); - - setFfmpegCaptureProcess(null); - setFfmpegCaptureTargetPath(null); - setFfmpegScreenRecordingActive(false); - - return await finalizeStoredVideo(finalVideoPath); - } catch (error) { - console.error("Failed to stop FFmpeg recording:", error); - try { - ffmpegCaptureProcess?.kill(); - } catch { - // ignore cleanup failures - } - setFfmpegCaptureProcess(null); - setFfmpegCaptureTargetPath(null); - setFfmpegScreenRecordingActive(false); - return { - success: false, - message: "Failed to stop FFmpeg recording", - error: String(error), - }; - } - }); - - ipcMain.handle( - "store-microphone-sidecar", - async ( - _, - audioData: ArrayBuffer, - videoPath: string, - options?: { - startDelayMs?: number; - browserMicrophoneProfile?: string; - requestedBrowserMicrophoneProfile?: string | null; - requestedConstraints?: unknown; - mediaTrackSettings?: Record; - audioInputDevices?: unknown; - mediaRecorder?: unknown; - chunkEvents?: unknown; - pauseIntervals?: unknown; - }, - ) => { - const baseName = videoPath.replace(/\.[^.]+$/, ""); - const sidecarPath = `${baseName}.mic.wav`; - const sourceWebmPath = `${baseName}.mic.source.webm`; - const tempWebmPath = `${sourceWebmPath}.tmp`; - - try { - await fs.writeFile(tempWebmPath, Buffer.from(audioData)); - await execFileAsync( - getFfmpegBinaryPath(), - [ - "-y", - "-hide_banner", - "-nostdin", - "-nostats", - "-i", - tempWebmPath, - "-vn", - "-ac", - "1", - "-ar", - "48000", - "-af", - [ - ...getBrowserMicSidecarFilters(options?.browserMicrophoneProfile), - "aresample=async=1:first_pts=0", - ].join(","), - "-c:a", - "pcm_s16le", - sidecarPath, - ], - { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, - ); - if (shouldKeepRecordingAudioSidecars()) { - await fs.rename(tempWebmPath, sourceWebmPath).catch(async () => { - await fs.copyFile(tempWebmPath, sourceWebmPath); - await fs.rm(tempWebmPath, { force: true }); - }); - } else { - await fs.rm(tempWebmPath, { force: true }); - } - const startDelayMs = options?.startDelayMs; - const mediaTrackSettings = pickPrimitiveRecord(options?.mediaTrackSettings); - const audioInputDevices = pickAudioInputDevices(options?.audioInputDevices); - const mediaRecorder = isRecord(options?.mediaRecorder) - ? { - ...(typeof options.mediaRecorder.mimeType === "string" - ? { mimeType: options.mediaRecorder.mimeType } - : {}), - ...(typeof options.mediaRecorder.audioBitsPerSecond === "number" - ? { - audioBitsPerSecond: Math.round( - options.mediaRecorder.audioBitsPerSecond, - ), - } - : {}), - ...(typeof options.mediaRecorder.timesliceMs === "number" - ? { timesliceMs: Math.round(options.mediaRecorder.timesliceMs) } - : {}), - } - : null; - const chunkEvents = pickMicrophoneChunkEvents(options?.chunkEvents); - const pauseIntervals = pickMicrophonePauseIntervals(options?.pauseIntervals); - const chunkTiming = - chunkEvents || pauseIntervals - ? summarizeMicrophoneChunkTiming( - chunkEvents, - pauseIntervals, - mediaRecorder?.timesliceMs, - ) - : null; - const metadata = { - ...(Number.isFinite(startDelayMs) && (startDelayMs ?? 0) >= 0 - ? { startDelayMs: Math.round(startDelayMs ?? 0) } - : {}), - ...(typeof options?.browserMicrophoneProfile === "string" - ? { browserMicrophoneProfile: options.browserMicrophoneProfile } - : {}), - ...(typeof options?.requestedBrowserMicrophoneProfile === "string" - ? { - requestedBrowserMicrophoneProfile: - options.requestedBrowserMicrophoneProfile, - } - : {}), - ...(isRecord(options?.requestedConstraints) - ? { requestedConstraints: options.requestedConstraints } - : {}), - ...(mediaTrackSettings ? { mediaTrackSettings } : {}), - ...(audioInputDevices ? { audioInputDevices } : {}), - ...(mediaRecorder && Object.keys(mediaRecorder).length > 0 - ? { mediaRecorder } - : {}), - ...(chunkEvents ? { chunkEvents } : {}), - ...(pauseIntervals ? { pauseIntervals } : {}), - ...(chunkTiming ? { chunkTiming } : {}), - }; - if (Object.keys(metadata).length > 0) { - try { - await fs.writeFile(`${sidecarPath}.json`, JSON.stringify(metadata)); - } catch (metadataError) { - console.warn( - "Failed to store microphone sidecar timing metadata:", - metadataError, - ); - } - } - await writeRecordingDiagnosticsSnapshot(videoPath, { - backend: "browser-store", - phase: "mic-sidecar", - outputPath: videoPath, - microphonePath: sidecarPath, - details: { - sourceBytes: audioData.byteLength, - sourceWebmPath: shouldKeepRecordingAudioSidecars() ? sourceWebmPath : null, - metadata, - }, - }).catch((diagnosticsError) => { - console.warn( - "Failed to write microphone sidecar diagnostics:", - diagnosticsError, - ); - }); - return { success: true, path: sidecarPath }; - } catch (error) { - await Promise.all([ - fs.rm(tempWebmPath, { force: true }).catch(() => undefined), - fs.rm(sidecarPath, { force: true }).catch(() => undefined), - ]); - console.error("Failed to store microphone sidecar:", error); - return { success: false, error: String(error) }; - } - }, - ); - - ipcMain.handle("store-recorded-video", async (_, videoData: ArrayBuffer, fileName: string) => { - try { - const recordingsDir = await getRecordingsDir(); - const videoPath = path.join(recordingsDir, fileName); - await fs.writeFile(videoPath, Buffer.from(videoData)); - return await finalizeStoredVideo(videoPath); - } catch (error) { - console.error("Failed to store video:", error); - return { - success: false, - message: "Failed to store video", - error: String(error), - }; - } - }); - - ipcMain.handle("get-recorded-video-path", async () => { - try { - const recordingsDir = await getRecordingsDir(); - const entries = await fs.readdir(recordingsDir, { withFileTypes: true }); - const candidates = await Promise.all( - entries - .filter( - (entry) => - entry.isFile() && /^recording-\d+\.(webm|mov|mp4)$/i.test(entry.name), - ) - .map(async (entry) => { - const fullPath = path.join(recordingsDir, entry.name); - const stat = await fs.stat(fullPath).catch(() => null); - return stat ? { path: fullPath, mtimeMs: stat.mtimeMs } : null; - }), - ); - const sortedCandidates = candidates - .filter( - (candidate): candidate is { path: string; mtimeMs: number } => - candidate !== null, - ) - .sort((left, right) => right.mtimeMs - left.mtimeMs); - - for (const candidate of sortedCandidates) { - try { - await validateRecordedVideo(candidate.path); - return { success: true, path: candidate.path }; - } catch (error) { - console.warn( - "Skipping unusable recovered recording candidate:", - candidate.path, - error, - ); - } - } - - if (sortedCandidates.length === 0) { - return { success: false, message: "No recorded video found" }; - } - - return { success: false, message: "No usable recorded video found" }; - } catch (error) { - console.error("Failed to get video path:", error); - return { success: false, message: "Failed to get video path", error: String(error) }; - } - }); - - ipcMain.handle("set-recording-state", (_, recording: boolean) => { - if (recording) { - stopCursorCapture(); - stopInteractionCapture(); - startWindowBoundsCapture(); - void startNativeCursorMonitor(); - setIsCursorCaptureActive(true); - setActiveCursorSamples([]); - setPendingCursorSamples([]); - setCursorCaptureStartTimeMs(Date.now()); - resetCursorCaptureClock(); - setLinuxCursorScreenPoint(null); - setLastLeftClick(null); - sampleCursorPoint(); - startCursorSampling(); - void startInteractionCapture(); - } else { - setIsCursorCaptureActive(false); - stopCursorCapture(); - stopInteractionCapture(); - stopWindowBoundsCapture(); - stopNativeCursorMonitor(); - showCursor(); - setLinuxCursorScreenPoint(null); - resetCursorCaptureClock(); - snapshotCursorTelemetryForPersistence(); - setActiveCursorSamples([]); - } - - const source = selectedSource || { name: "Screen" }; - BrowserWindow.getAllWindows().forEach((window) => { - if (!window.isDestroyed()) { - window.webContents.send("recording-state-changed", { - recording, - sourceName: source.name, - }); - } - }); - - if (onRecordingStateChange) { - onRecordingStateChange(recording, source.name); - } - }); - - ipcMain.handle("pause-cursor-capture", (_, pausedAtMs?: unknown) => { - pauseCursorCaptureAtBoundary(normalizeRendererTimestampMs(pausedAtMs)); - return { success: true }; - }); - - ipcMain.handle("resume-cursor-capture", (_, resumedAtMs?: unknown) => { - resumeCursorCapture(normalizeRendererTimestampMs(resumedAtMs)); - sampleCursorPoint(); - return { success: true }; - }); - - ipcMain.handle("get-cursor-telemetry", async (_, videoPath?: string) => { - const targetVideoPath = normalizeVideoSourcePath(videoPath ?? currentVideoPath); - if (!targetVideoPath) { - return { success: true, samples: [] }; - } - - const telemetryPath = getTelemetryPathForVideo(targetVideoPath); - try { - const content = await fs.readFile(telemetryPath, "utf-8"); - const parsed = parseJsonWithByteOrderMark(content); - const samples = normalizeCursorTelemetrySamples(parsed); - - return { success: true, samples }; - } catch (error) { - const nodeError = error as NodeJS.ErrnoException; - if (nodeError.code === "ENOENT") { - return { success: true, samples: [] }; - } - console.error("Failed to load cursor telemetry:", error); - return { - success: false, - message: "Failed to load cursor telemetry", - error: String(error), - samples: [], - }; - } - }); - - ipcMain.handle( - "set-cursor-telemetry", - async (_, videoPath: string | undefined, samples: CursorTelemetryPoint[]) => { - const targetVideoPath = normalizeVideoSourcePath(videoPath ?? currentVideoPath); - if (!targetVideoPath) { - return { - success: false, - samples: [], - message: "No video path available for cursor telemetry", - error: "Missing video path", - }; - } - - try { - const normalizedSamples = await writeCursorTelemetry(targetVideoPath, samples); - return { success: true, samples: normalizedSamples }; - } catch (error) { - console.error("Failed to save cursor telemetry:", error); - return { - success: false, - samples: [], - message: "Failed to save cursor telemetry", - error: String(error), - }; - } - }, - ); -} +} \ No newline at end of file diff --git a/electron/preload.ts b/electron/preload.ts index 932db07b0..2bb5f7db6 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -529,6 +529,70 @@ contextBridge.exposeInMainWorld("electronAPI", { storeRecordedVideo: (videoData: ArrayBuffer, fileName: string) => { return ipcRenderer.invoke("store-recorded-video", videoData, fileName); }, + openRecordingStream: (fileName: string) => { + return ipcRenderer.invoke("recording-stream-open", fileName); + }, + writeRecordingStreamChunk: (streamId: string, position: number, chunk: Uint8Array) => { + return ipcRenderer.invoke("recording-stream-write", streamId, position, chunk); + }, + closeRecordingStream: ( + streamId: string, + options?: { abort?: boolean; mimeType?: string }, + ) => { + return ipcRenderer.invoke("recording-stream-close", streamId, options); + }, + openMicrophoneSidecarStream: () => { + return ipcRenderer.invoke("microphone-sidecar-stream-open"); + }, + writeMicrophoneSidecarStreamChunk: ( + streamId: string, + position: number, + chunk: Uint8Array, + ) => { + return ipcRenderer.invoke("microphone-sidecar-stream-write", streamId, position, chunk); + }, + closeMicrophoneSidecarStream: ( + streamId: string, + videoPath: string, + options?: { + abort?: boolean; + startDelayMs?: number; + browserMicrophoneProfile?: string; + requestedBrowserMicrophoneProfile?: string | null; + requestedConstraints?: unknown; + mediaTrackSettings?: Record; + audioInputDevices?: Array<{ + deviceId: string; + groupId?: string; + label: string; + }>; + mediaRecorder?: { + mimeType?: string; + audioBitsPerSecond?: number; + timesliceMs?: number; + }; + chunkEvents?: Array<{ + index: number; + size: number; + elapsedMs: number; + deltaMs: number | null; + recordedElapsedMs?: number; + recordedDeltaMs?: number | null; + }>; + pauseIntervals?: Array<{ + startElapsedMs: number; + endElapsedMs?: number; + durationMs?: number; + }>; + }, + ) => { + return ipcRenderer.invoke( + "microphone-sidecar-stream-close", + streamId, + videoPath, + options, + ); + }, storeMicrophoneSidecar: ( audioData: ArrayBuffer, videoPath: string, diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index c64e6b425..55895d706 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -206,6 +206,7 @@ type PixiRendererAttempt = { message: string; }; const PIXI_RENDERER_INIT_TIMEOUT_MS = 8_000; +const REACT_PLAYBACK_TIME_UPDATE_INTERVAL_MS = 80; function isCanvasRenderer(application: Application): boolean { const rendererName = application?.renderer?.constructor?.name?.toLowerCase(); @@ -263,21 +264,46 @@ function getCursorPositionAtTime( return null; } - let closest = telemetry[0]; - let minDist = Math.abs(telemetry[0].timeMs - timeMs); + if (timeMs <= telemetry[0].timeMs) { + const first = telemetry[0]; + return mapCursorToCanvasNormalized( + { + cx: first.cx, + cy: first.cy, + interactionType: first.interactionType, + }, + params ?? { canvasWidth: 1, canvasHeight: 1 }, + ); + } - for (let index = 1; index < telemetry.length; index++) { - const point = telemetry[index]; - const distance = Math.abs(point.timeMs - timeMs); - if (distance < minDist) { - minDist = distance; - closest = point; - } - if (point.timeMs > timeMs) { - break; + if (timeMs >= telemetry[telemetry.length - 1].timeMs) { + const last = telemetry[telemetry.length - 1]; + return mapCursorToCanvasNormalized( + { + cx: last.cx, + cy: last.cy, + interactionType: last.interactionType, + }, + params ?? { canvasWidth: 1, canvasHeight: 1 }, + ); + } + + let lo = 0; + let hi = telemetry.length - 1; + while (lo < hi - 1) { + const mid = (lo + hi) >> 1; + if (telemetry[mid].timeMs <= timeMs) { + lo = mid; + } else { + hi = mid; } } + const before = telemetry[lo]; + const after = telemetry[hi]; + const closest = + Math.abs(before.timeMs - timeMs) <= Math.abs(after.timeMs - timeMs) ? before : after; + return mapCursorToCanvasNormalized( { cx: closest.cx, @@ -1314,12 +1340,24 @@ const VideoPlayback = forwardRef( const bgVideo = bgVideoRef.current; if (bgVideo) { if (isPlaying) { - bgVideo.play().catch(() => undefined); + if (bgVideo.paused) { + bgVideo.play().catch(() => undefined); + } } else { bgVideo.pause(); } } - }, [isPlaying]); + const webcamVideo = webcamVideoRef.current; + if (webcamVideo) { + if (isPlaying && webcamEnabled && webcamVideoPath) { + if (webcamVideo.paused) { + webcamVideo.play().catch(() => undefined); + } + } else { + webcamVideo.pause(); + } + } + }, [isPlaying, webcamEnabled, webcamVideoPath]); useEffect(() => { suspendRenderingRef.current = suspendRendering; @@ -1413,15 +1451,6 @@ const VideoPlayback = forwardRef( } } - if (isPlaying) { - const playPromise = bgVideo.play(); - if (playPromise) { - playPromise.catch(() => undefined); - } - } else { - bgVideo.pause(); - } - lastBackgroundSyncTimeRef.current = clipTimelineTime; }, [currentTime, isPlaying]); @@ -1738,15 +1767,6 @@ const VideoPlayback = forwardRef( } } - if (isPlaying) { - const playPromise = webcamVideo.play(); - if (playPromise) { - playPromise.catch(() => undefined); - } - } else { - webcamVideo.pause(); - } - lastWebcamSyncTimeRef.current = targetTime; }, [currentTime, isPlaying, webcamEnabled, webcamTimeOffsetMs, webcamVideoPath]); @@ -1996,6 +2016,7 @@ const VideoPlayback = forwardRef( timeUpdateAnimationRef, onPlayStateChange, onTimeUpdate, + onTimeUpdateMinIntervalMs: REACT_PLAYBACK_TIME_UPDATE_INTERVAL_MS, trimRegionsRef, speedRegionsRef, }); diff --git a/src/components/video-editor/captionLayout.ts b/src/components/video-editor/captionLayout.ts index 6c0d987e9..9a08eca53 100644 --- a/src/components/video-editor/captionLayout.ts +++ b/src/components/video-editor/captionLayout.ts @@ -48,6 +48,13 @@ export interface ActiveCaptionLayout { scale: number; } +interface CaptionStaticLayout { + sourceWords: ReturnType; + lines: CaptionLineLayout[]; + pages: CaptionPageLayout[]; + maxRows: number; +} + type CaptionSourceWord = { cueId: string; cueWordIndex: number; @@ -61,6 +68,7 @@ type CaptionSourceWord = { const CAPTION_ENTER_MS = 180; const CAPTION_EXIT_MS = 140; const CAPTION_BLOCK_GAP_BREAK_MS = 500; +const captionStaticLayoutCache = new WeakMap>(); function clamp(value: number, min: number, max: number) { return Math.min(max, Math.max(min, value)); @@ -405,31 +413,83 @@ function getVisibleCaptionText(lines: CaptionLineLayout[]) { .join(" "); } -export function buildActiveCaptionLayout(options: { +function getCaptionLayoutCacheKey(settings: AutoCaptionSettings, maxWidthPx: number) { + return [ + Math.round(maxWidthPx * 100) / 100, + clamp(Math.round(settings.maxRows || 1), 1, 4), + ].join(":"); +} + +function getCaptionWordState(index: number, activeWordIndex: number): CaptionWordState { + return index < activeWordIndex + ? "spoken" + : index === activeWordIndex + ? "active" + : "upcoming"; +} + +function applyCaptionWordStates( + lines: CaptionLineLayout[], + activeWordIndex: number, +): CaptionLineLayout[] { + return lines.map((line) => ({ + ...line, + words: line.words.map((word) => ({ + ...word, + state: getCaptionWordState(word.index, activeWordIndex), + })), + })); +} + +function findActiveCaptionWordIndex( + sourceWords: CaptionStaticLayout["sourceWords"], + timeMs: number, +) { + if (sourceWords.length === 0) { + return -1; + } + + let lo = 0; + let hi = sourceWords.length - 1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + const word = sourceWords[mid]; + if (timeMs < word.startMs) { + hi = mid - 1; + } else if (timeMs >= word.endMs) { + lo = mid + 1; + } else { + return mid; + } + } + + return clamp(hi, 0, sourceWords.length - 1); +} + +function getOrBuildCaptionStaticLayout(options: { cues: CaptionCue[]; - timeMs: number; settings: AutoCaptionSettings; maxWidthPx: number; measureText: (text: string) => number; -}) { +}): CaptionStaticLayout | null { + let cueCache = captionStaticLayoutCache.get(options.cues); + if (!cueCache) { + cueCache = new Map(); + captionStaticLayoutCache.set(options.cues, cueCache); + } + + const cacheKey = getCaptionLayoutCacheKey(options.settings, options.maxWidthPx); + const cached = cueCache.get(cacheKey); + if (cached) { + return cached; + } + const sourceWords = flattenCaptionWords(options.cues); if (sourceWords.length === 0) { return null; } - let activeWordIndex = -1; - activeWordIndex = sourceWords.findIndex( - (word) => options.timeMs >= word.startMs && options.timeMs < word.endMs, - ); - if (activeWordIndex < 0) { - activeWordIndex = sourceWords.findIndex((word) => options.timeMs < word.startMs); - activeWordIndex = - activeWordIndex < 0 - ? sourceWords.length - 1 - : clamp(activeWordIndex - 1, 0, sourceWords.length - 1); - } const maxRows = clamp(Math.round(options.settings.maxRows || 1), 1, 4); - const words: CaptionWordLayout[] = sourceWords.map((word, index) => { return { cueId: word.cueId, @@ -441,12 +501,7 @@ export function buildActiveCaptionLayout(options: { startMs: word.startMs, endMs: word.endMs, hasRealTiming: word.hasRealTiming, - state: - index < activeWordIndex - ? "spoken" - : index === activeWordIndex - ? "active" - : "upcoming", + state: "upcoming", }; }); @@ -468,13 +523,39 @@ export function buildActiveCaptionLayout(options: { text: "", }, }); + + const layout = { sourceWords, lines, pages, maxRows }; + cueCache.set(cacheKey, layout); + return layout; +} + +export function buildActiveCaptionLayout(options: { + cues: CaptionCue[]; + timeMs: number; + settings: AutoCaptionSettings; + maxWidthPx: number; + measureText: (text: string) => number; +}) { + const staticLayout = getOrBuildCaptionStaticLayout(options); + if (!staticLayout) { + return null; + } + + const { sourceWords, lines, pages, maxRows } = staticLayout; + const activeWordIndex = findActiveCaptionWordIndex(sourceWords, options.timeMs); const visiblePageIndex = getVisibleCaptionPageIndex(pages, options.timeMs); if (visiblePageIndex < 0) { return null; } const visiblePage = pages[visiblePageIndex] ?? null; - const visibleLines = visiblePage?.lines ?? lines.slice(0, maxRows); - const activeWord = activeWordIndex >= 0 ? words[activeWordIndex] : null; + const visibleLines = applyCaptionWordStates( + visiblePage?.lines ?? lines.slice(0, maxRows), + activeWordIndex, + ); + const activeWord = + activeWordIndex >= 0 + ? visibleLines.flatMap((line) => line.words).find((word) => word.index === activeWordIndex) + : null; const activeWordProgress = activeWord ? clamp01( (options.timeMs - activeWord.startMs) / diff --git a/src/components/video-editor/videoPlayback/cursorRenderer.ts b/src/components/video-editor/videoPlayback/cursorRenderer.ts index 644cf8992..a7926e14b 100644 --- a/src/components/video-editor/videoPlayback/cursorRenderer.ts +++ b/src/components/video-editor/videoPlayback/cursorRenderer.ts @@ -682,11 +682,28 @@ function findLatestSample(samples: CursorTelemetryPoint[], timeMs: number) { } function findLatestInteractionSample(samples: CursorTelemetryPoint[], timeMs: number) { - for (let index = samples.length - 1; index >= 0; index -= 1) { + if (samples.length === 0) return null; + + const oldestRelevantTimeMs = timeMs - CLICK_RING_FADE_MS; + let lo = 0; + let hi = samples.length - 1; + while (lo < hi) { + const mid = Math.ceil((lo + hi) / 2); + if (samples[mid].timeMs <= timeMs) { + lo = mid; + } else { + hi = mid - 1; + } + } + + for (let index = lo; index >= 0; index -= 1) { const sample = samples[index]; if (sample.timeMs > timeMs) { continue; } + if (sample.timeMs < oldestRelevantTimeMs) { + break; + } if ( sample.interactionType === "click" || diff --git a/src/components/video-editor/videoPlayback/videoEventHandlers.ts b/src/components/video-editor/videoPlayback/videoEventHandlers.ts index b85bf00d6..b46a8fae5 100644 --- a/src/components/video-editor/videoPlayback/videoEventHandlers.ts +++ b/src/components/video-editor/videoPlayback/videoEventHandlers.ts @@ -23,6 +23,7 @@ interface VideoEventHandlersParams { timeUpdateAnimationRef: React.MutableRefObject; onPlayStateChange: (playing: boolean) => void; onTimeUpdate: (time: number) => void; + onTimeUpdateMinIntervalMs?: number; trimRegionsRef: React.MutableRefObject; speedRegionsRef: React.MutableRefObject; } @@ -37,17 +38,34 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) { timeUpdateAnimationRef, onPlayStateChange, onTimeUpdate, + onTimeUpdateMinIntervalMs = 0, trimRegionsRef, speedRegionsRef, } = params; const presentedFrameVideo = video as PresentedFrameVideoElement; let videoFrameRequestId: number | null = null; + let lastReactTimeUpdateAtMs = 0; + let lastReactTimeUpdateValue = Number.NaN; enablePitchPreservingPlayback(video); - const emitTime = (timeValue: number) => { + const emitTime = (timeValue: number, options?: { forceReactUpdate?: boolean }) => { currentTimeRef.current = timeValue * 1000; - onTimeUpdate(timeValue); extensionHost.emitEvent({ type: "playback:timeupdate", timeMs: timeValue * 1000 }); + + const now = performance.now(); + const timeJumped = + Number.isFinite(lastReactTimeUpdateValue) && + Math.abs(timeValue - lastReactTimeUpdateValue) > 0.25; + if ( + options?.forceReactUpdate || + onTimeUpdateMinIntervalMs <= 0 || + now - lastReactTimeUpdateAtMs >= onTimeUpdateMinIntervalMs || + timeJumped + ) { + lastReactTimeUpdateAtMs = now; + lastReactTimeUpdateValue = timeValue; + onTimeUpdate(timeValue); + } }; // Helper function to check if current time is within a trim region @@ -74,7 +92,7 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) { const clampedSkipToTime = Math.min(skipToTime, video.duration); video.currentTime = clampedSkipToTime; - emitTime(clampedSkipToTime); + emitTime(clampedSkipToTime, { forceReactUpdate: true }); if (clampedSkipToTime >= video.duration) { video.pause(); @@ -161,7 +179,7 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) { isPlayingRef.current = false; onPlayStateChange(false); cancelScheduledUpdate(); - emitTime(video.currentTime); + emitTime(video.currentTime, { forceReactUpdate: true }); }; const handleSeeked = () => { @@ -174,13 +192,13 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) { if (activeTrimRegion) { skipPastTrimRegion(activeTrimRegion); } else { - emitTime(video.currentTime); + emitTime(video.currentTime, { forceReactUpdate: true }); } }; const handleSeeking = () => { isSeekingRef.current = true; - emitTime(video.currentTime); + emitTime(video.currentTime, { forceReactUpdate: true }); }; return { diff --git a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts b/src/components/video-editor/videoPlayback/zoomRegionUtils.ts index 4566455dc..4bdef5690 100644 --- a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts +++ b/src/components/video-editor/videoPlayback/zoomRegionUtils.ts @@ -34,6 +34,14 @@ type ConnectedPanTransition = { endScale: number; }; +type ConnectedRegionCache = { + pairs: ConnectedRegionPair[]; + outgoingByRegionId: Map; + incomingByRegionId: Map; +}; + +const connectedRegionCache = new WeakMap(); + function lerp(start: number, end: number, amount: number) { return start + (end - start) * amount; } @@ -115,50 +123,74 @@ function getConnectedRegionPairs(regions: ZoomRegion[]) { return pairs; } +function getConnectedRegionCache(regions: ZoomRegion[]): ConnectedRegionCache { + const cached = connectedRegionCache.get(regions); + if (cached) { + return cached; + } + + const pairs = getConnectedRegionPairs(regions); + const outgoingByRegionId = new Map(); + const incomingByRegionId = new Map(); + + for (const pair of pairs) { + outgoingByRegionId.set(pair.currentRegion.id, pair); + incomingByRegionId.set(pair.nextRegion.id, pair); + } + + const cache = { pairs, outgoingByRegionId, incomingByRegionId }; + connectedRegionCache.set(regions, cache); + return cache; +} + function getActiveRegion( regions: ZoomRegion[], timeMs: number, - connectedPairs: ConnectedRegionPair[], + connectedCache: ConnectedRegionCache, options: DominantRegionOptions, ) { - const activeRegions = regions - .map((region) => { - const outgoingPair = connectedPairs.find((pair) => pair.currentRegion.id === region.id); - if (outgoingPair && timeMs >= outgoingPair.transitionStart) { - return { region, strength: 0 }; - } - - const incomingPair = connectedPairs.find((pair) => pair.nextRegion.id === region.id); + let activeRegion: ZoomRegion | null = null; + let activeStrength = 0; + + for (const region of regions) { + let strength = 0; + const outgoingPair = connectedCache.outgoingByRegionId.get(region.id); + if (outgoingPair && timeMs >= outgoingPair.transitionStart) { + strength = 0; + } else { + const incomingPair = connectedCache.incomingByRegionId.get(region.id); if (incomingPair) { if (timeMs < incomingPair.transitionStart) { - return { region, strength: 0 }; + strength = 0; + } else { + const nextRegionZoomOutStart = + incomingPair.nextRegion.endMs - + ZOOM_OUT_EARLY_START_MS + + ZOOM_ANIMATION_LEAD_MS; + strength = timeMs < nextRegionZoomOutStart + ? 1 + : computeRegionStrength(region, timeMs, options); } - - const nextRegionZoomOutStart = - incomingPair.nextRegion.endMs - - ZOOM_OUT_EARLY_START_MS + - ZOOM_ANIMATION_LEAD_MS; - if (timeMs < nextRegionZoomOutStart) { - return { region, strength: 1 }; - } - } - - return { region, strength: computeRegionStrength(region, timeMs, options) }; - }) - .filter((entry) => entry.strength > 0) - .sort((left, right) => { - if (right.strength !== left.strength) { - return right.strength - left.strength; + } else { + strength = computeRegionStrength(region, timeMs, options); } + } - return right.region.startMs - left.region.startMs; - }); + if ( + strength > 0 && + (!activeRegion || + strength > activeStrength || + (strength === activeStrength && region.startMs > activeRegion.startMs)) + ) { + activeRegion = region; + activeStrength = strength; + } + } - if (activeRegions.length === 0) { + if (!activeRegion) { return null; } - const activeRegion = activeRegions[0].region; const activeScale = ZOOM_DEPTH_SCALES[activeRegion.depth]; return { @@ -166,7 +198,7 @@ function getActiveRegion( ...activeRegion, focus: getResolvedFocus(activeRegion, activeScale), }, - strength: activeRegions[0].strength, + strength: activeStrength, blendedScale: null, }; } @@ -237,21 +269,23 @@ export function findDominantRegion( blendedScale: number | null; transition: ConnectedPanTransition | null; } { - const connectedPairs = options.connectZooms ? getConnectedRegionPairs(regions) : []; + const connectedCache = options.connectZooms + ? getConnectedRegionCache(regions) + : { pairs: [], outgoingByRegionId: new Map(), incomingByRegionId: new Map() }; if (options.connectZooms) { - const connectedTransition = getConnectedRegionTransition(connectedPairs, timeMs); + const connectedTransition = getConnectedRegionTransition(connectedCache.pairs, timeMs); if (connectedTransition) { return connectedTransition; } - const connectedHold = getConnectedRegionHold(timeMs, connectedPairs); + const connectedHold = getConnectedRegionHold(timeMs, connectedCache.pairs); if (connectedHold) { return { ...connectedHold, transition: null }; } } - const activeRegion = getActiveRegion(regions, timeMs, connectedPairs, options); + const activeRegion = getActiveRegion(regions, timeMs, connectedCache, options); return activeRegion ? { ...activeRegion, transition: null } : { region: null, strength: 0, blendedScale: null, transition: null }; diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 6f021761a..f3a99bb25 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -107,6 +107,150 @@ type MicrophoneSidecarOptions = { chunkEvents?: MicrophoneFallbackChunkEvent[]; pauseIntervals?: MicrophoneFallbackPauseInterval[]; }; + +type ChunkStreamWriter = { + readonly path?: string; + write: (blob: Blob) => void; + close: (options?: { + mimeType?: string; + }) => Promise<{ success: boolean; path?: string; message?: string; error?: string }>; + abort: () => Promise; +}; + +type MicrophoneSidecarStreamWriter = { + write: (blob: Blob) => void; + close: ( + videoPath: string, + options?: MicrophoneSidecarOptions, + ) => Promise<{ success: boolean; path?: string; error?: string }>; + abort: () => Promise; +}; + +function createQueuedBlobWriter(params: { + streamId: string; + writeChunk: ( + streamId: string, + position: number, + chunk: Uint8Array, + ) => Promise<{ success: boolean; error?: string }>; +}) { + let position = 0; + let writeQueue = Promise.resolve(); + let writeError: Error | null = null; + + const write = (blob: Blob) => { + if (blob.size <= 0 || writeError) { + return; + } + + writeQueue = writeQueue + .then(async () => { + const buffer = await blob.arrayBuffer(); + const bytes = new Uint8Array(buffer); + const writePosition = position; + position += bytes.byteLength; + const result = await params.writeChunk(params.streamId, writePosition, bytes); + if (!result.success) { + throw new Error(result.error || "Failed to write recording chunk"); + } + }) + .catch((error) => { + writeError = error instanceof Error ? error : new Error(String(error)); + }); + }; + + const waitForWrites = async () => { + await writeQueue; + if (writeError) { + throw writeError; + } + }; + + return { write, waitForWrites }; +} + +async function createRecordingStreamWriter(fileName: string): Promise { + if ( + !window.electronAPI?.openRecordingStream || + !window.electronAPI?.writeRecordingStreamChunk || + !window.electronAPI?.closeRecordingStream + ) { + return null; + } + + const openResult = await window.electronAPI.openRecordingStream(fileName); + if (!openResult.success || !openResult.streamId) { + console.warn("Recording stream unavailable:", openResult.error); + return null; + } + + const writer = createQueuedBlobWriter({ + streamId: openResult.streamId, + writeChunk: window.electronAPI.writeRecordingStreamChunk, + }); + + return { + path: openResult.path, + write: writer.write, + close: async (options) => { + try { + await writer.waitForWrites(); + return window.electronAPI.closeRecordingStream(openResult.streamId!, options); + } catch (error) { + await window.electronAPI.closeRecordingStream(openResult.streamId!, { abort: true }); + throw error; + } + }, + abort: async () => { + await window.electronAPI.closeRecordingStream(openResult.streamId!, { abort: true }); + }, + }; +} + +async function createMicrophoneSidecarStreamWriter(): Promise { + if ( + !window.electronAPI?.openMicrophoneSidecarStream || + !window.electronAPI?.writeMicrophoneSidecarStreamChunk || + !window.electronAPI?.closeMicrophoneSidecarStream + ) { + return null; + } + + const openResult = await window.electronAPI.openMicrophoneSidecarStream(); + if (!openResult.success || !openResult.streamId) { + console.warn("Microphone sidecar stream unavailable:", openResult.error); + return null; + } + + const writer = createQueuedBlobWriter({ + streamId: openResult.streamId, + writeChunk: window.electronAPI.writeMicrophoneSidecarStreamChunk, + }); + + return { + write: writer.write, + close: async (videoPath, options) => { + try { + await writer.waitForWrites(); + return window.electronAPI.closeMicrophoneSidecarStream( + openResult.streamId!, + videoPath, + options, + ); + } catch (error) { + await window.electronAPI.closeMicrophoneSidecarStream(openResult.streamId!, "", { + abort: true, + }); + throw error; + } + }, + abort: async () => { + await window.electronAPI.closeMicrophoneSidecarStream(openResult.streamId!, "", { + abort: true, + }); + }, + }; +} const LINUX_PORTAL_SOURCE: ProcessedDesktopSource = { id: "screen:linux-portal", name: "Linux Portal", @@ -280,6 +424,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const mixingContext = useRef(null); const chunks = useRef([]); const webcamChunks = useRef([]); + const recordingStreamWriter = useRef(null); + const webcamStreamWriter = useRef(null); const startTime = useRef(0); const webcamStartTime = useRef(null); const webcamTimeOffsetMs = useRef(0); @@ -299,6 +445,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const pauseStartedAtMs = useRef(null); const micFallbackRecorder = useRef(null); const micFallbackChunks = useRef([]); + const micFallbackStreamWriter = useRef(null); const micFallbackStartDelayMs = useRef(null); const micFallbackTrackSettings = useRef(null); const micFallbackRequestedConstraints = useRef(null); @@ -314,12 +461,35 @@ export function useScreenRecorder(): UseScreenRecorderReturn { ); const requestedBrowserMicrophoneProfile = useRef(null); const hideEditorOverlayCursorByDefault = useRef(false); + const discardRecording = useRef(false); const notifyRecordingFinalizationFailure = useCallback(async (message: string) => { setFinalizing(false); toast.error(message, { duration: 10000 }); }, []); + const abortOpenChunkStreams = useCallback(() => { + const abortWriter = ( + writer: { abort: () => Promise } | null, + label: string, + ) => { + if (!writer) { + return; + } + + void writer.abort().catch((error) => { + console.warn(`Failed to abort ${label} stream:`, error); + }); + }; + + abortWriter(recordingStreamWriter.current, "recording"); + abortWriter(webcamStreamWriter.current, "webcam"); + abortWriter(micFallbackStreamWriter.current, "microphone sidecar"); + recordingStreamWriter.current = null; + webcamStreamWriter.current = null; + micFallbackStreamWriter.current = null; + }, []); + const logNativeCaptureDiagnostics = useCallback(async (context: string) => { if (typeof window.electronAPI?.getLastNativeCaptureDiagnostics !== "function") { return; @@ -476,10 +646,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return selectWebcamRecordingMimeType(); }, []); - const computeBitrate = (width: number, height: number) => { + const computeBitrate = (width: number, height: number, frameRate = TARGET_FRAME_RATE) => { const pixels = width * height; + const normalizedFrameRate = Number.isFinite(frameRate) ? frameRate : TARGET_FRAME_RATE; const highFrameRateBoost = - TARGET_FRAME_RATE >= HIGH_FRAME_RATE_THRESHOLD ? HIGH_FRAME_RATE_BOOST : 1; + normalizedFrameRate >= HIGH_FRAME_RATE_THRESHOLD + ? Math.min(HIGH_FRAME_RATE_BOOST, normalizedFrameRate / MIN_FRAME_RATE) + : 1; if (pixels >= FOUR_K_PIXELS) { return Math.round(BITRATE_4K * highFrameRateBoost); @@ -543,7 +716,12 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return; } - micFallbackChunks.current.push(event.data); + const streamWriter = micFallbackStreamWriter.current; + if (streamWriter) { + streamWriter.write(event.data); + } else { + micFallbackChunks.current.push(event.data); + } const startedAt = micFallbackRecorderStartedAt.current; if (startedAt === null) { return; @@ -688,8 +866,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { closeMicFallbackPauseInterval(); recorder.ondataavailable = appendMicFallbackChunk; recorder.onstop = () => { - const blob = - micFallbackChunks.current.length > 0 + const blob = micFallbackStreamWriter.current + ? null + : micFallbackChunks.current.length > 0 ? new Blob(micFallbackChunks.current, { type: recorder.mimeType }) : null; micFallbackChunks.current = []; @@ -741,55 +920,66 @@ export function useScreenRecorder(): UseScreenRecorderReturn { mediaTrackSettings?: MicrophoneTrackSettingsSnapshot | null, ) => { const micFallbackBlob = await micFallbackBlobPromise; - if (!micFallbackBlob) { - micFallbackStartDelayMs.current = null; - micFallbackTrackSettings.current = null; - micFallbackRequestedConstraints.current = null; - micFallbackAudioInputDevices.current = null; - micFallbackRecorderMetadata.current = null; - resetMicFallbackTimingDiagnostics(); - return; - } + const effectiveStartDelayMs = startDelayMs ?? micFallbackStartDelayMs.current; + const effectiveTrackSettings = + mediaTrackSettings ?? micFallbackTrackSettings.current; + const sidecarOptions: MicrophoneSidecarOptions = { + ...(Number.isFinite(effectiveStartDelayMs) && (effectiveStartDelayMs ?? 0) >= 0 + ? { startDelayMs: effectiveStartDelayMs ?? 0 } + : {}), + browserMicrophoneProfile: browserMicrophoneProfile.current, + ...(requestedBrowserMicrophoneProfile.current + ? { + requestedBrowserMicrophoneProfile: + requestedBrowserMicrophoneProfile.current, + } + : {}), + ...(micFallbackRequestedConstraints.current + ? { requestedConstraints: micFallbackRequestedConstraints.current } + : {}), + ...(effectiveTrackSettings + ? { mediaTrackSettings: effectiveTrackSettings } + : {}), + ...(micFallbackAudioInputDevices.current + ? { audioInputDevices: micFallbackAudioInputDevices.current } + : {}), + ...(micFallbackRecorderMetadata.current + ? { mediaRecorder: micFallbackRecorderMetadata.current } + : {}), + ...(micFallbackChunkEvents.current.length > 0 + ? { chunkEvents: [...micFallbackChunkEvents.current] } + : {}), + ...(micFallbackPauseIntervals.current.length > 0 + ? { + pauseIntervals: micFallbackPauseIntervals.current.map((interval) => ({ + ...interval, + })), + } + : {}), + }; try { + const streamWriter = micFallbackStreamWriter.current; + if (streamWriter) { + micFallbackStreamWriter.current = null; + const result = await streamWriter.close(finalPath, sidecarOptions); + if (!result.success) { + const errorMessage = + result.error || "Failed to save the fallback microphone audio track"; + console.warn("Failed to store microphone sidecar:", errorMessage); + toast.error( + `${errorMessage}. Recording was saved without the fallback microphone track.`, + { id: MICROPHONE_SIDECAR_ERROR_TOAST_ID, duration: 10000 }, + ); + } + return; + } + + if (!micFallbackBlob) { + return; + } + const arrayBuffer = await micFallbackBlob.arrayBuffer(); - const effectiveStartDelayMs = startDelayMs ?? micFallbackStartDelayMs.current; - const effectiveTrackSettings = - mediaTrackSettings ?? micFallbackTrackSettings.current; - const sidecarOptions: MicrophoneSidecarOptions = { - ...(Number.isFinite(effectiveStartDelayMs) && (effectiveStartDelayMs ?? 0) >= 0 - ? { startDelayMs: effectiveStartDelayMs ?? 0 } - : {}), - browserMicrophoneProfile: browserMicrophoneProfile.current, - ...(requestedBrowserMicrophoneProfile.current - ? { - requestedBrowserMicrophoneProfile: - requestedBrowserMicrophoneProfile.current, - } - : {}), - ...(micFallbackRequestedConstraints.current - ? { requestedConstraints: micFallbackRequestedConstraints.current } - : {}), - ...(effectiveTrackSettings - ? { mediaTrackSettings: effectiveTrackSettings } - : {}), - ...(micFallbackAudioInputDevices.current - ? { audioInputDevices: micFallbackAudioInputDevices.current } - : {}), - ...(micFallbackRecorderMetadata.current - ? { mediaRecorder: micFallbackRecorderMetadata.current } - : {}), - ...(micFallbackChunkEvents.current.length > 0 - ? { chunkEvents: [...micFallbackChunkEvents.current] } - : {}), - ...(micFallbackPauseIntervals.current.length > 0 - ? { - pauseIntervals: micFallbackPauseIntervals.current.map( - (interval) => ({ ...interval }), - ), - } - : {}), - }; const result = await window.electronAPI.storeMicrophoneSidecar( arrayBuffer, finalPath, @@ -914,7 +1104,15 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }); const mimeType = selectWebcamMimeType(); + const sessionTimestamp = recordingSessionTimestamp.current ?? Date.now(); + const webcamFileName = `${RECORDING_FILE_PREFIX}${sessionTimestamp}${WEBCAM_SUFFIX}${getVideoExtensionForMimeType(mimeType)}`; webcamChunks.current = []; + webcamStreamWriter.current = await createRecordingStreamWriter(webcamFileName).catch( + (error) => { + console.warn("Webcam recording stream unavailable:", error); + return null; + }, + ); resolvedWebcamPath.current = null; webcamStopPromise.current = new Promise((resolve) => { webcamStopResolver.current = resolve; @@ -928,7 +1126,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn { webcamRecorder.current = recorder; recorder.ondataavailable = (event) => { - if (event.data && event.data.size > 0) { + if (!event.data || event.data.size <= 0 || discardRecording.current) { + return; + } + + if (webcamStreamWriter.current) { + webcamStreamWriter.current.write(event.data); + } else { webcamChunks.current.push(event.data); } }; @@ -937,12 +1141,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn { webcamStopResolver.current = null; }; recorder.onstop = async () => { - const sessionTimestamp = recordingSessionTimestamp.current ?? Date.now(); const webcamMimeType = recorder.mimeType || mimeType; - const webcamFileName = `${RECORDING_FILE_PREFIX}${sessionTimestamp}${WEBCAM_SUFFIX}${getVideoExtensionForMimeType(webcamMimeType)}`; + const streamWriter = webcamStreamWriter.current; + webcamStreamWriter.current = null; try { - if (webcamChunks.current.length === 0) { + if (discardRecording.current) { + webcamChunks.current = []; webcamStopResolver.current?.(null); return; } @@ -951,6 +1156,26 @@ export function useScreenRecorder(): UseScreenRecorderReturn { 0, getRecordingDurationMs(Date.now()) - webcamTimeOffsetMs.current, ); + + if (streamWriter) { + const result = await streamWriter.close({ + mimeType: webcamMimeType, + }); + if (!result.success) { + console.warn( + "Failed to store webcam recording:", + result.error ?? result.message, + ); + } + webcamStopResolver.current?.(result.success ? (result.path ?? null) : null); + return; + } + + if (webcamChunks.current.length === 0) { + webcamStopResolver.current?.(null); + return; + } + const webcamBlob = new Blob( webcamChunks.current, webcamMimeType ? { type: webcamMimeType } : undefined, @@ -979,6 +1204,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } }; } catch (error) { + const streamWriter = webcamStreamWriter.current; + webcamStreamWriter.current = null; + void streamWriter?.abort().catch(() => undefined); console.warn( "Failed to start webcam recording; continuing without webcam layer:", error, @@ -1270,6 +1498,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { cleanup?.(); removeRecordingStateListener?.(); removeRecordingInterruptedListener?.(); + discardRecording.current = true; + abortOpenChunkStreams(); if (nativeScreenRecording.current) { nativeScreenRecording.current = false; @@ -1284,7 +1514,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { cleanupCapturedMedia(); }; - }, [cleanupCapturedMedia, recoverNativeRecordingSession]); + }, [abortOpenChunkStreams, cleanupCapturedMedia, recoverNativeRecordingSession]); const startRecording = async () => { if (startInFlight.current) { @@ -1292,6 +1522,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } hasPromptedForReselect.current = false; + discardRecording.current = false; startInFlight.current = true; setStarting(true); @@ -1449,6 +1680,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { micFallbackAudioInputDevices.current, ); micFallbackChunks.current = []; + micFallbackStreamWriter.current = + await createMicrophoneSidecarStreamWriter().catch((error) => { + console.warn("Microphone sidecar stream unavailable:", error); + return null; + }); const recorder = new MediaRecorder(micStream, { mimeType: "audio/webm;codecs=opus", audioBitsPerSecond: AUDIO_BITRATE_VOICE, @@ -1468,6 +1704,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { recorder.start(RECORDER_TIMESLICE_MS); micFallbackRecorder.current = recorder; } catch (micError) { + const streamWriter = micFallbackStreamWriter.current; + micFallbackStreamWriter.current = null; + void streamWriter?.abort().catch(() => undefined); micFallbackStartDelayMs.current = null; micFallbackTrackSettings.current = null; micFallbackRequestedConstraints.current = null; @@ -1693,7 +1932,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { width = Math.floor(width / CODEC_ALIGNMENT) * CODEC_ALIGNMENT; height = Math.floor(height / CODEC_ALIGNMENT) * CODEC_ALIGNMENT; - const videoBitsPerSecond = computeBitrate(width, height); + const videoBitsPerSecond = computeBitrate( + width, + height, + frameRate ?? TARGET_FRAME_RATE, + ); const mimeType = selectMimeType(); console.log( @@ -1703,6 +1946,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { ); chunks.current = []; + const timestamp = recordingSessionTimestamp.current ?? Date.now(); + const videoFileName = `${RECORDING_FILE_PREFIX}${timestamp}${VIDEO_FILE_EXTENSION}`; + recordingStreamWriter.current = await createRecordingStreamWriter(videoFileName).catch( + (error) => { + console.warn("Recording stream unavailable:", error); + return null; + }, + ); const hasAudio = stream.current.getAudioTracks().length > 0; const recorder = new MediaRecorder(stream.current, { videoBitsPerSecond, @@ -1718,43 +1969,30 @@ export function useScreenRecorder(): UseScreenRecorderReturn { mediaRecorder.current = recorder; recorder.ondataavailable = (event) => { - if (event.data && event.data.size > 0) chunks.current.push(event.data); + if (!event.data || event.data.size <= 0 || discardRecording.current) { + return; + } + + if (recordingStreamWriter.current) { + recordingStreamWriter.current.write(event.data); + } else { + chunks.current.push(event.data); + } }; recorder.onstop = async () => { cleanupCapturedMedia(); - if (chunks.current.length === 0) { + const streamWriter = recordingStreamWriter.current; + recordingStreamWriter.current = null; + + if (discardRecording.current) { + chunks.current = []; + discardRecording.current = false; setFinalizing(false); return; } - const duration = getRecordingDurationMs(Date.now()); - const recordedChunks = chunks.current; - const recordingBlobType = recorder.mimeType || mimeType; - const buggyBlob = new Blob( - recordedChunks, - recordingBlobType ? { type: recordingBlobType } : undefined, - ); - chunks.current = []; - const timestamp = recordingSessionTimestamp.current ?? Date.now(); - const videoFileName = `${RECORDING_FILE_PREFIX}${timestamp}${VIDEO_FILE_EXTENSION}`; - try { - const videoBlob = await fixWebmDuration(buggyBlob, duration); - const arrayBuffer = await videoBlob.arrayBuffer(); - const videoResult = await window.electronAPI.storeRecordedVideo( - arrayBuffer, - videoFileName, - ); - if (!videoResult.success) { - console.error("Failed to store video:", videoResult.message); - await notifyRecordingFinalizationFailure( - videoResult.message || "Failed to store the recording.", - ); - return; - } - - if (videoResult.path) { - const finalVideoPath = videoResult.path; + const finalizeStoredBrowserVideo = async (finalVideoPath: string) => { // 1. Launch editor immediately (Optimistic UI) await finalizeRecordingSession(finalVideoPath, null); @@ -1777,11 +2015,63 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // After all background tasks are done (webcam), // we can safely close the HUD window to release hardware and resources. if (typeof window.electronAPI?.hudOverlayClose === "function") { - console.log("[useScreenRecorder:browser] All background tasks finished, closing HUD"); + console.log( + "[useScreenRecorder:browser] All background tasks finished, closing HUD", + ); window.electronAPI.hudOverlayClose(); } } })(); + }; + const duration = getRecordingDurationMs(Date.now()); + const recordingBlobType = recorder.mimeType || mimeType; + + if (streamWriter) { + const videoResult = await streamWriter.close({ + mimeType: recordingBlobType, + }); + if (!videoResult.success || !videoResult.path) { + console.error( + "Failed to store video:", + videoResult.error ?? videoResult.message, + ); + await notifyRecordingFinalizationFailure( + videoResult.message || videoResult.error || "Failed to store the recording.", + ); + return; + } + + await finalizeStoredBrowserVideo(videoResult.path); + return; + } + + if (chunks.current.length === 0) { + setFinalizing(false); + return; + } + + const recordedChunks = chunks.current; + const buggyBlob = new Blob( + recordedChunks, + recordingBlobType ? { type: recordingBlobType } : undefined, + ); + chunks.current = []; + const videoBlob = await fixWebmDuration(buggyBlob, duration); + const arrayBuffer = await videoBlob.arrayBuffer(); + const videoResult = await window.electronAPI.storeRecordedVideo( + arrayBuffer, + videoFileName, + ); + if (!videoResult.success) { + console.error("Failed to store video:", videoResult.message); + await notifyRecordingFinalizationFailure( + videoResult.message || "Failed to store the recording.", + ); + return; + } + + if (videoResult.path) { + await finalizeStoredBrowserVideo(videoResult.path); } else { await notifyRecordingFinalizationFailure("Failed to save the recording."); } @@ -1805,6 +2095,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { setRecording(true); window.electronAPI?.setRecordingState(true); } catch (error) { + discardRecording.current = true; + abortOpenChunkStreams(); console.error("Failed to start recording:", error); alert( error instanceof Error @@ -1920,6 +2212,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const cancelRecording = useCallback(() => { if (!recording) return; + discardRecording.current = true; + abortOpenChunkStreams(); setPaused(false); markRecordingResumed(Date.now()); @@ -1963,7 +2257,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { setRecording(false); window.electronAPI?.setRecordingState(false); } - }, [cleanupCapturedMedia, markRecordingResumed, recording]); + }, [abortOpenChunkStreams, cleanupCapturedMedia, markRecordingResumed, recording]); const toggleRecording = async () => { if (starting || countdownActive || finalizing) {