diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 32ed313e0..58c97df73 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -64,6 +64,51 @@ interface UpdateStatusSummary { detail?: string; } +type RendererPhoneRemoteSessionStatus = + | "waiting" + | "phone-connected" + | "preview-live" + | "mic-active" + | "reconnecting" + | "disconnected" + | "camera-permission-denied" + | "microphone-permission-denied" + | "no-audio-track" + | "phone-backgrounded" + | "phone-sleeping" + | "error"; + +type RendererPhoneRemoteSignalMessage = + | { + type: "offer" | "answer"; + description: RTCSessionDescriptionInit; + } + | { + type: "ice-candidate"; + candidate: RTCIceCandidateInit | null; + }; + +interface RendererPhoneRemoteStatusMessage { + status: RendererPhoneRemoteSessionStatus; + detail?: string; + hasAudio?: boolean; + hasVideo?: boolean; + facingMode?: "user" | "environment"; +} + +interface RendererPhoneRemoteSession { + id: string; + code: string; + joinUrl: string; + localJoinUrl: string; + lanJoinUrl: string; + tunnelJoinUrl?: string; + urlMode: "secure-tunnel" | "lan"; + tunnelError?: string; + expiresAt: number; + status: RendererPhoneRemoteSessionStatus; +} + type RendererExtensionInfo = import("./extensions/extensionTypes").ExtensionInfo; type RendererExtensionReview = import("./extensions/extensionTypes").ExtensionReview; type RendererMarketplaceExtension = import("./extensions/extensionTypes").MarketplaceExtension; @@ -862,6 +907,28 @@ interface Window { cancelCountdown: () => Promise<{ success: boolean }>; getActiveCountdown: () => Promise<{ success: boolean; seconds: number | null }>; onCountdownTick: (callback: (seconds: number) => void) => () => void; + createPhoneRemoteSession: () => Promise<{ + success: boolean; + session: RendererPhoneRemoteSession; + error?: string; + }>; + endPhoneRemoteSession: (sessionId: string) => Promise<{ success: boolean }>; + sendPhoneRemoteSignal: ( + sessionId: string, + message: RendererPhoneRemoteSignalMessage, + ) => Promise<{ success: boolean; index?: number; error?: string }>; + onPhoneRemoteSignal: ( + callback: (payload: { + sessionId: string; + message: RendererPhoneRemoteSignalMessage; + }) => void, + ) => () => void; + onPhoneRemoteStatus: ( + callback: (payload: { + sessionId: string; + status: RendererPhoneRemoteStatusMessage; + }) => void, + ) => () => void; extensionsDiscover: () => Promise; extensionsList: () => Promise; extensionsGet: (id: string) => Promise; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index f6e4dc029..e6ba62f55 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -3,6 +3,7 @@ import { registerAssetHandlers } from "./register/assets"; import { registerCaptionHandlers } from "./register/captions"; import { registerExportHandlers } from "./register/export"; import { registerPermissionHandlers } from "./register/permissions"; +import { registerPhoneRemoteHandlers } from "./register/phoneRemote"; import { registerProjectHandlers } from "./register/project"; import { registerRecordingHandlers } from "./register/recording"; import { registerSettingsHandlers } from "./register/settings"; @@ -69,4 +70,5 @@ export function registerIpcHandlers( registerCaptionHandlers(); registerProjectHandlers(); registerSettingsHandlers(); + registerPhoneRemoteHandlers(); } diff --git a/electron/ipc/monitorResolver.ts b/electron/ipc/monitorResolver.ts index e71f36c11..f14bbd38c 100644 --- a/electron/ipc/monitorResolver.ts +++ b/electron/ipc/monitorResolver.ts @@ -13,7 +13,7 @@ export interface WinMonitorHandle { /** * Retrieves raw HMONITOR handles from the Windows OS using a PowerShell bridge. - * This is necessary because Electron's display IDs are often internal hashes that + * This is necessary because Electron's display IDs are often internal hashes that * cannot be used directly with native Windows APIs like Graphics Capture (WGC). */ export function getMonitorHandles(): WinMonitorHandle[] { @@ -53,10 +53,14 @@ public class MonitorHelper { [MonitorHelper]::GetMonitors() `.trim(); - const result = spawnSync("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", psScript], { - encoding: "utf-8", - timeout: 5000, - }); + const result = spawnSync( + "powershell.exe", + ["-NoProfile", "-NonInteractive", "-Command", psScript], + { + encoding: "utf-8", + timeout: 5000, + }, + ); if (result.error || result.status !== 0) { // Silent failure is preferred; the caller will fall back to coordinate-based matching. diff --git a/electron/ipc/recording/diagnostics.test.ts b/electron/ipc/recording/diagnostics.test.ts index 21192e912..b93be8a0a 100644 --- a/electron/ipc/recording/diagnostics.test.ts +++ b/electron/ipc/recording/diagnostics.test.ts @@ -381,4 +381,24 @@ describe("getCompanionAudioFallbackPaths", () => { "Recorded output is too small to contain playable video", ); }); + + it("keeps a non-empty recording usable when FFmpeg is missing in a dev install", async () => { + vi.doMock("../ffmpeg/binary", () => ({ + getFfmpegBinaryPath: () => { + throw new Error( + "FFmpeg binary is unavailable. Install ffmpeg-static for this platform or make ffmpeg available on PATH.", + ); + }, + })); + + const videoPath = path.join(tempRoot, "recording-456.mp4"); + await fs.writeFile(videoPath, Buffer.alloc(4096)); + + const { validateRecordedVideo } = await import("./diagnostics"); + + await expect(validateRecordedVideo(videoPath)).resolves.toEqual({ + fileSizeBytes: 4096, + durationSeconds: null, + }); + }); }); diff --git a/electron/ipc/recording/diagnostics.ts b/electron/ipc/recording/diagnostics.ts index 811e0920e..fa049dcb7 100644 --- a/electron/ipc/recording/diagnostics.ts +++ b/electron/ipc/recording/diagnostics.ts @@ -201,9 +201,7 @@ export async function probeMediaDurationSeconds(filePath: string): Promise(await fs.readFile(projectPath, "utf-8")); } catch (error) { - console.warn("[prune] Aborting recording prune because a saved project is unreadable", { - projectPath, - error, - }); + console.warn( + "[prune] Aborting recording prune because a saved project is unreadable", + { + projectPath, + error, + }, + ); throw error; } const candidatePaths = [ diff --git a/electron/ipc/register/phoneRemote.ts b/electron/ipc/register/phoneRemote.ts new file mode 100644 index 000000000..712cf1925 --- /dev/null +++ b/electron/ipc/register/phoneRemote.ts @@ -0,0 +1,150 @@ +import { ipcMain, webContents } from "electron"; +import { getPhoneRemoteJoinUrls } from "../../phoneRemote/server"; +import { + addLaptopSignal, + createPhoneRemoteSession, + endPhoneRemoteSession, + subscribePhoneRemoteStore, +} from "../../phoneRemote/sessionStore"; +import type { PhoneRemoteSignalMessage } from "../../phoneRemote/types"; + +let subscribed = false; + +function sendToOwner(ownerWebContentsId: number, channel: string, payload: unknown) { + const target = webContents.fromId(ownerWebContentsId); + if (!target || target.isDestroyed()) { + return; + } + target.send(channel, payload); +} + +function parseSignalMessage(value: unknown): PhoneRemoteSignalMessage | null { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + + const record = value as Record; + if (record.type === "offer" || record.type === "answer") { + const description = record.description as Record | null; + if ( + !description || + typeof description.type !== "string" || + typeof description.sdp !== "string" + ) { + return null; + } + if (description.type !== "offer" && description.type !== "answer") { + return null; + } + if (description.type !== record.type) { + return null; + } + return { + type: record.type, + description: { + type: description.type, + sdp: description.sdp, + }, + }; + } + + if (record.type === "ice-candidate") { + if (record.candidate === null) { + return { + type: "ice-candidate", + candidate: null, + }; + } + if ( + !record.candidate || + typeof record.candidate !== "object" || + Array.isArray(record.candidate) + ) { + return null; + } + const candidate = record.candidate as Record; + if (typeof candidate.candidate !== "string") { + return null; + } + return { + type: "ice-candidate", + candidate: { + candidate: candidate.candidate, + sdpMid: typeof candidate.sdpMid === "string" ? candidate.sdpMid : null, + sdpMLineIndex: + typeof candidate.sdpMLineIndex === "number" ? candidate.sdpMLineIndex : null, + usernameFragment: + typeof candidate.usernameFragment === "string" + ? candidate.usernameFragment + : undefined, + }, + }; + } + + return null; +} + +export function registerPhoneRemoteHandlers() { + if (subscribed) { + return; + } + + subscribed = true; + subscribePhoneRemoteStore((event) => { + if (event.type === "signal") { + sendToOwner(event.ownerWebContentsId, "phone-remote-signal", { + sessionId: event.sessionId, + message: event.message, + }); + return; + } + + sendToOwner(event.ownerWebContentsId, "phone-remote-status", { + sessionId: event.sessionId, + status: event.status, + }); + }); + + ipcMain.handle("phone-remote:create-session", async (event) => { + const urls = await getPhoneRemoteJoinUrls(); + const session = createPhoneRemoteSession(event.sender.id, { + ...urls, + joinUrl: `${urls.joinUrl}?code=`, + localJoinUrl: `${urls.localJoinUrl}?code=`, + lanJoinUrl: `${urls.lanJoinUrl}?code=`, + tunnelJoinUrl: urls.tunnelJoinUrl ? `${urls.tunnelJoinUrl}?code=` : undefined, + }); + + return { + success: true, + session: { + ...session, + joinUrl: `${session.joinUrl}${encodeURIComponent(session.code)}`, + localJoinUrl: `${session.localJoinUrl}${encodeURIComponent(session.code)}`, + lanJoinUrl: `${session.lanJoinUrl}${encodeURIComponent(session.code)}`, + tunnelJoinUrl: session.tunnelJoinUrl + ? `${session.tunnelJoinUrl}${encodeURIComponent(session.code)}` + : undefined, + }, + }; + }); + + ipcMain.handle("phone-remote:end-session", (_event, sessionId: string) => { + return { success: endPhoneRemoteSession(sessionId) }; + }); + + ipcMain.handle("phone-remote:send-signal", (event, sessionId: string, value: unknown) => { + const message = parseSignalMessage(value); + if (!message) { + return { + success: false, + error: "Invalid phone remote signal payload.", + }; + } + + const envelope = addLaptopSignal(sessionId, event.sender.id, message); + return envelope + ? { success: true, index: envelope.index } + : { success: false, error: "Phone remote session was not found." }; + }); +} diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index e88f6f17e..13c0508ba 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -13,7 +13,6 @@ import { systemPreferences, } from "electron"; import { showCursor } from "../../cursorHider"; -import { getMonitorHandles } from "../monitorResolver"; import { ALLOW_RECORDLY_WINDOW_CAPTURE } from "../constants"; import { startWindowBoundsCapture, stopWindowBoundsCapture } from "../cursor/bounds"; import { startInteractionCapture, stopInteractionCapture } from "../cursor/interaction"; @@ -31,6 +30,7 @@ import { writeCursorTelemetry, } from "../cursor/telemetry"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; +import { getMonitorHandles } from "../monitorResolver"; import { ensureNativeCaptureHelperBinary, ensureSwiftHelperBinary, @@ -433,13 +433,16 @@ export function registerRecordingHandlers( const recordingsDir = await getRecordingsDir(); const timestamp = Date.now(); const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`); - tempVideoPath = path.join(app.getPath("temp"), `recordly-native-${timestamp}.mp4`); - + tempVideoPath = path.join( + app.getPath("temp"), + `recordly-native-${timestamp}.mp4`, + ); + let captureOutput = ""; let systemAudioPath: string | null = null; let microphonePath: string | null = null; let orphanedMicAudioPath: string | null = null; - + const browserMicFallbackRequested = shouldStartWindowsBrowserMicrophoneFallback(options); const windowId = parseWindowId(source?.id); @@ -476,7 +479,7 @@ export function registerRecordingHandlers( // Fallback to coordinate-based matching if handle resolution fails config.displayId = resolvedDisplay.displayId; } - + config.displayX = Math.round(resolvedDisplay.bounds.x); config.displayY = Math.round(resolvedDisplay.bounds.y); config.displayW = Math.round(resolvedDisplay.bounds.width); @@ -501,7 +504,10 @@ export function registerRecordingHandlers( if (options?.capturesMicrophone && !browserMicFallbackRequested) { microphonePath = path.join(recordingsDir, `recording-${timestamp}.mic.wav`); - tempMicPath = path.join(app.getPath("temp"), `recordly-native-${timestamp}.mic.wav`); + tempMicPath = path.join( + app.getPath("temp"), + `recordly-native-${timestamp}.mic.wav`, + ); config.captureMic = true; config.micOutputPath = tempMicPath; if (options.microphoneLabel) { @@ -899,333 +905,337 @@ export function registerRecordingHandlers( const start = Date.now(); console.log("[PERF:MAIN] Handler: stop-native-screen-recording: STARTED"); try { - // Windows native capture stop path - if (process.platform === "win32" && windowsNativeCaptureActive) { - let stagedTempVideoPath: string | null = null; - let stagedTempSystemAudioPath: string | null = null; - let stagedTempMicAudioPath: string | null = null; - try { - if (!windowsCaptureProcess) { - throw new Error("Native Windows capture process is not running"); - } - - const proc = windowsCaptureProcess; - const preferredVideoPath = windowsCaptureTargetPath; - const preferredOrphanedMicAudioPath = windowsOrphanedMicAudioPath; - const diagnosticsSystemAudioPath = windowsSystemAudioPath; - const diagnosticsMicAudioPath = windowsMicAudioPath; - setWindowsCaptureStopRequested(true); - proc.stdin.write("stop\n"); - const tempVideoPath = await waitForWindowsCaptureStop(proc); - stagedTempVideoPath = tempVideoPath; - const finalVideoPath = preferredVideoPath ?? tempVideoPath; + // Windows native capture stop path + if (process.platform === "win32" && windowsNativeCaptureActive) { + let stagedTempVideoPath: string | null = null; + let stagedTempSystemAudioPath: string | null = null; + let stagedTempMicAudioPath: string | null = null; + try { + if (!windowsCaptureProcess) { + throw new Error("Native Windows capture process is not running"); + } - // Native Windows capture results are initially written to a safe temporary path - // (to avoid encoding failures with non-ASCII characters). We move them to the final - // destination now using Node.js, which handles Unicode paths correctly. - if (tempVideoPath !== finalVideoPath) { - await moveFileWithOverwrite(tempVideoPath, finalVideoPath); - } + const proc = windowsCaptureProcess; + const preferredVideoPath = windowsCaptureTargetPath; + const preferredOrphanedMicAudioPath = windowsOrphanedMicAudioPath; + const diagnosticsSystemAudioPath = windowsSystemAudioPath; + const diagnosticsMicAudioPath = windowsMicAudioPath; + setWindowsCaptureStopRequested(true); + proc.stdin.write("stop\n"); + const tempVideoPath = await waitForWindowsCaptureStop(proc); + stagedTempVideoPath = tempVideoPath; + const finalVideoPath = preferredVideoPath ?? tempVideoPath; + + // Native Windows capture results are initially written to a safe temporary path + // (to avoid encoding failures with non-ASCII characters). We move them to the final + // destination now using Node.js, which handles Unicode paths correctly. + if (tempVideoPath !== finalVideoPath) { + await moveFileWithOverwrite(tempVideoPath, finalVideoPath); + } - if (windowsSystemAudioPath && tempVideoPath.endsWith(".mp4")) { - const tempAudioPath = tempVideoPath.replace(".mp4", ".system.wav"); - stagedTempSystemAudioPath = tempAudioPath; - const finalAudioPath = windowsSystemAudioPath; - if (await pathExists(tempAudioPath)) { - await moveFileWithOverwrite(tempAudioPath, finalAudioPath); - const tempJson = tempAudioPath + ".json"; - if (await pathExists(tempJson)) { - await moveFileWithOverwrite(tempJson, finalAudioPath + ".json"); + if (windowsSystemAudioPath && tempVideoPath.endsWith(".mp4")) { + const tempAudioPath = tempVideoPath.replace(".mp4", ".system.wav"); + stagedTempSystemAudioPath = tempAudioPath; + const finalAudioPath = windowsSystemAudioPath; + if (await pathExists(tempAudioPath)) { + await moveFileWithOverwrite(tempAudioPath, finalAudioPath); + const tempJson = tempAudioPath + ".json"; + if (await pathExists(tempJson)) { + await moveFileWithOverwrite(tempJson, finalAudioPath + ".json"); + } } } - } - if (windowsMicAudioPath && tempVideoPath.endsWith(".mp4")) { - const tempMicPath = tempVideoPath.replace(".mp4", ".mic.wav"); - stagedTempMicAudioPath = tempMicPath; - const finalMicPath = windowsMicAudioPath; - if (await pathExists(tempMicPath)) { - await moveFileWithOverwrite(tempMicPath, finalMicPath); - const tempJson = tempMicPath + ".json"; - if (await pathExists(tempJson)) { - await moveFileWithOverwrite(tempJson, finalMicPath + ".json"); + if (windowsMicAudioPath && tempVideoPath.endsWith(".mp4")) { + const tempMicPath = tempVideoPath.replace(".mp4", ".mic.wav"); + stagedTempMicAudioPath = tempMicPath; + const finalMicPath = windowsMicAudioPath; + if (await pathExists(tempMicPath)) { + await moveFileWithOverwrite(tempMicPath, finalMicPath); + const tempJson = tempMicPath + ".json"; + if (await pathExists(tempJson)) { + await moveFileWithOverwrite(tempJson, finalMicPath + ".json"); + } } } - } - const validation = await validateRecordedVideo(finalVideoPath); + const validation = await validateRecordedVideo(finalVideoPath); - setWindowsCaptureProcess(null); - setWindowsNativeCaptureActive(false); - setNativeScreenRecordingActive(false); - setWindowsCaptureTargetPath(null); - setWindowsCaptureStopRequested(false); - setWindowsCapturePaused(false); - setWindowsOrphanedMicAudioPath(null); - await cleanupWindowsOrphanedMicAudioPath(preferredOrphanedMicAudioPath); - setWindowsPendingVideoPath(finalVideoPath); - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "stop", - outputPath: finalVideoPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: validation.fileSizeBytes, - }); - await writeWindowsRecordingDiagnostics(finalVideoPath, { - phase: "stop", - outputPath: finalVideoPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - details: { + setWindowsCaptureProcess(null); + setWindowsNativeCaptureActive(false); + setNativeScreenRecordingActive(false); + setWindowsCaptureTargetPath(null); + setWindowsCaptureStopRequested(false); + setWindowsCapturePaused(false); + setWindowsOrphanedMicAudioPath(null); + await cleanupWindowsOrphanedMicAudioPath(preferredOrphanedMicAudioPath); + setWindowsPendingVideoPath(finalVideoPath); + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "stop", + outputPath: finalVideoPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, fileSizeBytes: validation.fileSizeBytes, - durationSeconds: validation.durationSeconds, - }, - }); + }); + await writeWindowsRecordingDiagnostics(finalVideoPath, { + phase: "stop", + outputPath: finalVideoPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + details: { + fileSizeBytes: validation.fileSizeBytes, + durationSeconds: validation.durationSeconds, + }, + }); - // Persist cursor telemetry before returning so the editor can find it immediately - snapshotCursorTelemetryForPersistence(); - try { - await persistPendingCursorTelemetry(finalVideoPath); - } catch (error) { - console.warn("Failed to persist cursor telemetry during native stop:", error); - } + // Persist cursor telemetry before returning so the editor can find it immediately + snapshotCursorTelemetryForPersistence(); + try { + await persistPendingCursorTelemetry(finalVideoPath); + } catch (error) { + console.warn( + "Failed to persist cursor telemetry during native stop:", + error, + ); + } - return { success: true, path: finalVideoPath }; - } catch (error) { - console.error("Failed to stop native Windows capture:", error); - const fallbackPath = await resolveExistingPath( - windowsCaptureTargetPath, - stagedTempVideoPath, - ); - const recoveredSystemAudioPath = await resolveExistingPath( - windowsSystemAudioPath, - stagedTempSystemAudioPath, - ); - const recoveredMicAudioPath = await resolveExistingPath( - windowsMicAudioPath, - stagedTempMicAudioPath, - ); - const fallbackOrphanedMicAudioPath = windowsOrphanedMicAudioPath; - const diagnosticsSystemAudioPath = recoveredSystemAudioPath ?? windowsSystemAudioPath; - const diagnosticsMicAudioPath = recoveredMicAudioPath ?? windowsMicAudioPath; - setWindowsNativeCaptureActive(false); - setNativeScreenRecordingActive(false); - setWindowsCaptureProcess(null); - setWindowsCaptureTargetPath(null); - setWindowsCaptureStopRequested(false); - setWindowsCapturePaused(false); - setWindowsOrphanedMicAudioPath(null); + return { success: true, path: finalVideoPath }; + } catch (error) { + console.error("Failed to stop native Windows capture:", error); + const fallbackPath = await resolveExistingPath( + windowsCaptureTargetPath, + stagedTempVideoPath, + ); + const recoveredSystemAudioPath = await resolveExistingPath( + windowsSystemAudioPath, + stagedTempSystemAudioPath, + ); + const recoveredMicAudioPath = await resolveExistingPath( + windowsMicAudioPath, + stagedTempMicAudioPath, + ); + const fallbackOrphanedMicAudioPath = windowsOrphanedMicAudioPath; + const diagnosticsSystemAudioPath = + recoveredSystemAudioPath ?? windowsSystemAudioPath; + const diagnosticsMicAudioPath = recoveredMicAudioPath ?? windowsMicAudioPath; + setWindowsNativeCaptureActive(false); + setNativeScreenRecordingActive(false); + setWindowsCaptureProcess(null); + setWindowsCaptureTargetPath(null); + setWindowsCaptureStopRequested(false); + setWindowsCapturePaused(false); + setWindowsOrphanedMicAudioPath(null); - if (fallbackPath) { - try { - const validation = await validateRecordedVideo(fallbackPath); - setWindowsPendingVideoPath(fallbackPath); - setWindowsSystemAudioPath(recoveredSystemAudioPath); - setWindowsMicAudioPath(recoveredMicAudioPath); - await cleanupWindowsOrphanedMicAudioPath(fallbackOrphanedMicAudioPath); - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: validation.fileSizeBytes, - error: String(error), - }); - await writeWindowsRecordingDiagnostics(fallbackPath, { - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - error: String(error), - details: { + if (fallbackPath) { + try { + const validation = await validateRecordedVideo(fallbackPath); + setWindowsPendingVideoPath(fallbackPath); + setWindowsSystemAudioPath(recoveredSystemAudioPath); + setWindowsMicAudioPath(recoveredMicAudioPath); + await cleanupWindowsOrphanedMicAudioPath(fallbackOrphanedMicAudioPath); + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "stop", + outputPath: fallbackPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, fileSizeBytes: validation.fileSizeBytes, - durationSeconds: validation.durationSeconds, - recoveredAfterStopFailure: true, - }, - }); - return { success: true, path: fallbackPath }; - } catch { - // File is absent or failed validation. + error: String(error), + }); + await writeWindowsRecordingDiagnostics(fallbackPath, { + phase: "stop", + outputPath: fallbackPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + error: String(error), + details: { + fileSizeBytes: validation.fileSizeBytes, + durationSeconds: validation.durationSeconds, + recoveredAfterStopFailure: true, + }, + }); + return { success: true, path: fallbackPath }; + } catch { + // File is absent or failed validation. + } } - } - setWindowsSystemAudioPath(null); - setWindowsMicAudioPath(null); - setWindowsPendingVideoPath(null); - await cleanupWindowsOrphanedMicAudioPath(fallbackOrphanedMicAudioPath); + setWindowsSystemAudioPath(null); + setWindowsMicAudioPath(null); + setWindowsPendingVideoPath(null); + await cleanupWindowsOrphanedMicAudioPath(fallbackOrphanedMicAudioPath); - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: await getFileSizeIfPresent(fallbackPath), - error: String(error), - }); - await writeWindowsRecordingDiagnostics(fallbackPath, { - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - error: String(error), - details: { + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "stop", + outputPath: fallbackPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, fileSizeBytes: await getFileSizeIfPresent(fallbackPath), - }, - }); + error: String(error), + }); + await writeWindowsRecordingDiagnostics(fallbackPath, { + phase: "stop", + outputPath: fallbackPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + error: String(error), + details: { + fileSizeBytes: await getFileSizeIfPresent(fallbackPath), + }, + }); + return { + success: false, + message: "Failed to stop native Windows capture", + error: String(error), + }; + } + } + + if (process.platform !== "darwin") { return { success: false, - message: "Failed to stop native Windows capture", - error: String(error), + message: "Native screen recording is only available on macOS.", }; } - } - - if (process.platform !== "darwin") { - return { - success: false, - message: "Native screen recording is only available on macOS.", - }; - } - - if (!nativeScreenRecordingActive) { - const recovered = await recoverNativeMacCaptureOutput(); - if (recovered) { - return recovered; - } - return { success: false, message: "No native screen recording is active." }; - } + if (!nativeScreenRecordingActive) { + const recovered = await recoverNativeMacCaptureOutput(); + if (recovered) { + return recovered; + } - try { - if (!nativeCaptureProcess) { - throw new Error("Native capture helper process is not running"); + return { success: false, message: "No native screen recording is active." }; } - const process = nativeCaptureProcess; - const preferredVideoPath = nativeCaptureTargetPath; - const preferredSystemAudioPath = nativeCaptureSystemAudioPath; - const preferredMicrophonePath = nativeCaptureMicrophonePath; - console.log( - "[stop-native] Audio paths — system:", - preferredSystemAudioPath, - "mic:", - preferredMicrophonePath, - ); - setNativeCaptureStopRequested(true); - process.stdin.write("stop\n"); - const tempVideoPath = await waitForNativeCaptureStop(process); - console.log("[stop-native] Helper stopped, tempVideoPath:", tempVideoPath); - setNativeCaptureProcess(null); - setNativeScreenRecordingActive(false); - setNativeCaptureTargetPath(null); - setNativeCaptureSystemAudioPath(null); - setNativeCaptureMicrophonePath(null); - setNativeCaptureStopRequested(false); - setNativeCapturePaused(false); - - const finalVideoPath = preferredVideoPath ?? tempVideoPath; - if (tempVideoPath !== finalVideoPath) { - await moveFileWithOverwrite(tempVideoPath, finalVideoPath); - } + try { + if (!nativeCaptureProcess) { + throw new Error("Native capture helper process is not running"); + } - if (preferredSystemAudioPath || preferredMicrophonePath) { + const process = nativeCaptureProcess; + const preferredVideoPath = nativeCaptureTargetPath; + const preferredSystemAudioPath = nativeCaptureSystemAudioPath; + const preferredMicrophonePath = nativeCaptureMicrophonePath; console.log( - "[stop-native] Attempting audio mux (merging separate tracks) into:", - finalVideoPath, + "[stop-native] Audio paths — system:", + preferredSystemAudioPath, + "mic:", + preferredMicrophonePath, ); - try { - await muxNativeMacRecordingWithAudio( + setNativeCaptureStopRequested(true); + process.stdin.write("stop\n"); + const tempVideoPath = await waitForNativeCaptureStop(process); + console.log("[stop-native] Helper stopped, tempVideoPath:", tempVideoPath); + setNativeCaptureProcess(null); + setNativeScreenRecordingActive(false); + setNativeCaptureTargetPath(null); + setNativeCaptureSystemAudioPath(null); + setNativeCaptureMicrophonePath(null); + setNativeCaptureStopRequested(false); + setNativeCapturePaused(false); + + const finalVideoPath = preferredVideoPath ?? tempVideoPath; + if (tempVideoPath !== finalVideoPath) { + await moveFileWithOverwrite(tempVideoPath, finalVideoPath); + } + + if (preferredSystemAudioPath || preferredMicrophonePath) { + console.log( + "[stop-native] Attempting audio mux (merging separate tracks) into:", finalVideoPath, - preferredSystemAudioPath, - preferredMicrophonePath, - ); - console.log("[stop-native] Audio mux completed successfully"); - } catch (error) { - console.warn( - "[stop-native] Audio mux failed (video still has inline audio):", - error, ); + try { + await muxNativeMacRecordingWithAudio( + finalVideoPath, + preferredSystemAudioPath, + preferredMicrophonePath, + ); + console.log("[stop-native] Audio mux completed successfully"); + } catch (error) { + console.warn( + "[stop-native] Audio mux failed (video still has inline audio):", + error, + ); + } + } else { + console.log("[stop-native] No separate audio tracks to mux"); } - } else { - console.log("[stop-native] No separate audio tracks to mux"); - } - return await finalizeStoredVideo(finalVideoPath); - } catch (error) { - console.error("Failed to stop native ScreenCaptureKit recording:", error); - const fallbackPath = nativeCaptureTargetPath; - const fallbackSystemAudioPath = nativeCaptureSystemAudioPath; - const fallbackMicrophonePath = nativeCaptureMicrophonePath; - const fallbackFileSizeBytes = await getFileSizeIfPresent(fallbackPath); - setNativeScreenRecordingActive(false); - setNativeCaptureProcess(null); - setNativeCaptureTargetPath(null); - setNativeCaptureSystemAudioPath(null); - setNativeCaptureMicrophonePath(null); - setNativeCaptureStopRequested(false); - setNativeCapturePaused(false); + return await finalizeStoredVideo(finalVideoPath); + } catch (error) { + console.error("Failed to stop native ScreenCaptureKit recording:", error); + const fallbackPath = nativeCaptureTargetPath; + const fallbackSystemAudioPath = nativeCaptureSystemAudioPath; + const fallbackMicrophonePath = nativeCaptureMicrophonePath; + const fallbackFileSizeBytes = await getFileSizeIfPresent(fallbackPath); + setNativeScreenRecordingActive(false); + setNativeCaptureProcess(null); + setNativeCaptureTargetPath(null); + setNativeCaptureSystemAudioPath(null); + setNativeCaptureMicrophonePath(null); + setNativeCaptureStopRequested(false); + setNativeCapturePaused(false); - recordNativeCaptureDiagnostics({ - backend: "mac-screencapturekit", - phase: "stop", - sourceId: lastNativeCaptureDiagnostics?.sourceId ?? null, - sourceType: lastNativeCaptureDiagnostics?.sourceType ?? "unknown", - displayId: lastNativeCaptureDiagnostics?.displayId ?? null, - displayBounds: lastNativeCaptureDiagnostics?.displayBounds ?? null, - windowHandle: lastNativeCaptureDiagnostics?.windowHandle ?? null, - helperPath: lastNativeCaptureDiagnostics?.helperPath ?? null, - outputPath: fallbackPath, - systemAudioPath: fallbackSystemAudioPath, - microphonePath: fallbackMicrophonePath, - osRelease: lastNativeCaptureDiagnostics?.osRelease, - supported: lastNativeCaptureDiagnostics?.supported, - helperExists: lastNativeCaptureDiagnostics?.helperExists, - processOutput: nativeCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: fallbackFileSizeBytes, - error: String(error), - }); + recordNativeCaptureDiagnostics({ + backend: "mac-screencapturekit", + phase: "stop", + sourceId: lastNativeCaptureDiagnostics?.sourceId ?? null, + sourceType: lastNativeCaptureDiagnostics?.sourceType ?? "unknown", + displayId: lastNativeCaptureDiagnostics?.displayId ?? null, + displayBounds: lastNativeCaptureDiagnostics?.displayBounds ?? null, + windowHandle: lastNativeCaptureDiagnostics?.windowHandle ?? null, + helperPath: lastNativeCaptureDiagnostics?.helperPath ?? null, + outputPath: fallbackPath, + systemAudioPath: fallbackSystemAudioPath, + microphonePath: fallbackMicrophonePath, + osRelease: lastNativeCaptureDiagnostics?.osRelease, + supported: lastNativeCaptureDiagnostics?.supported, + helperExists: lastNativeCaptureDiagnostics?.helperExists, + processOutput: nativeCaptureOutputBuffer.trim() || undefined, + fileSizeBytes: fallbackFileSizeBytes, + error: String(error), + }); - // Try to recover: if the target file exists on disk, finalize with it - if (fallbackPath) { - try { - await fs.access(fallbackPath); - console.log( - "[stop-native-screen-recording] Recovering with fallback path:", - fallbackPath, - ); - if (fallbackSystemAudioPath || fallbackMicrophonePath) { - try { - await muxNativeMacRecordingWithAudio( - fallbackPath, - fallbackSystemAudioPath, - fallbackMicrophonePath, - ); - } catch (muxError) { - console.warn( - "Failed to mux recovered native macOS audio into capture:", - muxError, - ); + // Try to recover: if the target file exists on disk, finalize with it + if (fallbackPath) { + try { + await fs.access(fallbackPath); + console.log( + "[stop-native-screen-recording] Recovering with fallback path:", + fallbackPath, + ); + if (fallbackSystemAudioPath || fallbackMicrophonePath) { + try { + await muxNativeMacRecordingWithAudio( + fallbackPath, + fallbackSystemAudioPath, + fallbackMicrophonePath, + ); + } catch (muxError) { + console.warn( + "Failed to mux recovered native macOS audio into capture:", + muxError, + ); + } } + return await finalizeStoredVideo(fallbackPath); + } catch { + // File doesn't exist or isn't accessible } - return await finalizeStoredVideo(fallbackPath); - } catch { - // File doesn't exist or isn't accessible } - } - const recovered = await recoverNativeMacCaptureOutput(); - if (recovered) { - return recovered; - } + const recovered = await recoverNativeMacCaptureOutput(); + if (recovered) { + return recovered; + } return { success: false, diff --git a/electron/main.ts b/electron/main.ts index 0a50b6f93..4b507602b 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -26,6 +26,7 @@ import { registerIpcHandlers, } from "./ipc/handlers"; import { ensureMediaServer } from "./mediaServer"; +import { cleanupPhoneRemoteServer } from "./phoneRemote/server"; import { ensurePackagedRendererServer } from "./rendererServer"; import type { UpdateToastPayload } from "./updater"; import { @@ -877,6 +878,7 @@ function createSourceSelectorWindowWrapper() { app.on("before-quit", () => { killWindowsCaptureProcess(); showCursor(); + cleanupPhoneRemoteServer(); cleanupNativeVideoExportSessions(); void cleanupAllExportStreams(); }); diff --git a/electron/phoneRemote/mobilePage.ts b/electron/phoneRemote/mobilePage.ts new file mode 100644 index 000000000..424ac2eb5 --- /dev/null +++ b/electron/phoneRemote/mobilePage.ts @@ -0,0 +1,535 @@ +export function getPhoneRemoteMobilePage(): string { + return ` + + + + + Recordly Phone Camera + + + +
+
+
+

Recordly Camera

+

Send this phone camera and mic to your laptop recording.

+
+
Waiting
+
+ +
+ +
Camera preview will appear here after permission is granted.
+
+ +
+ + + +
+ +
+ + +
+
+ + + +`; +} diff --git a/electron/phoneRemote/server.ts b/electron/phoneRemote/server.ts new file mode 100644 index 000000000..cbe4ff075 --- /dev/null +++ b/electron/phoneRemote/server.ts @@ -0,0 +1,477 @@ +import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import os from "node:os"; +import { getPhoneRemoteMobilePage } from "./mobilePage"; +import { + addPhoneSignal, + getLaptopSignalsSince, + getPhoneRemoteSessionByCode, + pruneExpiredPhoneRemoteSessions, + toPublicSession, + updatePhoneRemoteStatus, +} from "./sessionStore"; +import type { + PhoneRemoteJoinUrls, + PhoneRemoteSignalMessage, + PhoneRemoteStatusMessage, +} from "./types"; + +const JSON_LIMIT_BYTES = 256 * 1024; +const TUNNEL_READY_TIMEOUT_MS = 8000; +const TRY_CLOUDFLARE_URL_PATTERN = /https:\/\/[a-z0-9-]+\.trycloudflare\.com/i; +const PHONE_REMOTE_STATUSES = new Set([ + "waiting", + "phone-connected", + "preview-live", + "mic-active", + "reconnecting", + "disconnected", + "camera-permission-denied", + "microphone-permission-denied", + "no-audio-track", + "phone-backgrounded", + "phone-sleeping", + "error", +]); + +let serverBasePort: number | null = null; +let phoneRemoteServerStartPromise: Promise | null = null; +let tunnelProcess: ChildProcessWithoutNullStreams | null = null; +let tunnelBaseUrl: string | null = null; +let tunnelError: string | null = null; +let tunnelStartPromise: Promise | null = null; +let pruneTimer: NodeJS.Timeout | null = null; + +function getLanHost(): string { + const interfaces = os.networkInterfaces(); + + for (const addresses of Object.values(interfaces)) { + for (const address of addresses ?? []) { + if (address.family === "IPv4" && !address.internal) { + return address.address; + } + } + } + + return "127.0.0.1"; +} + +function writeJson(response: ServerResponse, statusCode: number, payload: unknown) { + response.writeHead(statusCode, { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type", + "Cache-Control": "no-store", + "Content-Type": "application/json; charset=utf-8", + }); + response.end(JSON.stringify(payload)); +} + +function writeText(response: ServerResponse, statusCode: number, message: string) { + response.writeHead(statusCode, { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type", + "Cache-Control": "no-store", + "Content-Type": "text/plain; charset=utf-8", + }); + response.end(message); +} + +function readJsonBody(request: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let totalBytes = 0; + const chunks: Buffer[] = []; + + request.on("data", (chunk: Buffer) => { + totalBytes += chunk.byteLength; + if (totalBytes > JSON_LIMIT_BYTES) { + reject(new Error("Request body is too large")); + request.destroy(); + return; + } + chunks.push(chunk); + }); + + request.on("end", () => { + if (chunks.length === 0) { + resolve({}); + return; + } + + try { + resolve(JSON.parse(Buffer.concat(chunks).toString("utf8"))); + } catch { + reject(new Error("Request body must be valid JSON")); + } + }); + + request.on("error", reject); + }); +} + +function asRecord(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +function parsePhoneRemoteSignal(value: unknown): PhoneRemoteSignalMessage | null { + const record = asRecord(value); + if (!record || typeof record.type !== "string") { + return null; + } + + if (record.type === "offer" || record.type === "answer") { + const description = asRecord(record.description); + if ( + !description || + typeof description.type !== "string" || + typeof description.sdp !== "string" + ) { + return null; + } + if (description.type !== "offer" && description.type !== "answer") { + return null; + } + if (description.type !== record.type) { + return null; + } + return { + type: record.type, + description: { + type: description.type, + sdp: description.sdp, + }, + }; + } + + if (record.type === "ice-candidate") { + if (record.candidate === null) { + return { + type: "ice-candidate", + candidate: null, + }; + } + + const candidate = asRecord(record.candidate); + if (!candidate || typeof candidate.candidate !== "string") { + return null; + } + + return { + type: "ice-candidate", + candidate: { + candidate: candidate.candidate, + sdpMid: typeof candidate.sdpMid === "string" ? candidate.sdpMid : null, + sdpMLineIndex: + typeof candidate.sdpMLineIndex === "number" ? candidate.sdpMLineIndex : null, + usernameFragment: + typeof candidate.usernameFragment === "string" + ? candidate.usernameFragment + : undefined, + }, + }; + } + + return null; +} + +function parsePhoneRemoteStatus(value: unknown): PhoneRemoteStatusMessage | null { + const record = asRecord(value); + if (!record || typeof record.status !== "string") { + return null; + } + + if (!PHONE_REMOTE_STATUSES.has(record.status as PhoneRemoteStatusMessage["status"])) { + return null; + } + + return { + status: record.status as PhoneRemoteStatusMessage["status"], + detail: typeof record.detail === "string" ? record.detail : undefined, + hasAudio: typeof record.hasAudio === "boolean" ? record.hasAudio : undefined, + hasVideo: typeof record.hasVideo === "boolean" ? record.hasVideo : undefined, + facingMode: + record.facingMode === "user" || record.facingMode === "environment" + ? record.facingMode + : undefined, + }; +} + +function getCodeFromUrl(url: URL): string | null { + const code = url.searchParams.get("code"); + return code && code.trim().length > 0 ? code.trim().toUpperCase() : null; +} + +async function resolveSessionFromRequest( + request: IncomingMessage, + url: URL, +): Promise | null> { + let code = getCodeFromUrl(url); + + if (!code && request.method === "POST") { + const body = asRecord(await readJsonBody(request)); + code = typeof body?.code === "string" ? body.code.trim().toUpperCase() : null; + (request as IncomingMessage & { parsedJsonBody?: Record }).parsedJsonBody = + body ?? {}; + } + + return code ? getPhoneRemoteSessionByCode(code) : null; +} + +function getParsedJsonBody(request: IncomingMessage): Record { + return ( + (request as IncomingMessage & { parsedJsonBody?: Record }) + .parsedJsonBody ?? {} + ); +} + +async function handlePhoneRemoteRequest(request: IncomingMessage, response: ServerResponse) { + try { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + + if (request.method === "OPTIONS") { + writeText(response, 204, ""); + return; + } + + if (request.method === "GET" && (url.pathname === "/" || url.pathname === "/phone")) { + response.writeHead(url.pathname === "/" ? 302 : 200, { + ...(url.pathname === "/" ? { Location: "/phone" } : {}), + "Cache-Control": "no-store", + "Content-Type": "text/html; charset=utf-8", + }); + response.end(url.pathname === "/" ? "" : getPhoneRemoteMobilePage()); + return; + } + + if (request.method === "GET" && url.pathname === "/api/phone-remote/session") { + const session = getCodeFromUrl(url) + ? getPhoneRemoteSessionByCode(getCodeFromUrl(url) ?? "") + : null; + if (!session) { + writeText(response, 404, "Phone session was not found or has expired."); + return; + } + + writeJson(response, 200, { + session: toPublicSession(session), + }); + return; + } + + if (request.method === "GET" && url.pathname === "/api/phone-remote/signals") { + const session = await resolveSessionFromRequest(request, url); + if (!session) { + writeText(response, 404, "Phone session was not found or has expired."); + return; + } + + const after = Number.parseInt(url.searchParams.get("after") ?? "0", 10); + const afterIndex = Number.isFinite(after) ? Math.max(0, after) : 0; + const signals = getLaptopSignalsSince(session, afterIndex); + const lastSignal = signals.length > 0 ? signals[signals.length - 1] : null; + writeJson(response, 200, { + signals, + nextIndex: lastSignal?.index ?? afterIndex, + }); + return; + } + + if (request.method === "POST" && url.pathname === "/api/phone-remote/signal") { + const session = await resolveSessionFromRequest(request, url); + if (!session) { + writeText(response, 404, "Phone session was not found or has expired."); + return; + } + + const body = getParsedJsonBody(request); + const message = parsePhoneRemoteSignal(body.message); + if (!message) { + writeText(response, 400, "Invalid phone signal payload."); + return; + } + + addPhoneSignal(session, message); + writeJson(response, 200, { success: true }); + return; + } + + if (request.method === "POST" && url.pathname === "/api/phone-remote/status") { + const session = await resolveSessionFromRequest(request, url); + if (!session) { + writeText(response, 404, "Phone session was not found or has expired."); + return; + } + + const status = parsePhoneRemoteStatus(getParsedJsonBody(request)); + if (!status) { + writeText(response, 400, "Invalid phone status payload."); + return; + } + + updatePhoneRemoteStatus(session, status); + writeJson(response, 200, { success: true }); + return; + } + + writeText(response, 404, "Not Found"); + } catch (error) { + writeText(response, 500, error instanceof Error ? error.message : String(error)); + } +} + +async function ensurePhoneRemoteServerPort(): Promise { + if (serverBasePort) { + return serverBasePort; + } + + if (phoneRemoteServerStartPromise) { + return phoneRemoteServerStartPromise; + } + + phoneRemoteServerStartPromise = new Promise((resolve, reject) => { + const server = createServer((request, response) => { + void handlePhoneRemoteRequest(request, response); + }); + + server.once("error", reject); + server.listen(0, "0.0.0.0", () => { + const address = server.address() as AddressInfo | null; + if (!address) { + reject(new Error("Phone remote server did not expose a TCP address")); + return; + } + + serverBasePort = address.port; + console.log(`[phone-remote] Listening on port ${serverBasePort}`); + if (!pruneTimer) { + pruneTimer = setInterval(() => { + pruneExpiredPhoneRemoteSessions(); + }, 60_000); + pruneTimer.unref?.(); + } + resolve(serverBasePort); + }); + }); + + return phoneRemoteServerStartPromise; +} + +async function startQuickTunnel(localBaseUrl: string): Promise { + if (tunnelBaseUrl || tunnelError) { + return tunnelBaseUrl; + } + + if (tunnelStartPromise) { + return tunnelStartPromise; + } + + tunnelStartPromise = new Promise((resolve) => { + let settled = false; + const settle = (value: string | null) => { + if (settled) { + return; + } + settled = true; + resolve(value); + }; + + try { + tunnelProcess = spawn( + "cloudflared", + ["tunnel", "--url", localBaseUrl, "--no-autoupdate"], + { windowsHide: true }, + ); + } catch (error) { + tunnelError = error instanceof Error ? error.message : String(error); + settle(null); + return; + } + + const spawnedProcess = tunnelProcess; + const timeout = setTimeout(() => { + tunnelError = "Secure tunnel did not become ready in time."; + if (!spawnedProcess.killed) { + spawnedProcess.kill(); + } + if (tunnelProcess === spawnedProcess) { + tunnelProcess = null; + } + settle(null); + }, TUNNEL_READY_TIMEOUT_MS); + + const inspectOutput = (chunk: Buffer) => { + const output = chunk.toString("utf8"); + const match = output.match(TRY_CLOUDFLARE_URL_PATTERN); + if (match) { + tunnelBaseUrl = match[0].replace(/\/$/, ""); + clearTimeout(timeout); + settle(tunnelBaseUrl); + } + }; + + spawnedProcess.stdout.on("data", inspectOutput); + spawnedProcess.stderr.on("data", inspectOutput); + spawnedProcess.once("error", (error) => { + tunnelError = error.message; + clearTimeout(timeout); + settle(null); + }); + spawnedProcess.once("exit", (code) => { + if (!settled) { + tunnelError = + typeof code === "number" + ? `Secure tunnel exited with code ${code}.` + : "Secure tunnel exited before it became ready."; + clearTimeout(timeout); + settle(null); + } + if (tunnelProcess === spawnedProcess) { + tunnelProcess = null; + } + }); + }); + + return tunnelStartPromise; +} + +export async function getPhoneRemoteJoinUrls(): Promise { + const port = await ensurePhoneRemoteServerPort(); + const lanBaseUrl = `http://${getLanHost()}:${port}`; + const localBaseUrl = `http://127.0.0.1:${port}`; + const secureTunnelBaseUrl = await startQuickTunnel(localBaseUrl); + const buildJoinUrl = (baseUrl: string) => `${baseUrl}/phone`; + + if (secureTunnelBaseUrl) { + return { + joinUrl: buildJoinUrl(secureTunnelBaseUrl), + localJoinUrl: buildJoinUrl(localBaseUrl), + lanJoinUrl: buildJoinUrl(lanBaseUrl), + tunnelJoinUrl: buildJoinUrl(secureTunnelBaseUrl), + urlMode: "secure-tunnel", + }; + } + + return { + joinUrl: buildJoinUrl(lanBaseUrl), + localJoinUrl: buildJoinUrl(localBaseUrl), + lanJoinUrl: buildJoinUrl(lanBaseUrl), + urlMode: "lan", + tunnelError: tunnelError ?? "Secure tunnel is unavailable.", + }; +} + +export function cleanupPhoneRemoteServer() { + if (tunnelProcess) { + tunnelProcess.kill(); + tunnelProcess = null; + } + + if (pruneTimer) { + clearInterval(pruneTimer); + pruneTimer = null; + } +} diff --git a/electron/phoneRemote/sessionStore.test.ts b/electron/phoneRemote/sessionStore.test.ts new file mode 100644 index 000000000..2867f1ece --- /dev/null +++ b/electron/phoneRemote/sessionStore.test.ts @@ -0,0 +1,113 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + addLaptopSignal, + addPhoneSignal, + clearPhoneRemoteSessionsForTests, + createPhoneRemoteSession, + getLaptopSignalsSince, + getPhoneRemoteSession, + subscribePhoneRemoteStore, + updatePhoneRemoteStatus, +} from "./sessionStore"; + +const NOW = new Date("2026-01-01T00:00:00.000Z"); +const urls = { + joinUrl: "https://example.test/phone", + localJoinUrl: "http://127.0.0.1:1234/phone", + lanJoinUrl: "http://192.168.1.2:1234/phone", + urlMode: "secure-tunnel" as const, +}; + +afterEach(() => { + vi.useRealTimers(); + clearPhoneRemoteSessionsForTests(); +}); + +describe("phone remote session store", () => { + it("creates an expiring pairing session with a code", () => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + const now = NOW.getTime(); + const session = createPhoneRemoteSession(42, urls, now); + + expect(session.id).toBeTruthy(); + expect(session.code).toMatch(/^[A-Z0-9]+$/); + expect(session.code).toHaveLength(8); + expect(session.status).toBe("waiting"); + expect(session.expiresAt).toBe(now + 600_000); + }); + + it("accepts laptop signals only from the owning webContents", () => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + const session = createPhoneRemoteSession(42, urls, NOW.getTime()); + const offer = { + type: "offer" as const, + description: { type: "offer" as const, sdp: "v=0" }, + }; + + expect(addLaptopSignal(session.id, 7, offer)).toBeNull(); + + const envelope = addLaptopSignal(session.id, 42, offer); + const storedSession = getPhoneRemoteSession(session.id); + + expect(envelope?.index).toBe(1); + expect(storedSession ? getLaptopSignalsSince(storedSession, 0) : []).toHaveLength(1); + }); + + it("emits phone signal and status updates to the session owner", () => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + const listener = vi.fn(); + subscribePhoneRemoteStore(listener); + const publicSession = createPhoneRemoteSession(42, urls, NOW.getTime()); + const session = getPhoneRemoteSession(publicSession.id); + + expect(session).not.toBeNull(); + if (!session) { + return; + } + + addPhoneSignal(session, { + type: "answer", + description: { type: "answer", sdp: "v=0" }, + }); + updatePhoneRemoteStatus(session, { + status: "mic-active", + hasAudio: true, + hasVideo: true, + }); + + expect(listener).toHaveBeenCalledWith( + expect.objectContaining({ + type: "signal", + sessionId: publicSession.id, + ownerWebContentsId: 42, + }), + ); + expect(listener).toHaveBeenCalledWith( + expect.objectContaining({ + type: "status", + sessionId: publicSession.id, + ownerWebContentsId: 42, + status: expect.objectContaining({ status: "mic-active" }), + }), + ); + }); + + it("refreshes the session expiry on phone status heartbeats", () => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + const createdAt = NOW.getTime(); + const session = createPhoneRemoteSession(42, urls, createdAt); + + vi.setSystemTime(createdAt + 300_000); + updatePhoneRemoteStatus(session, { + status: "preview-live", + hasAudio: true, + hasVideo: true, + }); + + expect(session.expiresAt).toBe(createdAt + 900_000); + }); +}); diff --git a/electron/phoneRemote/sessionStore.ts b/electron/phoneRemote/sessionStore.ts new file mode 100644 index 000000000..2049f9e8c --- /dev/null +++ b/electron/phoneRemote/sessionStore.ts @@ -0,0 +1,204 @@ +import crypto from "node:crypto"; +import type { + PhoneRemoteJoinUrls, + PhoneRemotePublicSession, + PhoneRemoteSession, + PhoneRemoteSignalEnvelope, + PhoneRemoteSignalMessage, + PhoneRemoteStatusMessage, + PhoneRemoteStoreEvent, +} from "./types"; + +const SESSION_TTL_MS = 10 * 60 * 1000; +const MAX_SIGNAL_HISTORY = 200; + +const sessions = new Map(); +const listeners = new Set<(event: PhoneRemoteStoreEvent) => void>(); + +function createPairingCode(): string { + let code = ""; + while (code.length < 8) { + code += crypto + .randomBytes(6) + .toString("base64url") + .replace(/[^A-Z0-9]/gi, "") + .toUpperCase(); + } + return code.slice(0, 8); +} + +function appendSignal( + signals: PhoneRemoteSignalEnvelope[], + message: PhoneRemoteSignalMessage, +): PhoneRemoteSignalEnvelope { + const previousIndex = signals.length > 0 ? signals[signals.length - 1].index : 0; + const envelope = { + index: previousIndex + 1, + sentAt: Date.now(), + message, + }; + + signals.push(envelope); + if (signals.length > MAX_SIGNAL_HISTORY) { + signals.splice(0, signals.length - MAX_SIGNAL_HISTORY); + } + + return envelope; +} + +function emit(event: PhoneRemoteStoreEvent) { + for (const listener of listeners) { + listener(event); + } +} + +export function subscribePhoneRemoteStore(listener: (event: PhoneRemoteStoreEvent) => void) { + listeners.add(listener); + return () => listeners.delete(listener); +} + +export function pruneExpiredPhoneRemoteSessions(now = Date.now()) { + for (const [sessionId, session] of sessions.entries()) { + if (session.expiresAt <= now) { + sessions.delete(sessionId); + emit({ + type: "status", + sessionId, + ownerWebContentsId: session.ownerWebContentsId, + status: { + status: "disconnected", + detail: "Phone session expired.", + }, + }); + } + } +} + +export function createPhoneRemoteSession( + ownerWebContentsId: number, + urls: PhoneRemoteJoinUrls, + now = Date.now(), +): PhoneRemotePublicSession { + pruneExpiredPhoneRemoteSessions(now); + + const session: PhoneRemoteSession = { + ...urls, + id: crypto.randomUUID(), + code: createPairingCode(), + createdAt: now, + expiresAt: now + SESSION_TTL_MS, + ownerWebContentsId, + status: "waiting", + laptopSignals: [], + phoneSignals: [], + }; + + sessions.set(session.id, session); + return toPublicSession(session); +} + +export function getPhoneRemoteSession(sessionId: string): PhoneRemoteSession | null { + pruneExpiredPhoneRemoteSessions(); + return sessions.get(sessionId) ?? null; +} + +export function getPhoneRemoteSessionByCode(code: string): PhoneRemoteSession | null { + pruneExpiredPhoneRemoteSessions(); + const normalizedCode = code.trim().toUpperCase(); + + for (const session of sessions.values()) { + if (session.code === normalizedCode) { + return session; + } + } + + return null; +} + +export function endPhoneRemoteSession(sessionId: string): boolean { + const session = sessions.get(sessionId); + if (!session) { + return false; + } + + sessions.delete(sessionId); + emit({ + type: "status", + sessionId, + ownerWebContentsId: session.ownerWebContentsId, + status: { + status: "disconnected", + detail: "Phone session ended.", + }, + }); + return true; +} + +export function addLaptopSignal( + sessionId: string, + ownerWebContentsId: number, + message: PhoneRemoteSignalMessage, +): PhoneRemoteSignalEnvelope | null { + const session = getPhoneRemoteSession(sessionId); + if (!session || session.ownerWebContentsId !== ownerWebContentsId) { + return null; + } + + return appendSignal(session.laptopSignals, message); +} + +export function getLaptopSignalsSince( + session: PhoneRemoteSession, + afterIndex: number, +): PhoneRemoteSignalEnvelope[] { + return session.laptopSignals.filter((signal) => signal.index > afterIndex); +} + +export function addPhoneSignal( + session: PhoneRemoteSession, + message: PhoneRemoteSignalMessage, +): PhoneRemoteSignalEnvelope { + const envelope = appendSignal(session.phoneSignals, message); + emit({ + type: "signal", + sessionId: session.id, + ownerWebContentsId: session.ownerWebContentsId, + message, + }); + return envelope; +} + +export function updatePhoneRemoteStatus( + session: PhoneRemoteSession, + status: PhoneRemoteStatusMessage, +) { + session.expiresAt = Date.now() + SESSION_TTL_MS; + session.status = status.status; + session.lastStatusDetail = status.detail; + emit({ + type: "status", + sessionId: session.id, + ownerWebContentsId: session.ownerWebContentsId, + status, + }); +} + +export function toPublicSession(session: PhoneRemoteSession): PhoneRemotePublicSession { + return { + id: session.id, + code: session.code, + expiresAt: session.expiresAt, + status: session.status, + joinUrl: session.joinUrl, + localJoinUrl: session.localJoinUrl, + lanJoinUrl: session.lanJoinUrl, + tunnelJoinUrl: session.tunnelJoinUrl, + urlMode: session.urlMode, + tunnelError: session.tunnelError, + }; +} + +export function clearPhoneRemoteSessionsForTests() { + sessions.clear(); + listeners.clear(); +} diff --git a/electron/phoneRemote/types.ts b/electron/phoneRemote/types.ts new file mode 100644 index 000000000..41abc1491 --- /dev/null +++ b/electron/phoneRemote/types.ts @@ -0,0 +1,75 @@ +export type PhoneRemoteSessionStatus = + | "waiting" + | "phone-connected" + | "preview-live" + | "mic-active" + | "reconnecting" + | "disconnected" + | "camera-permission-denied" + | "microphone-permission-denied" + | "no-audio-track" + | "phone-backgrounded" + | "phone-sleeping" + | "error"; + +export type PhoneRemoteSignalMessage = + | { + type: "offer" | "answer"; + description: RTCSessionDescriptionInit; + } + | { + type: "ice-candidate"; + candidate: RTCIceCandidateInit | null; + }; + +export type PhoneRemoteStatusMessage = { + status: PhoneRemoteSessionStatus; + detail?: string; + hasAudio?: boolean; + hasVideo?: boolean; + facingMode?: "user" | "environment"; +}; + +export type PhoneRemoteSignalEnvelope = { + index: number; + sentAt: number; + message: PhoneRemoteSignalMessage; +}; + +export type PhoneRemoteJoinUrls = { + joinUrl: string; + localJoinUrl: string; + lanJoinUrl: string; + tunnelJoinUrl?: string; + urlMode: "secure-tunnel" | "lan"; + tunnelError?: string; +}; + +export type PhoneRemotePublicSession = PhoneRemoteJoinUrls & { + id: string; + code: string; + expiresAt: number; + status: PhoneRemoteSessionStatus; +}; + +export type PhoneRemoteSession = PhoneRemotePublicSession & { + ownerWebContentsId: number; + createdAt: number; + laptopSignals: PhoneRemoteSignalEnvelope[]; + phoneSignals: PhoneRemoteSignalEnvelope[]; + lastStatusDetail?: string; +}; + +export type PhoneRemoteStoreEvent = + | { + type: "status"; + sessionId: string; + ownerWebContentsId: number; + status: PhoneRemoteStatusMessage; + } + | { + type: "signal"; + sessionId: string; + ownerWebContentsId: number; + message: PhoneRemoteSignalMessage; + }; diff --git a/electron/preload.ts b/electron/preload.ts index 932db07b0..f509aa880 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -933,6 +933,27 @@ contextBridge.exposeInMainWorld("electronAPI", { startCountdown: (seconds: number) => ipcRenderer.invoke("start-countdown", seconds), cancelCountdown: () => ipcRenderer.invoke("cancel-countdown"), getActiveCountdown: () => ipcRenderer.invoke("get-active-countdown"), + createPhoneRemoteSession: () => ipcRenderer.invoke("phone-remote:create-session"), + endPhoneRemoteSession: (sessionId: string) => + ipcRenderer.invoke("phone-remote:end-session", sessionId), + sendPhoneRemoteSignal: (sessionId: string, message: unknown) => + ipcRenderer.invoke("phone-remote:send-signal", sessionId, message), + onPhoneRemoteSignal: (callback: (payload: { sessionId: string; message: unknown }) => void) => { + const listener = ( + _event: Electron.IpcRendererEvent, + payload: { sessionId: string; message: unknown }, + ) => callback(payload); + ipcRenderer.on("phone-remote-signal", listener); + return () => ipcRenderer.removeListener("phone-remote-signal", listener); + }, + onPhoneRemoteStatus: (callback: (payload: { sessionId: string; status: unknown }) => void) => { + const listener = ( + _event: Electron.IpcRendererEvent, + payload: { sessionId: string; status: unknown }, + ) => callback(payload); + ipcRenderer.on("phone-remote-status", listener); + return () => ipcRenderer.removeListener("phone-remote-status", listener); + }, onCountdownTick: (callback: (seconds: number) => void) => { const listener = (_event: Electron.IpcRendererEvent, seconds: number) => callback(seconds); ipcRenderer.on("countdown-tick", listener); diff --git a/src/components/launch/LaunchWindow.module.css b/src/components/launch/LaunchWindow.module.css index 3638e7563..c5aed7b83 100644 --- a/src/components/launch/LaunchWindow.module.css +++ b/src/components/launch/LaunchWindow.module.css @@ -175,6 +175,115 @@ color: var(--launch-accent); } +.phoneStatusPill { + margin-left: auto; + max-width: 92px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + border-radius: 999px; + background: rgba(255, 255, 255, 0.07); + padding: 2px 7px; + color: #a5a5b3; + font-size: 10px; + font-weight: 700; + text-transform: capitalize; +} + +.phoneRemotePanel { + display: flex; + flex-direction: column; + gap: 8px; + margin: 6px 2px 8px; + border-radius: 12px; + border: 1px solid rgba(61, 139, 255, 0.18); + background: rgba(61, 139, 255, 0.08); + padding: 10px; + color: #d8d8df; + font-size: 12px; +} + +.phoneRemoteHeader { + display: flex; + justify-content: space-between; + gap: 8px; + color: #9ea8bd; + font-size: 10px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.06em; +} + +.phoneRemoteCode { + border-radius: 10px; + background: rgba(255, 255, 255, 0.08); + padding: 10px; + text-align: center; + color: #eeeef2; + font-size: 22px; + font-weight: 800; + letter-spacing: 0.12em; + user-select: text; +} + +.phoneRemoteQr { + display: flex; + justify-content: center; + border-radius: 12px; + background: rgba(255, 255, 255, 0.08); + padding: 10px; +} + +.phoneRemoteQr svg { + display: block; + width: min(100%, 176px); + height: auto; + border-radius: 8px; + box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.12); +} + +.phoneRemoteLink { + overflow-wrap: anywhere; + border-radius: 8px; + background: rgba(0, 0, 0, 0.18); + padding: 7px 8px; + color: #9cbfff; + font-size: 11px; + line-height: 1.35; + user-select: text; +} + +.phoneRemoteButton { + width: 100%; + border: 0; + border-radius: 9px; + background: #3d8bff; + padding: 8px 10px; + color: #fff; + font-size: 12px; + font-weight: 800; + cursor: pointer; +} + +.phoneRemoteButton:hover { + background: #62a4ff; +} + +.phoneRemoteWarning { + border-radius: 9px; + background: rgba(245, 158, 11, 0.12); + padding: 8px 9px; + color: #f8d28a; + font-size: 11px; + line-height: 1.35; +} + +.phoneRemoteDetail { + color: #a5a5b3; + font-size: 11px; + line-height: 1.35; +} + .finalizingState { display: inline-flex; align-items: center; diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index 49547069d..a518ced00 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -1,46 +1,49 @@ import { + ArrowClockwiseIcon, CaretUpIcon, + DotsThreeVerticalIcon, MicrophoneIcon, MicrophoneSlashIcon, MinusIcon, MonitorIcon, - DotsThreeVerticalIcon, TimerIcon, VideoCameraIcon, VideoCameraSlashIcon, XIcon, - ArrowClockwiseIcon, } from "@phosphor-icons/react"; import { AnimatePresence, motion } from "motion/react"; +import { useEffect, useMemo, useRef } from "react"; import { RxDragHandleDots2 } from "react-icons/rx"; +import { toast } from "sonner"; +import { Separator } from "@/components/ui/separator"; import { useScopedT } from "../../contexts/I18nContext"; -import { useHudBarDrag } from "./hooks/useHudBarDrag"; import { useMicrophoneDevices } from "../../hooks/useMicrophoneDevices"; -import { useLaunchWindowSystemState } from "./hooks/useLaunchWindowSystemState"; +import { usePhoneRemoteMedia } from "../../hooks/usePhoneRemoteMedia"; +import { useScreenRecorder } from "../../hooks/useScreenRecorder"; +import { useVideoDevices } from "../../hooks/useVideoDevices"; +import { createQrSvgPath } from "../../lib/simpleQr"; +import { Button } from "../ui/button"; +import { HudInteractionContext } from "./contexts/HudInteractionContext"; +import { canToggleFloatingWebcamPreview } from "./floatingWebcamPreview"; +import { useHudBarDrag } from "./hooks/useHudBarDrag"; import { useLaunchHudInteractionState } from "./hooks/useLaunchHudInteractionState"; import { useLaunchWindowActions } from "./hooks/useLaunchWindowActions"; +import { useLaunchWindowSystemState } from "./hooks/useLaunchWindowSystemState"; import { useRecordingTimer } from "./hooks/useRecordingTimer"; -import { useScreenRecorder } from "../../hooks/useScreenRecorder"; -import { useVideoDevices } from "../../hooks/useVideoDevices"; import { useWebcamPreviewOverlay } from "./hooks/useWebcamPreviewOverlay"; -import { - canToggleFloatingWebcamPreview, -} from "./floatingWebcamPreview"; -import { LaunchPopoverCoordinatorProvider, useLaunchPopoverCoordinator } from "./popovers/LaunchPopoverCoordinator"; +import styles from "./LaunchWindow.module.css"; import { CountdownPopover } from "./popovers/CountdownPopover"; +import { + LaunchPopoverCoordinatorProvider, + useLaunchPopoverCoordinator, +} from "./popovers/LaunchPopoverCoordinator"; import { MicPopover } from "./popovers/MicPopover"; import { MorePopover } from "./popovers/MorePopover"; import { ProjectPopover } from "./popovers/ProjectPopover"; import { SourcePopover } from "./popovers/SourcePopover"; import { WebcamPopover } from "./popovers/WebcamPopover"; -import { HudInteractionContext } from "./contexts/HudInteractionContext"; -import { MarqueeText } from "./SourceSelector"; -import styles from "./LaunchWindow.module.css"; - -import { Separator } from "@/components/ui/separator"; -import { Button } from "../ui/button"; import { RecordingControls } from "./RecordingControls"; -import { useEffect, useRef } from "react"; +import { MarqueeText } from "./SourceSelector"; const SHOW_DEV_UPDATE_PREVIEW = import.meta.env.DEV; @@ -55,6 +58,7 @@ export function LaunchWindow() { function LaunchWindowContent() { const t = useScopedT("launch"); const { openId, requestClose, requestOpen } = useLaunchPopoverCoordinator(); + const phoneRemote = usePhoneRemoteMedia(); const { recording, @@ -73,17 +77,40 @@ function LaunchWindowContent() { setSystemAudioEnabled, webcamEnabled, setWebcamEnabled, + webcamSource, + setWebcamSource, webcamDeviceId, setWebcamDeviceId, + phoneMicrophoneEnabled, + setPhoneMicrophoneEnabled, countdownDelay, setCountdownDelay, preparePermissions, - } = useScreenRecorder(); + } = useScreenRecorder({ + remoteWebcamStream: phoneRemote.videoActive ? phoneRemote.videoCaptureStream : null, + remoteMicrophoneStream: phoneRemote.micActive ? phoneRemote.audioStream : null, + }); const { elapsed, formatTime } = useRecordingTimer(recording, paused); const hudContentRef = useRef(null); const hudBarRef = useRef(null); - + const micPopoverOpen = openId === "mic"; + const webcamPopoverOpen = openId === "webcam"; + const isPhoneWebcamSelected = webcamEnabled && webcamSource === "phone"; + const isPhoneMicAvailable = phoneRemote.micActive && phoneMicrophoneEnabled; + const phoneJoinUrl = phoneRemote.session?.joinUrl ?? null; + const phoneJoinQr = useMemo( + () => (phoneJoinUrl ? createQrSvgPath(phoneJoinUrl) : null), + [phoneJoinUrl], + ); + const narratorMicrophoneActive = microphoneEnabled || isPhoneMicAvailable; + const narratorMicrophoneTitle = isPhoneMicAvailable + ? "Phone microphone active" + : phoneMicrophoneEnabled + ? "Phone microphone waiting" + : microphoneEnabled + ? t("recording.disableMicrophone") + : t("recording.enableMicrophone"); const { selectedSource, @@ -98,14 +125,14 @@ function LaunchWindowContent() { const showWebcamControls = webcamEnabled && !recording; const { devices, selectedDeviceId, setSelectedDeviceId } = useMicrophoneDevices( - microphoneEnabled || openId === "mic", + microphoneEnabled || micPopoverOpen, microphoneDeviceId, ); const { devices: videoDevices, selectedDeviceId: selectedVideoDeviceId, setSelectedDeviceId: setSelectedVideoDeviceId, - } = useVideoDevices(webcamEnabled || openId === "webcam"); + } = useVideoDevices(webcamSource === "local" && (webcamEnabled || webcamPopoverOpen)); const { hudOverlayMousePassthroughSupported, @@ -147,9 +174,11 @@ function LaunchWindowContent() { setRecordingWebcamPreviewNode, } = useWebcamPreviewOverlay({ webcamEnabled, + webcamSource, webcamDeviceId, + remotePreviewStream: phoneRemote.previewStream, showWebcamControls, - webcamPopoverOpen: openId === "webcam", + webcamPopoverOpen, hudOverlayMousePassthroughSupported, }); @@ -167,12 +196,13 @@ function LaunchWindowContent() { recordingWebcamPreviewContainerRef, }); - const { handleHudMouseEnter, handleHudMouseLeave, beginInteractiveHudAction } = useLaunchHudInteractionState({ - openId, - isHudDraggingRef, - isWebcamPreviewDraggingRef, - webcamPreviewDragStartRef, - }); + const { handleHudMouseEnter, handleHudMouseLeave, beginInteractiveHudAction } = + useLaunchHudInteractionState({ + openId, + isHudDraggingRef, + isWebcamPreviewDraggingRef, + webcamPreviewDragStartRef, + }); useEffect(() => { let mounted = true; @@ -196,13 +226,118 @@ function LaunchWindowContent() { ease: [0.22, 1, 0.36, 1] as const, }; + const phoneRemoteStatusLabel = (() => { + switch (phoneRemote.status) { + case "idle": + return "Not connected"; + case "waiting": + return "Waiting for phone"; + case "phone-connected": + return "Phone connected"; + case "preview-live": + return "Camera live"; + case "mic-active": + return "Mic active"; + case "reconnecting": + return "Reconnecting"; + case "disconnected": + return "Disconnected"; + case "error": + return "Needs attention"; + } + })(); + + const showRecordingMicrophoneStatus = () => { + if (isPhoneMicAvailable) { + toast.info("Phone microphone is recording."); + return; + } + + if (phoneMicrophoneEnabled) { + toast.info( + "Phone microphone is waiting or disconnected. Screen recording is continuing.", + ); + return; + } + + if (microphoneEnabled) { + toast.info("Laptop microphone is recording."); + return; + } + + toast.info("No narrator microphone is active."); + }; + + const togglePhoneMicrophone = () => { + if (!phoneMicrophoneEnabled && !phoneRemote.session) { + void selectPhoneAsCamera(); + return; + } + + const nextEnabled = !phoneMicrophoneEnabled; + setPhoneMicrophoneEnabled(nextEnabled); + if (nextEnabled) { + setMicrophoneEnabled(false); + } + }; + + const selectPhoneAsCamera = async () => { + if (recording) { + return; + } + + setWebcamSource("phone"); + setWebcamEnabled(true); + setPhoneMicrophoneEnabled(true); + setMicrophoneEnabled(false); + if (!phoneRemote.session) { + await phoneRemote.startSession(); + } + }; + + const selectLaptopWebcam = () => { + if (recording) { + return; + } + + setWebcamSource("local"); + setPhoneMicrophoneEnabled(false); + phoneRemote.stopSession(); + }; + + const handleRecordClick = () => { + if (!hasSelectedSource && platform !== "linux") { + beginInteractiveHudAction(); + requestOpen("sources"); + return; + } + + if ( + webcamSource === "phone" && + webcamEnabled && + !phoneRemote.videoActive && + !phoneRemote.micActive + ) { + const shouldContinue = window.confirm( + "Phone is not connected yet. Start recording the laptop screen without phone camera or phone mic?", + ); + if (!shouldContinue) { + beginInteractiveHudAction(); + requestOpen("webcam"); + return; + } + } + + toggleRecording(); + }; const recordingControls = ( setMicrophoneEnabled(!microphoneEnabled)} + microphoneTitle={narratorMicrophoneTitle} + onToggleMicrophone={showRecordingMicrophoneStatus} onPauseResume={paused ? resumeRecording : pauseRecording} onStopRecording={toggleRecording} onHideHud={() => window.electronAPI?.hudOverlayHide?.()} @@ -249,6 +384,10 @@ function LaunchWindowContent() { systemAudioEnabled={systemAudioEnabled} onToggleSystemAudio={() => setSystemAudioEnabled(!systemAudioEnabled)} microphoneEnabled={microphoneEnabled} + phoneMicrophoneEnabled={phoneMicrophoneEnabled} + isPhoneMicAvailable={isPhoneMicAvailable} + phoneRemoteStatusLabel={phoneRemoteStatusLabel} + onTogglePhoneMicrophone={togglePhoneMicrophone} onDisableMicrophone={() => setMicrophoneEnabled(false)} devices={devices} microphoneDeviceId={microphoneDeviceId} @@ -263,14 +402,14 @@ function LaunchWindowContent() { variant="ghost" size="icon" iconSize="lg" - title={ - microphoneEnabled - ? t("recording.disableMicrophone") - : t("recording.enableMicrophone") - } - className={microphoneEnabled ? styles.ibActive : ""} + title={narratorMicrophoneTitle} + className={narratorMicrophoneActive ? styles.ibActive : ""} > - {microphoneEnabled ? : } + {narratorMicrophoneActive ? ( + + ) : ( + + )} } /> @@ -278,20 +417,37 @@ function LaunchWindowContent() { setWebcamEnabled(false)} + webcamSource={webcamSource} + onDisableWebcam={() => { + setWebcamEnabled(false); + if (webcamSource === "phone") { + setPhoneMicrophoneEnabled(false); + phoneRemote.stopSession(); + } + }} canToggleFloatingPreview={canToggleFloatingWebcamPreview( hudOverlayMousePassthroughSupported, )} showFloatingWebcamPreview={showFloatingWebcamPreview} - onToggleFloatingPreview={() => - setShowFloatingWebcamPreview((current) => !current) - } + onToggleFloatingPreview={() => setShowFloatingWebcamPreview((current) => !current)} showWebcamControls={showWebcamControls} setWebcamPreviewNode={setWebcamPreviewNode} + isPhoneWebcamSelected={isPhoneWebcamSelected} + phoneRemoteStatusLabel={phoneRemoteStatusLabel} + phoneRemoteSession={phoneRemote.session} + phoneJoinQr={phoneJoinQr} + phoneRemoteMicActive={phoneRemote.micActive} + phoneRemoteSecureJoinReady={phoneRemote.secureJoinReady} + phoneRemoteError={phoneRemote.error} + phoneRemoteStatusDetail={phoneRemote.statusDetail} + onSelectPhoneAsCamera={() => void selectPhoneAsCamera()} + onSelectLaptopWebcam={selectLaptopWebcam} + onCopyPhoneJoinLink={() => void phoneRemote.copyJoinLink()} videoDevices={videoDevices} webcamDeviceId={webcamDeviceId} selectedVideoDeviceId={selectedVideoDeviceId} onSelectVideoDevice={(deviceId) => { + selectLaptopWebcam(); setWebcamEnabled(true); setSelectedVideoDeviceId(deviceId); setWebcamDeviceId(deviceId); @@ -308,7 +464,11 @@ function LaunchWindowContent() { } className={webcamEnabled ? styles.ibActive : ""} > - {webcamEnabled ? : } + {webcamEnabled ? ( + + ) : ( + + )} } /> @@ -329,18 +489,10 @@ function LaunchWindowContent() { } /> - } @@ -429,117 +576,121 @@ function LaunchWindowContent() { const hudMode = finalizing ? "finalizing" : recording ? "recording" : "idle"; return ( - +
-
- -
- -
- -
- - - {finalizing - ? finalizingControls - : recording - ? recordingControls - : idleControls} - - -
- -
- {showRecordingWebcamPreview && (
-
- )} + {showRecordingWebcamPreview && ( +
+
+ )} +
-
- ); } diff --git a/src/components/launch/RecordingControls.tsx b/src/components/launch/RecordingControls.tsx index 4e24f943a..d55b34540 100644 --- a/src/components/launch/RecordingControls.tsx +++ b/src/components/launch/RecordingControls.tsx @@ -1,13 +1,22 @@ -import { MicrophoneIcon, MicrophoneSlashIcon, MinusIcon, PauseIcon, PlayIcon, SquareIcon, XIcon } from "@phosphor-icons/react"; +import { + MicrophoneIcon, + MicrophoneSlashIcon, + MinusIcon, + PauseIcon, + PlayIcon, + SquareIcon, + XIcon, +} from "@phosphor-icons/react"; import { useMemo } from "react"; -import { useScopedT } from "@/contexts/I18nContext"; import { Button } from "@/components/ui/button"; import { Separator } from "@/components/ui/separator"; +import { useScopedT } from "@/contexts/I18nContext"; import styles from "./LaunchWindow.module.css"; interface RecordingControlsProps { paused: boolean; microphoneEnabled: boolean; + microphoneTitle?: string; elapsed: number; onToggleMicrophone: () => void; onPauseResume: () => void; @@ -20,6 +29,7 @@ interface RecordingControlsProps { export const RecordingControls = ({ paused, microphoneEnabled, + microphoneTitle, elapsed, onToggleMicrophone, onPauseResume, @@ -58,13 +68,13 @@ export const RecordingControls = ({ - + + {!phoneRemoteSecureJoinReady ? ( +
+ Phone browsers usually require HTTPS for camera and mic access. + A secure tunnel was unavailable, so use the LAN link only if + your browser allows it. +
+ ) : null} + {phoneRemoteError || phoneRemoteStatusDetail ? ( +
+ {phoneRemoteError ?? phoneRemoteStatusDetail} +
+ ) : null} + + ) : ( + + )} + + ) : null} + {webcamSource === "phone" ? ( + } onClick={onSelectLaptopWebcam}> + Use laptop webcam instead + + ) : null} {webcamEnabled && ( <> - } onClick={() => { - onDisableWebcam(); - requestClose(POPOVER_ID); - }}> + } + onClick={() => { + onDisableWebcam(); + requestClose(POPOVER_ID); + }} + > {t("recording.turnOffWebcam")} {canToggleFloatingPreview ? ( : } + icon={ + showFloatingWebcamPreview ? : + } selected={showFloatingWebcamPreview} onClick={onToggleFloatingPreview} > @@ -86,7 +196,9 @@ export function WebcamPopover({ )} {!webcamEnabled && ( -
{t("recording.selectWebcamToEnable")}
+
+ {t("recording.selectWebcamToEnable")} +
)} {showWebcamControls && (
@@ -96,33 +208,41 @@ export function WebcamPopover({ className="h-full w-full object-cover" muted playsInline - style={{ transform: "scaleX(-1)" }} + style={{ + transform: webcamSource === "local" ? "scaleX(-1)" : undefined, + }} />
)} - {videoDevices.map((device) => ( - - ) : ( - - ) - } - selected={ - webcamEnabled && - (webcamDeviceId === device.deviceId || selectedVideoDeviceId === device.deviceId) - } - onClick={() => onSelectVideoDevice(device.deviceId)} - > - {device.label} - - ))} - {videoDevices.length === 0 && ( -
{t("recording.noWebcamsFound")}
+ {webcamSource === "local" + ? videoDevices.map((device) => ( + + ) : ( + + ) + } + selected={ + webcamEnabled && + (webcamDeviceId === device.deviceId || + selectedVideoDeviceId === device.deviceId) + } + onClick={() => onSelectVideoDevice(device.deviceId)} + > + {device.label} + + )) + : null} + {webcamSource === "local" && videoDevices.length === 0 && ( +
+ {t("recording.noWebcamsFound")} +
)} ); diff --git a/src/components/ui/button.tsx b/src/components/ui/button.tsx index 2ea10d35c..4ad994787 100644 --- a/src/components/ui/button.tsx +++ b/src/components/ui/button.tsx @@ -64,4 +64,3 @@ const Button = React.forwardRef( Button.displayName = "Button"; export { Button, buttonVariants }; - diff --git a/src/components/video-editor/editorPreferences.test.ts b/src/components/video-editor/editorPreferences.test.ts index 4f80b660d..818618020 100644 --- a/src/components/video-editor/editorPreferences.test.ts +++ b/src/components/video-editor/editorPreferences.test.ts @@ -174,7 +174,6 @@ describe("editorPreferences", () => { backgroundBlur: 3.5, showCursor: false, aspectRatio: "native", - zoomInOverlapMs: 200, exportFormat: "gif", gifFrameRate: 30, gifLoop: false, @@ -212,7 +211,6 @@ describe("editorPreferences", () => { expect(loadEditorPreferences()).toEqual({ ...DEFAULT_EDITOR_PREFERENCES, aspectRatio: "16:9", - zoomInOverlapMs: 200, customAspectWidth: "21", customAspectHeight: "9", }); @@ -287,7 +285,6 @@ describe("editorPreferences", () => { backgroundBlur: 1.5, zoomMotionBlur: 0.75, connectZooms: false, - zoomInOverlapMs: 200, showCursor: false, loopCursor: true, cursorStyle: "figma", @@ -301,6 +298,7 @@ describe("editorPreferences", () => { padding: { top: 30, right: 30, bottom: 30, left: 30, linked: true }, aspectRatio: "4:5", exportEncodingMode: "quality", + exportQuality: "source", exportFormat: "gif", gifFrameRate: 20, gifLoop: false, diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index bb3d00dfe..b632cb23b 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -419,7 +419,7 @@ export function normalizeProjectEditor(editor: Partial): Pro : DEFAULT_MOTION_PRESET.zoomInDurationMs; const normalizedZoomInOverlapMs = isFiniteNumber(editor.zoomInOverlapMs) ? clamp(editor.zoomInOverlapMs, 0, normalizedZoomInDurationMs) - : DEFAULT_ZOOM_IN_OVERLAP_MS; + : clamp(DEFAULT_ZOOM_IN_OVERLAP_MS, 0, normalizedZoomInDurationMs); const normalizedZoomOutDurationMs = isFiniteNumber(editor.zoomOutDurationMs) ? clamp(editor.zoomOutDurationMs, 60, 4000) : DEFAULT_MOTION_PRESET.zoomOutDurationMs; diff --git a/src/hooks/usePhoneRemoteMedia.ts b/src/hooks/usePhoneRemoteMedia.ts new file mode 100644 index 000000000..c1b1dfe95 --- /dev/null +++ b/src/hooks/usePhoneRemoteMedia.ts @@ -0,0 +1,595 @@ +import { useCallback, useEffect, useRef, useState } from "react"; + +const PHONE_VIDEO_WIDTH = 1280; +const PHONE_VIDEO_HEIGHT = 720; +const PHONE_VIDEO_FRAME_RATE = 30; +const REMOTE_STALE_TIMEOUT_MS = 5000; + +type PhoneRemoteConnectionState = + | "idle" + | "waiting" + | "phone-connected" + | "preview-live" + | "mic-active" + | "reconnecting" + | "disconnected" + | "error"; + +type PhoneRemoteSignalMessage = RendererPhoneRemoteSignalMessage; +type PhoneRemoteStatusMessage = RendererPhoneRemoteStatusMessage; + +type UsePhoneRemoteMediaReturn = { + session: RendererPhoneRemoteSession | null; + status: PhoneRemoteConnectionState; + statusDetail: string | null; + error: string | null; + previewStream: MediaStream | null; + videoCaptureStream: MediaStream | null; + audioStream: MediaStream | null; + micActive: boolean; + videoActive: boolean; + secureJoinReady: boolean; + startSession: () => Promise; + stopSession: () => void; + copyJoinLink: () => Promise; +}; + +function isPhoneSignalMessage(value: unknown): value is PhoneRemoteSignalMessage { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return false; + } + + const record = value as Record; + if (record.type === "offer" || record.type === "answer") { + const description = record.description as Record | null; + return ( + Boolean(description) && + (description?.type === "offer" || description?.type === "answer") && + typeof description?.sdp === "string" + ); + } + + if (record.type === "ice-candidate") { + return record.candidate === null || typeof record.candidate === "object"; + } + + return false; +} + +function normalizeStatus(status: PhoneRemoteStatusMessage["status"]): PhoneRemoteConnectionState { + switch (status) { + case "waiting": + case "phone-connected": + case "preview-live": + case "mic-active": + case "reconnecting": + case "disconnected": + return status; + case "camera-permission-denied": + case "microphone-permission-denied": + case "no-audio-track": + case "phone-backgrounded": + case "phone-sleeping": + return "reconnecting"; + case "error": + return "error"; + } +} + +function drawCoverImage( + context: CanvasRenderingContext2D, + video: HTMLVideoElement, + width: number, + height: number, +) { + const sourceWidth = video.videoWidth; + const sourceHeight = video.videoHeight; + if (sourceWidth <= 0 || sourceHeight <= 0) { + return; + } + + const scale = Math.max(width / sourceWidth, height / sourceHeight); + const drawWidth = sourceWidth * scale; + const drawHeight = sourceHeight * scale; + const drawX = (width - drawWidth) / 2; + const drawY = (height - drawHeight) / 2; + context.drawImage(video, drawX, drawY, drawWidth, drawHeight); +} + +export function usePhoneRemoteMedia(): UsePhoneRemoteMediaReturn { + const [session, setSession] = useState(null); + const [status, setStatus] = useState("idle"); + const [statusDetail, setStatusDetail] = useState(null); + const [error, setError] = useState(null); + const [previewStream, setPreviewStream] = useState(null); + const [videoCaptureStream, setVideoCaptureStream] = useState(null); + const [audioStream, setAudioStream] = useState(null); + const [micActive, setMicActive] = useState(false); + const [videoActive, setVideoActive] = useState(false); + const sessionRef = useRef(null); + const peerRef = useRef(null); + const remoteStreamRef = useRef(null); + const videoCaptureStreamRef = useRef(null); + const canvasRef = useRef(null); + const canvasVideoRef = useRef(null); + const drawFrameRef = useRef(null); + const audioContextRef = useRef(null); + const audioSourceRef = useRef(null); + const audioDestinationRef = useRef(null); + const remoteAudioTrackRef = useRef(null); + const remoteVideoTrackRef = useRef(null); + const pendingRemoteIceCandidatesRef = useRef([]); + const pendingLocalIceSignalsRef = useRef([]); + const offerSentRef = useRef(false); + const remoteStaleTimerRef = useRef(null); + + const setCurrentSession = useCallback((nextSession: RendererPhoneRemoteSession | null) => { + sessionRef.current = nextSession; + setSession(nextSession); + }, []); + + const sendSignal = useCallback(async (message: PhoneRemoteSignalMessage) => { + const currentSession = sessionRef.current; + if (!currentSession) { + return; + } + + const result = await window.electronAPI.sendPhoneRemoteSignal(currentSession.id, message); + if (!result.success) { + throw new Error(result.error ?? "Failed to send phone signal."); + } + }, []); + + const clearRemoteStaleTimer = useCallback(() => { + if (remoteStaleTimerRef.current !== null) { + window.clearTimeout(remoteStaleTimerRef.current); + remoteStaleTimerRef.current = null; + } + }, []); + + const armRemoteStaleTimer = useCallback(() => { + clearRemoteStaleTimer(); + remoteStaleTimerRef.current = window.setTimeout(() => { + setStatus((current) => + current === "preview-live" || current === "mic-active" ? "reconnecting" : current, + ); + setStatusDetail("Phone media has stalled. Screen recording can continue safely."); + }, REMOTE_STALE_TIMEOUT_MS); + }, [clearRemoteStaleTimer]); + + const ensureVideoCaptureStream = useCallback(() => { + if (videoCaptureStreamRef.current) { + return videoCaptureStreamRef.current; + } + + const canvas = document.createElement("canvas"); + canvas.width = PHONE_VIDEO_WIDTH; + canvas.height = PHONE_VIDEO_HEIGHT; + const context = canvas.getContext("2d"); + if (!context) { + throw new Error("Unable to prepare the phone camera canvas."); + } + + const video = document.createElement("video"); + video.muted = true; + video.playsInline = true; + video.autoplay = true; + canvasRef.current = canvas; + canvasVideoRef.current = video; + + const draw = () => { + context.fillStyle = "#0d0d12"; + context.fillRect(0, 0, PHONE_VIDEO_WIDTH, PHONE_VIDEO_HEIGHT); + const hasUsableVideo = + video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA && + video.videoWidth > 0 && + video.videoHeight > 0; + + if (hasUsableVideo) { + drawCoverImage(context, video, PHONE_VIDEO_WIDTH, PHONE_VIDEO_HEIGHT); + } else { + context.fillStyle = "#777783"; + context.font = "600 34px sans-serif"; + context.textAlign = "center"; + context.fillText( + "Phone camera waiting", + PHONE_VIDEO_WIDTH / 2, + PHONE_VIDEO_HEIGHT / 2, + ); + } + + drawFrameRef.current = window.requestAnimationFrame(draw); + }; + + drawFrameRef.current = window.requestAnimationFrame(draw); + const stream = canvas.captureStream(PHONE_VIDEO_FRAME_RATE); + videoCaptureStreamRef.current = stream; + setVideoCaptureStream(stream); + return stream; + }, []); + + const ensureAudioDestination = useCallback(() => { + if (audioDestinationRef.current) { + return audioDestinationRef.current; + } + + const context = new AudioContext({ sampleRate: 48000 }); + const destination = context.createMediaStreamDestination(); + audioContextRef.current = context; + audioDestinationRef.current = destination; + setAudioStream(destination.stream); + return destination; + }, []); + + const attachRemoteAudioTrack = useCallback( + (track: MediaStreamTrack) => { + remoteAudioTrackRef.current = track; + const destination = ensureAudioDestination(); + audioSourceRef.current?.disconnect(); + const context = audioContextRef.current; + if (!context) { + return; + } + + const source = context.createMediaStreamSource(new MediaStream([track])); + source.connect(destination); + audioSourceRef.current = source; + context.resume().catch(() => undefined); + + const updateAudioState = () => { + const active = track.readyState === "live" && track.enabled; + setMicActive(active); + if (active) { + setStatus("mic-active"); + setStatusDetail("Phone microphone is active."); + } + }; + + track.onmute = updateAudioState; + track.onunmute = updateAudioState; + track.onended = () => { + if (remoteAudioTrackRef.current === track) { + remoteAudioTrackRef.current = null; + } + setMicActive(false); + setStatus("reconnecting"); + setStatusDetail("Phone microphone track ended."); + }; + updateAudioState(); + }, + [ensureAudioDestination], + ); + + const attachRemoteStream = useCallback( + (stream: MediaStream) => { + remoteStreamRef.current = stream; + setPreviewStream(stream); + ensureVideoCaptureStream(); + + const video = canvasVideoRef.current; + if (video && video.srcObject !== stream) { + video.srcObject = stream; + video.play().catch(() => undefined); + } + + const videoTrack = stream.getVideoTracks()[0]; + if (videoTrack) { + remoteVideoTrackRef.current = videoTrack; + setVideoActive(videoTrack.readyState === "live"); + videoTrack.onmute = () => { + setVideoActive(false); + setStatus("reconnecting"); + setStatusDetail("Phone camera track muted."); + armRemoteStaleTimer(); + }; + videoTrack.onunmute = () => { + setVideoActive(true); + setStatus("preview-live"); + setStatusDetail("Phone camera preview is live."); + armRemoteStaleTimer(); + }; + videoTrack.onended = () => { + if (remoteVideoTrackRef.current === videoTrack) { + remoteVideoTrackRef.current = null; + } + setVideoActive(false); + setStatus("reconnecting"); + setStatusDetail("Phone camera track ended."); + armRemoteStaleTimer(); + }; + } + + const audioTrack = stream.getAudioTracks()[0]; + if (audioTrack) { + attachRemoteAudioTrack(audioTrack); + } else { + setMicActive(false); + } + + if (videoTrack || audioTrack) { + setStatus(videoTrack ? "preview-live" : "mic-active"); + setStatusDetail( + videoTrack ? "Phone camera preview is live." : "Phone microphone is active.", + ); + armRemoteStaleTimer(); + } + }, + [armRemoteStaleTimer, attachRemoteAudioTrack, ensureVideoCaptureStream], + ); + + const closePeer = useCallback(() => { + const peer = peerRef.current; + peerRef.current = null; + if (peer) { + peer.onicecandidate = null; + peer.ontrack = null; + peer.onconnectionstatechange = null; + peer.close(); + } + pendingRemoteIceCandidatesRef.current = []; + pendingLocalIceSignalsRef.current = []; + offerSentRef.current = false; + }, []); + + const createPeer = useCallback(() => { + closePeer(); + const peer = new RTCPeerConnection({ + iceServers: [ + { urls: "stun:stun.l.google.com:19302" }, + { urls: "stun:stun1.l.google.com:19302" }, + { urls: "stun:stun2.l.google.com:19302" }, + ], + }); + + peer.addTransceiver("video", { direction: "recvonly" }); + peer.addTransceiver("audio", { direction: "recvonly" }); + peer.onicecandidate = (event) => { + const message: PhoneRemoteSignalMessage = { + type: "ice-candidate", + candidate: event.candidate ? event.candidate.toJSON() : null, + }; + + if (!offerSentRef.current) { + pendingLocalIceSignalsRef.current.push(message); + return; + } + + sendSignal(message).catch((signalError) => { + console.warn("Failed to send phone remote ICE candidate:", signalError); + }); + }; + peer.ontrack = (event) => { + const stream = event.streams[0] ?? remoteStreamRef.current ?? new MediaStream(); + if (!event.streams[0] && event.track) { + stream.addTrack(event.track); + } + attachRemoteStream(stream); + }; + peer.onconnectionstatechange = () => { + switch (peer.connectionState) { + case "connected": + setStatus("phone-connected"); + setStatusDetail("Phone is connected."); + break; + case "disconnected": + case "failed": + setStatus("reconnecting"); + setStatusDetail("Phone connection dropped. Screen recording can continue."); + break; + case "closed": + setStatus("disconnected"); + setStatusDetail("Phone disconnected."); + break; + default: + break; + } + }; + peerRef.current = peer; + return peer; + }, [attachRemoteStream, closePeer, sendSignal]); + + const stopSession = useCallback(() => { + const currentSession = sessionRef.current; + if (currentSession) { + void window.electronAPI.endPhoneRemoteSession(currentSession.id); + } + closePeer(); + clearRemoteStaleTimer(); + audioSourceRef.current?.disconnect(); + audioSourceRef.current = null; + audioContextRef.current?.close().catch(() => undefined); + audioContextRef.current = null; + audioDestinationRef.current = null; + remoteAudioTrackRef.current = null; + remoteVideoTrackRef.current = null; + pendingRemoteIceCandidatesRef.current = []; + pendingLocalIceSignalsRef.current = []; + offerSentRef.current = false; + if (drawFrameRef.current !== null) { + window.cancelAnimationFrame(drawFrameRef.current); + drawFrameRef.current = null; + } + videoCaptureStreamRef.current?.getTracks().forEach((track) => track.stop()); + videoCaptureStreamRef.current = null; + remoteStreamRef.current = null; + canvasVideoRef.current = null; + canvasRef.current = null; + setCurrentSession(null); + setStatus("idle"); + setStatusDetail(null); + setError(null); + setPreviewStream(null); + setVideoCaptureStream(null); + setAudioStream(null); + setMicActive(false); + setVideoActive(false); + }, [clearRemoteStaleTimer, closePeer, setCurrentSession]); + + const startSession = useCallback(async () => { + setStatus("waiting"); + setStatusDetail("Creating phone connection."); + setError(null); + + try { + ensureVideoCaptureStream(); + ensureAudioDestination(); + const result = await window.electronAPI.createPhoneRemoteSession(); + if (!result.success || !result.session) { + throw new Error(result.error ?? "Failed to create phone session."); + } + + setCurrentSession(result.session); + const peer = createPeer(); + const offer = await peer.createOffer(); + await peer.setLocalDescription(offer); + await sendSignal({ type: "offer", description: offer }); + offerSentRef.current = true; + for (const candidateSignal of pendingLocalIceSignalsRef.current.splice(0)) { + await sendSignal(candidateSignal); + } + setStatus("waiting"); + setStatusDetail("Waiting for phone to connect."); + } catch (startError) { + const message = startError instanceof Error ? startError.message : String(startError); + setError(message); + setStatus("error"); + setStatusDetail(message); + } + }, [ + createPeer, + ensureAudioDestination, + ensureVideoCaptureStream, + sendSignal, + setCurrentSession, + ]); + + const copyJoinLink = useCallback(async () => { + const joinUrl = sessionRef.current?.joinUrl; + if (!joinUrl) { + return false; + } + + try { + await navigator.clipboard.writeText(joinUrl); + return true; + } catch { + return false; + } + }, []); + + useEffect(() => { + let active = true; + const removeSignalListener = window.electronAPI.onPhoneRemoteSignal(async (payload) => { + if (!active) { + return; + } + const currentSession = sessionRef.current; + if (!currentSession || payload.sessionId !== currentSession.id) { + return; + } + if (!isPhoneSignalMessage(payload.message)) { + return; + } + + const peer = peerRef.current; + if (!peer) { + return; + } + + try { + if (payload.message.type === "answer") { + if (peer.signalingState !== "have-local-offer") { + return; + } + await peer.setRemoteDescription(payload.message.description); + if (!active) { + return; + } + for (const candidate of pendingRemoteIceCandidatesRef.current.splice(0)) { + try { + await peer.addIceCandidate(candidate); + } catch (candidateError) { + console.warn( + "Failed to apply queued phone ICE candidate:", + candidateError, + ); + } + } + return; + } + if (payload.message.type === "ice-candidate" && payload.message.candidate) { + if (!peer.remoteDescription) { + pendingRemoteIceCandidatesRef.current.push(payload.message.candidate); + return; + } + await peer.addIceCandidate(payload.message.candidate); + } + } catch (signalError) { + if (!active) { + return; + } + console.warn("Failed to apply phone remote signal:", signalError); + setStatus("reconnecting"); + setStatusDetail("Phone signal failed. Waiting for reconnect."); + } + }); + + const removeStatusListener = window.electronAPI.onPhoneRemoteStatus((payload) => { + const currentSession = sessionRef.current; + if (!currentSession || payload.sessionId !== currentSession.id) { + return; + } + + setStatus(normalizeStatus(payload.status.status)); + setStatusDetail(payload.status.detail ?? null); + const actualAudioActive = + remoteAudioTrackRef.current?.readyState === "live" && + remoteAudioTrackRef.current.enabled; + const actualVideoActive = + remoteVideoTrackRef.current?.readyState === "live" && + !remoteVideoTrackRef.current.muted; + + if (typeof payload.status.hasAudio === "boolean") { + setMicActive(payload.status.hasAudio ? Boolean(actualAudioActive) : false); + } + if (typeof payload.status.hasVideo === "boolean") { + setVideoActive(payload.status.hasVideo ? Boolean(actualVideoActive) : false); + } + if ( + payload.status.status === "preview-live" || + payload.status.status === "mic-active" || + payload.status.status === "phone-connected" + ) { + armRemoteStaleTimer(); + } + }); + + return () => { + active = false; + removeSignalListener(); + removeStatusListener(); + }; + }, [armRemoteStaleTimer]); + + useEffect(() => { + return () => { + stopSession(); + }; + }, [stopSession]); + + return { + session, + status, + statusDetail, + error, + previewStream, + videoCaptureStream, + audioStream, + micActive, + videoActive, + secureJoinReady: session?.urlMode === "secure-tunnel", + startSession, + stopSession, + copyJoinLink, + }; +} diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 6f021761a..b852f6a9f 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -38,8 +38,10 @@ const WEBCAM_WIDTH = 1280; const WEBCAM_HEIGHT = 720; const WEBCAM_FRAME_RATE = 30; const WEBCAM_SUFFIX = "-webcam"; +const SOURCE_AUDIO_MUX_TOAST_ID = "recording-audio-mux-warning"; const MICROPHONE_FALLBACK_ERROR_TOAST_ID = "recording-microphone-fallback-error"; const MICROPHONE_SIDECAR_ERROR_TOAST_ID = "recording-microphone-sidecar-error"; +const DUAL_MIC_WARNING_TOAST_ID = "recording-dual-mic-warning"; export type BrowserMicrophoneProfile = | "processed" | "no-agc" @@ -121,6 +123,13 @@ type DesktopCaptureMediaDevices = { getDisplayMedia: (constraints: unknown) => Promise; }; +type WebcamSource = "local" | "phone"; + +type UseScreenRecorderOptions = { + remoteWebcamStream?: MediaStream | null; + remoteMicrophoneStream?: MediaStream | null; +}; + type UseScreenRecorderReturn = { recording: boolean; paused: boolean; @@ -140,8 +149,12 @@ type UseScreenRecorderReturn = { setSystemAudioEnabled: (enabled: boolean) => void; webcamEnabled: boolean; setWebcamEnabled: (enabled: boolean) => void; + webcamSource: WebcamSource; + setWebcamSource: (source: WebcamSource) => void; webcamDeviceId: string | undefined; setWebcamDeviceId: (deviceId: string | undefined) => void; + phoneMicrophoneEnabled: boolean; + setPhoneMicrophoneEnabled: (enabled: boolean) => void; countdownDelay: number; setCountdownDelay: (delay: number) => void; }; @@ -258,7 +271,10 @@ async function createAudioInputDeviceSnapshot(): Promise< return audioInputs.length > 0 ? audioInputs : null; } -export function useScreenRecorder(): UseScreenRecorderReturn { +export function useScreenRecorder({ + remoteWebcamStream, + remoteMicrophoneStream, +}: UseScreenRecorderOptions = {}): UseScreenRecorderReturn { const [recording, setRecording] = useState(false); const [paused, setPaused] = useState(false); const [starting, setStarting] = useState(false); @@ -269,7 +285,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const [microphoneDeviceId, setMicrophoneDeviceId] = useState(undefined); const [systemAudioEnabled, setSystemAudioEnabled] = useState(false); const [webcamEnabled, setWebcamEnabled] = useState(false); + const [webcamSource, setWebcamSource] = useState("local"); const [webcamDeviceId, setWebcamDeviceId] = useState(undefined); + const [phoneMicrophoneEnabled, setPhoneMicrophoneEnabled] = useState(false); const [countdownDelay, setCountdownDelayState] = useState(3); const mediaRecorder = useRef(null); const webcamRecorder = useRef(null); @@ -277,6 +295,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const screenStream = useRef(null); const microphoneStream = useRef(null); const webcamStream = useRef(null); + const webcamStreamOwnedByRecorder = useRef(true); const mixingContext = useRef(null); const chunks = useRef([]); const webcamChunks = useRef([]); @@ -509,8 +528,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } if (webcamStream.current) { - webcamStream.current.getTracks().forEach((track) => track.stop()); + if (webcamStreamOwnedByRecorder.current) { + webcamStream.current.getTracks().forEach((track) => track.stop()); + } webcamStream.current = null; + webcamStreamOwnedByRecorder.current = true; } if (mixingContext.current) { @@ -822,6 +844,67 @@ export function useScreenRecorder(): UseScreenRecorderReturn { [resetMicFallbackTimingDiagnostics], ); + const createMixedAudioStream = useCallback( + (inputs: Array<{ track: MediaStreamTrack; gain?: number }>) => { + const liveInputs = inputs.filter(({ track }) => track.readyState === "live"); + if (liveInputs.length === 0) { + return null; + } + + if ( + liveInputs.length === 1 && + (liveInputs[0].gain === undefined || liveInputs[0].gain === 1) + ) { + return new MediaStream([liveInputs[0].track]); + } + + const context = new AudioContext({ sampleRate: 48000 }); + mixingContext.current = context; + const destination = context.createMediaStreamDestination(); + for (const input of liveInputs) { + const source = context.createMediaStreamSource(new MediaStream([input.track])); + if (typeof input.gain === "number" && input.gain !== 1) { + const gain = context.createGain(); + gain.gain.value = input.gain; + source.connect(gain).connect(destination); + } else { + source.connect(destination); + } + } + + return destination.stream; + }, + [], + ); + + const startMicrophoneSidecarRecorder = useCallback( + (audioStreamForSidecar: MediaStream | null, mainStartedAt: number) => { + const audioTrack = audioStreamForSidecar?.getAudioTracks()[0]; + if (!audioTrack) { + return false; + } + + micFallbackChunks.current = []; + const recorder = new MediaRecorder(new MediaStream([audioTrack]), { + mimeType: "audio/webm;codecs=opus", + audioBitsPerSecond: AUDIO_BITRATE_VOICE, + }); + micFallbackRecorderMetadata.current = { + mimeType: recorder.mimeType, + audioBitsPerSecond: AUDIO_BITRATE_VOICE, + timesliceMs: RECORDER_TIMESLICE_MS, + }; + resetMicFallbackTimingDiagnostics(); + micFallbackRecorderStartedAt.current = performance.now(); + recorder.ondataavailable = appendMicFallbackChunk; + micFallbackStartDelayMs.current = Math.max(0, Date.now() - mainStartedAt); + recorder.start(RECORDER_TIMESLICE_MS); + micFallbackRecorder.current = recorder; + return true; + }, + [appendMicFallbackChunk, resetMicFallbackTimingDiagnostics], + ); + const stopWebcamRecorder = useCallback(async () => { const recorder = webcamRecorder.current; const pending = webcamStopPromise.current; @@ -866,8 +949,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { micFallbackBlobPromise ?? stopMicFallbackRecorder(); const webcamPath = await stopWebcamRecorder(); await storeMicrophoneSidecar(resolvedMicFallbackBlobPromise, result.path, startDelayMs); + cleanupCapturedMedia(); await finalizeRecordingSession(result.path, webcamPath); - + if (typeof window.electronAPI?.hudOverlayClose === "function") { window.electronAPI.hudOverlayClose(); } @@ -875,6 +959,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return result.path; }, [ + cleanupCapturedMedia, finalizeRecordingSession, stopMicFallbackRecorder, stopWebcamRecorder, @@ -897,21 +982,41 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } try { - webcamStream.current = await navigator.mediaDevices.getUserMedia({ - video: webcamDeviceId - ? { - deviceId: { exact: webcamDeviceId }, - width: { ideal: WEBCAM_WIDTH }, - height: { ideal: WEBCAM_HEIGHT }, - frameRate: { ideal: WEBCAM_FRAME_RATE, max: WEBCAM_FRAME_RATE }, - } - : { - width: { ideal: WEBCAM_WIDTH }, - height: { ideal: WEBCAM_HEIGHT }, - frameRate: { ideal: WEBCAM_FRAME_RATE, max: WEBCAM_FRAME_RATE }, - }, - audio: false, - }); + if (webcamSource === "phone") { + const remoteVideoTrack = remoteWebcamStream?.getVideoTracks()[0]; + if (!remoteVideoTrack || remoteVideoTrack.readyState !== "live") { + console.warn( + "Phone webcam stream is not ready; continuing without webcam layer.", + ); + resolvedWebcamPath.current = null; + pendingWebcamPathPromise.current = Promise.resolve(null); + webcamStopPromise.current = Promise.resolve(null); + webcamRecorder.current = null; + webcamStartTime.current = null; + webcamTimeOffsetMs.current = 0; + return; + } + + webcamStream.current = new MediaStream([remoteVideoTrack]); + webcamStreamOwnedByRecorder.current = false; + } else { + webcamStream.current = await navigator.mediaDevices.getUserMedia({ + video: webcamDeviceId + ? { + deviceId: { exact: webcamDeviceId }, + width: { ideal: WEBCAM_WIDTH }, + height: { ideal: WEBCAM_HEIGHT }, + frameRate: { ideal: WEBCAM_FRAME_RATE, max: WEBCAM_FRAME_RATE }, + } + : { + width: { ideal: WEBCAM_WIDTH }, + height: { ideal: WEBCAM_HEIGHT }, + frameRate: { ideal: WEBCAM_FRAME_RATE, max: WEBCAM_FRAME_RATE }, + }, + audio: false, + }); + webcamStreamOwnedByRecorder.current = true; + } const mimeType = selectWebcamMimeType(); webcamChunks.current = []; @@ -973,9 +1078,12 @@ export function useScreenRecorder(): UseScreenRecorderReturn { webcamRecorder.current = null; webcamStartTime.current = null; if (webcamStream.current) { - webcamStream.current.getTracks().forEach((track) => track.stop()); + if (webcamStreamOwnedByRecorder.current) { + webcamStream.current.getTracks().forEach((track) => track.stop()); + } webcamStream.current = null; } + webcamStreamOwnedByRecorder.current = true; } }; } catch (error) { @@ -990,11 +1098,21 @@ export function useScreenRecorder(): UseScreenRecorderReturn { webcamStartTime.current = null; webcamTimeOffsetMs.current = 0; if (webcamStream.current) { - webcamStream.current.getTracks().forEach((track) => track.stop()); + if (webcamStreamOwnedByRecorder.current) { + webcamStream.current.getTracks().forEach((track) => track.stop()); + } webcamStream.current = null; } + webcamStreamOwnedByRecorder.current = true; } - }, [getRecordingDurationMs, selectWebcamMimeType, webcamDeviceId, webcamEnabled]); + }, [ + getRecordingDurationMs, + remoteWebcamStream, + selectWebcamMimeType, + webcamDeviceId, + webcamEnabled, + webcamSource, + ]); /** Start the prepared webcam MediaRecorder. Call after main recording begins. */ const beginWebcamCapture = useCallback(() => { @@ -1072,52 +1190,69 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // We pass null for webcamPath initially to avoid blocking on webcam disk writes/muxing. await finalizeRecordingSession(finalPath, null); - // 2. Perform background finalization (webcam, muxing, sidecars) - // We don't await this to keep the UI responsive - void (async () => { - try { - // Await the webcam path in the background - const webcamPath = await webcamPathPromise; - console.log("[useScreenRecorder] Background native processing: webcamPath is", webcamPath); - - // Store sidecars - await storeMicrophoneSidecar( - micFallbackBlobPromise, - finalPath, - fallbackStartDelayMs, - fallbackTrackSettings, + // 2. Perform background finalization (webcam, muxing, sidecars). + // We don't await this to keep the UI responsive. + void (async () => { + try { + const webcamPath = await webcamPathPromise; + let finalizedPath = finalPath; + console.log( + "[useScreenRecorder] Background native processing: webcamPath is", + webcamPath, + ); + + if (isNativeWindows) { + const muxResult = + await window.electronAPI.muxNativeWindowsRecording(expectedDurationMs); + if (!muxResult?.success || !muxResult.path) { + void logNativeCaptureDiagnostics("mux-native-windows-recording"); + const warningMessage = + muxResult?.error || + muxResult?.message || + "Failed to finish the native Windows audio mux"; + toast.warning( + `${warningMessage}. Recording was saved, but audio playback or export may be incomplete.`, + { id: SOURCE_AUDIO_MUX_TOAST_ID, duration: 10000 }, ); + finalizedPath = muxResult?.path ?? finalizedPath; + } else { + finalizedPath = muxResult.path; + } + } - // Perform muxing/renaming if on Windows - if (isNativeWindows) { - await window.electronAPI.muxNativeWindowsRecording(expectedDurationMs); - } + await storeMicrophoneSidecar( + micFallbackBlobPromise, + finalizedPath, + fallbackStartDelayMs, + fallbackTrackSettings, + ); - console.log("[useScreenRecorder] Emitting setCurrentRecordingSession with:", { finalPath, webcamPath }); + console.log("[useScreenRecorder] Emitting setCurrentRecordingSession with:", { + finalPath: finalizedPath, + webcamPath, + }); - // Update the session state to notify the editor that all background assets (webcam, mic, etc.) are now ready. - // This broadcasts a 'recording-session-changed' event that the open editor listens to for re-scanning assets. - await window.electronAPI.setCurrentRecordingSession({ - videoPath: finalPath, - webcamPath, - timeOffsetMs: webcamTimeOffsetMs.current, - hideOverlayCursorByDefault: hideEditorOverlayCursorByDefault.current, - }); + // Notify the editor that background assets (webcam, mic, muxed video) are ready. + await window.electronAPI.setCurrentRecordingSession({ + videoPath: finalizedPath, + webcamPath, + timeOffsetMs: webcamTimeOffsetMs.current, + hideOverlayCursorByDefault: hideEditorOverlayCursorByDefault.current, + }); - console.log( - `[PERF:RENDERER] Background Stop Sequence: COMPLETED in ${(performance.now() - stopStart).toFixed(2)}ms`, - ); - } catch (bgError) { - console.error("Error in background finalization:", bgError); - } finally { - // After all background tasks are done (webcam, mic sidecars, muxing), - // we can safely close the HUD window to release hardware and resources. - if (typeof window.electronAPI?.hudOverlayClose === "function") { - console.log("[useScreenRecorder] All background tasks finished, closing HUD"); - window.electronAPI.hudOverlayClose(); - } - } - })(); + console.log( + `[PERF:RENDERER] Background Stop Sequence: COMPLETED in ${(performance.now() - stopStart).toFixed(2)}ms`, + ); + } catch (bgError) { + console.error("Error in background finalization:", bgError); + } finally { + cleanupCapturedMedia(); + if (typeof window.electronAPI?.hudOverlayClose === "function") { + console.log("[useScreenRecorder] All background tasks finished, closing HUD"); + window.electronAPI.hudOverlayClose(); + } + } + })(); })(); return; } @@ -1324,6 +1459,18 @@ export function useScreenRecorder(): UseScreenRecorderReturn { recordingSessionTimestamp.current = Date.now(); resetRecordingClock(recordingSessionTimestamp.current); await prepareWebcamRecorder(); + const phoneMicrophoneTrack = + phoneMicrophoneEnabled && remoteMicrophoneStream + ? remoteMicrophoneStream + .getAudioTracks() + .find((track) => track.readyState === "live") + : undefined; + if (phoneMicrophoneTrack && microphoneEnabled) { + toast.warning( + "Both phone mic and laptop mic are enabled. This can cause echo or doubled voice.", + { id: DUAL_MIC_WARNING_TOAST_ID, duration: 10000 }, + ); + } const useNativeMacScreenCapture = platform === "darwin" && (selectedSource.id?.startsWith("screen:") || @@ -1423,8 +1570,18 @@ export function useScreenRecorder(): UseScreenRecorderReturn { ? 0 : webcamStartTime.current - mainStartedAt; + const sidecarAudioInputs: Array<{ track: MediaStreamTrack; gain?: number }> = + []; + if (phoneMicrophoneTrack) { + sidecarAudioInputs.push({ + track: phoneMicrophoneTrack, + gain: MIC_GAIN_BOOST, + }); + } + // When native mic capture is unavailable or explicitly bypassed, - // record mic via browser getUserMedia as a sidecar file. + // record mic via browser getUserMedia so it can be mixed into + // the microphone sidecar with phone audio when enabled. if (nativeResult.microphoneFallbackRequired && microphoneEnabled) { void logNativeCaptureDiagnostics("start-browser-microphone-fallback"); console.info("Using browser microphone processing for this recording."); @@ -1434,10 +1591,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn { browserMicrophoneProfile.current, ); micFallbackRequestedConstraints.current = microphoneConstraints; - const micStream = + microphoneStream.current = await navigator.mediaDevices.getUserMedia(microphoneConstraints); micFallbackTrackSettings.current = - createMicrophoneTrackSettingsSnapshot(micStream); + createMicrophoneTrackSettingsSnapshot(microphoneStream.current); micFallbackAudioInputDevices.current = await createAudioInputDeviceSnapshot().catch(() => null); console.info( @@ -1448,25 +1605,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn { "Browser microphone audio input devices:", micFallbackAudioInputDevices.current, ); - micFallbackChunks.current = []; - const recorder = new MediaRecorder(micStream, { - mimeType: "audio/webm;codecs=opus", - audioBitsPerSecond: AUDIO_BITRATE_VOICE, - }); - micFallbackRecorderMetadata.current = { - mimeType: recorder.mimeType, - audioBitsPerSecond: AUDIO_BITRATE_VOICE, - timesliceMs: RECORDER_TIMESLICE_MS, - }; - resetMicFallbackTimingDiagnostics(); - micFallbackRecorderStartedAt.current = performance.now(); - recorder.ondataavailable = appendMicFallbackChunk; - micFallbackStartDelayMs.current = Math.max( - 0, - Date.now() - mainStartedAt, - ); - recorder.start(RECORDER_TIMESLICE_MS); - micFallbackRecorder.current = recorder; + const localMicTrack = microphoneStream.current.getAudioTracks()[0]; + if (localMicTrack) { + sidecarAudioInputs.push({ + track: localMicTrack, + gain: MIC_GAIN_BOOST, + }); + } } catch (micError) { micFallbackStartDelayMs.current = null; micFallbackTrackSettings.current = null; @@ -1488,6 +1633,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } } + if (sidecarAudioInputs.length > 0) { + const sidecarStream = createMixedAudioStream(sidecarAudioInputs); + if (!startMicrophoneSidecarRecorder(sidecarStream, mainStartedAt)) { + micFallbackStartDelayMs.current = null; + } + } + setRecording(true); window.electronAPI?.setRecordingState(true); @@ -1497,7 +1649,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { hideEditorOverlayCursorByDefault.current = true; - const wantsAudioCapture = microphoneEnabled || systemAudioEnabled; + const wantsAudioCapture = + microphoneEnabled || systemAudioEnabled || Boolean(phoneMicrophoneTrack); const browserCaptureSource = await resolveBrowserCaptureSource(selectedSource); if ( @@ -1532,13 +1685,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { cursor: "never" as const, }; - if (wantsAudioCapture) { - let screenMediaStream: MediaStream; - const acquireLinuxPortalStream = (withAudio: boolean) => - mediaDevices.getDisplayMedia({ + const acquireLinuxPortalStream = async (withAudio: boolean): Promise => { + try { + return await mediaDevices.getDisplayMedia({ audio: withAudio, video: { - displaySurface: "monitor", + displaySurface: selectedSource.id?.startsWith("window:") + ? "window" + : "monitor", width: { ideal: TARGET_WIDTH, max: TARGET_WIDTH }, height: { ideal: TARGET_HEIGHT, max: TARGET_HEIGHT }, frameRate: { ideal: TARGET_FRAME_RATE, max: TARGET_FRAME_RATE }, @@ -1547,6 +1701,43 @@ export function useScreenRecorder(): UseScreenRecorderReturn { selfBrowserSurface: "exclude", surfaceSwitching: "exclude", }); + } catch (err) { + console.warn( + "Linux portal failed, falling back to desktop capture(no audio):", + err, + ); + if (withAudio) { + alert( + "System audio is not supported in fallback mode. Recording will continue without audio.", + ); + } + + const sources = await window.electronAPI.getSources({ types: ["screen"] }); + + if (!sources.length) { + throw new Error("No screen sources available"); + } + + const source = sources[0]; + console.log("Using fallback source:", source); + + return await navigator.mediaDevices.getUserMedia({ + audio: false, //intentional + video: { + mandatory: { + chromeMediaSource: "desktop", + chromeMediaSourceId: source.id, + maxWidth: TARGET_WIDTH, + maxHeight: TARGET_HEIGHT, + maxFrameRate: TARGET_FRAME_RATE, + }, + }, + } as unknown as MediaStreamConstraints); + } + }; + + if (wantsAudioCapture) { + let screenMediaStream: MediaStream; if (systemAudioEnabled) { try { @@ -1614,55 +1805,32 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const systemAudioTrack = screenMediaStream.getAudioTracks()[0]; const micAudioTrack = microphoneStream.current?.getAudioTracks()[0]; + const audioInputs: Array<{ track: MediaStreamTrack; gain?: number }> = []; + if (systemAudioTrack) { + audioInputs.push({ track: systemAudioTrack }); + } + if (phoneMicrophoneTrack) { + audioInputs.push({ track: phoneMicrophoneTrack, gain: MIC_GAIN_BOOST }); + } + if (micAudioTrack) { + audioInputs.push({ track: micAudioTrack, gain: MIC_GAIN_BOOST }); + } - if (systemAudioTrack && micAudioTrack) { - const context = new AudioContext({ sampleRate: 48000 }); - mixingContext.current = context; - const systemSource = context.createMediaStreamSource( - new MediaStream([systemAudioTrack]), - ); - const micSource = context.createMediaStreamSource( - new MediaStream([micAudioTrack]), - ); - const micGain = context.createGain(); - micGain.gain.value = MIC_GAIN_BOOST; - const destination = context.createMediaStreamDestination(); - - systemSource.connect(destination); - micSource.connect(micGain).connect(destination); - - const mixedTrack = destination.stream.getAudioTracks()[0]; - if (mixedTrack) { - stream.current.addTrack(mixedTrack); - systemAudioIncluded = true; - } - } else if (systemAudioTrack) { - stream.current.addTrack(systemAudioTrack); - systemAudioIncluded = true; - } else if (micAudioTrack) { - stream.current.addTrack(micAudioTrack); + const mixedAudioStream = createMixedAudioStream(audioInputs); + const mixedAudioTrack = mixedAudioStream?.getAudioTracks()[0]; + if (mixedAudioTrack) { + stream.current.addTrack(mixedAudioTrack); + systemAudioIncluded = Boolean(systemAudioTrack); } } else { const mediaStream = useLinuxPortal - ? await mediaDevices.getDisplayMedia({ - audio: false, - video: { - displaySurface: selectedSource.id?.startsWith("window:") - ? "window" - : "monitor", - width: { ideal: TARGET_WIDTH, max: TARGET_WIDTH }, - height: { ideal: TARGET_HEIGHT, max: TARGET_HEIGHT }, - frameRate: { ideal: TARGET_FRAME_RATE, max: TARGET_FRAME_RATE }, - cursor: "never", - }, - selfBrowserSurface: "exclude", - surfaceSwitching: "exclude", - }) + ? await acquireLinuxPortalStream(false) : await mediaDevices.getUserMedia({ audio: false, video: browserScreenVideoConstraints, }); + screenStream.current = mediaStream; stream.current = mediaStream; videoTrack = mediaStream.getVideoTracks()[0]; } @@ -1931,8 +2099,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { webcamRecorder.current = null; webcamStartTime.current = null; webcamTimeOffsetMs.current = 0; - webcamStream.current?.getTracks().forEach((t) => t.stop()); + if (webcamStreamOwnedByRecorder.current) { + webcamStream.current?.getTracks().forEach((track) => track.stop()); + } webcamStream.current = null; + webcamStreamOwnedByRecorder.current = true; pendingWebcamPathPromise.current = null; resolvedWebcamPath.current = null; @@ -2010,8 +2181,12 @@ export function useScreenRecorder(): UseScreenRecorderReturn { setSystemAudioEnabled: persistSystemAudioEnabled, webcamEnabled, setWebcamEnabled, + webcamSource, + setWebcamSource, webcamDeviceId, setWebcamDeviceId, + phoneMicrophoneEnabled, + setPhoneMicrophoneEnabled, countdownDelay, setCountdownDelay, }; diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index 66a05e89c..567ae9e21 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -45,6 +45,11 @@ }, "backgroundBlur": "Desenfoque de fondo", "zoomMotionBlur": "Desenfoque de movimiento del zoom", + "temporalZoomMotionBlur": "Temporal Zoom Blur", + "temporalZoomMotionBlurDescription": "Control the shutter window and frame samples used by the newer zoom blur pass.", + "zoomMotionBlurSamples": "Blur Samples", + "zoomMotionBlurShutter": "Shutter", + "auto": "Auto", "connectZooms": "Conectar zooms", "connectZoomsDescription": "Suaviza regiones de zoom consecutivas convirtiéndolas en un movimiento continuo de cámara.", "autoApplyFreshRecordingZooms": "Aplicar automáticamente zooms a nuevas grabaciones", @@ -57,6 +62,20 @@ "zoomOutDescription": "Controla cómo sale la cámara de una región de zoom.", "connectedZoomTitle": "Entre zooms", "connectedZoomDescription": "Ajusta el deslizamiento entre regiones de zoom consecutivas cuando la conexión está activada.", + "motionPresetsTitle": "Motion Presets", + "motionPresetsZoomHint": "Zoom motion presets are available in Settings.", + "animationPresets": "Animation Presets", + "cursorMotionPresets": "Cursor Motion Presets", + "motionPresets": { + "focused": { + "label": "Focused", + "description": "Snappier motion for demos, walkthroughs, and everyday recordings." + }, + "smooth": { + "label": "Smooth", + "description": "Gentler motion for presentations, keynote-style videos, and polished reveals." + } + }, "zoomInDuration": "Duración de entrada", "zoomInOverlap": "Solapamiento de entrada", "zoomOutDuration": "Duración de salida", @@ -74,6 +93,9 @@ }, "cursorSize": "Tamaño del cursor", "cursorSmoothing": "Suavizado del cursor", + "cursorSpringStiffness": "Cursor Spring Stiffness", + "cursorSpringDamping": "Cursor Spring Damping", + "cursorSpringMass": "Cursor Spring Mass", "off": "Desactivado", "cursorMotionBlur": "Desenfoque de movimiento del cursor", "cursorClickBounce": "Rebote de clic del cursor", @@ -125,8 +147,6 @@ "generateFull": "Generar subtítulos", "regenerateFull": "Regenerar subtítulos", "clearFull": "Borrar subtítulos", - "editCurrent": "Editar subtítulo actual", - "editSaved": "Subtítulo actualizado", "fontSettings": "Tipografía", "defaultFont": "Predeterminado", "fontFamily": "Fuente", diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json index d3494ed1d..bf50f1fb9 100644 --- a/src/i18n/locales/fr/editor.json +++ b/src/i18n/locales/fr/editor.json @@ -113,6 +113,11 @@ "project": { "untitled": "Sans titre" }, + "nativeCaptureUnavailable": { + "title": "Rien n'est cassé, mais nous ne pourrons pas afficher une superposition de curseur animée.", + "description": "Votre appareil ne prend pas en charge la capture native. Cela peut avoir plusieurs causes que nous n'avons pas encore identifiées. Recordly continuera de fonctionner, mais le lissage du curseur sera impossible.", + "confirm": "D'accord" + }, "exportStatus": { "exporting": "Exportation", "renderingFile": "Rendu de votre fichier.", diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index 200ecaa02..ef8ba6bc2 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -45,6 +45,11 @@ }, "backgroundBlur": "Flou d’arrière-plan", "zoomMotionBlur": "Flou de mouvement du zoom", + "temporalZoomMotionBlur": "Temporal Zoom Blur", + "temporalZoomMotionBlurDescription": "Control the shutter window and frame samples used by the newer zoom blur pass.", + "zoomMotionBlurSamples": "Blur Samples", + "zoomMotionBlurShutter": "Shutter", + "auto": "Auto", "connectZooms": "Relier les zooms", "connectZoomsDescription": "Lisse les zones de zoom consécutives pour créer un mouvement de caméra continu.", "autoApplyFreshRecordingZooms": "Appliquer automatiquement les zooms des nouveaux enregistrements", @@ -57,6 +62,20 @@ "zoomOutDescription": "Contrôle la manière dont la caméra sort d’une zone de zoom.", "connectedZoomTitle": "Entre les zooms", "connectedZoomDescription": "Ajuste la transition entre des zones de zoom consécutives lorsque la connexion est activée.", + "motionPresetsTitle": "Motion Presets", + "motionPresetsZoomHint": "Zoom motion presets are available in Settings.", + "animationPresets": "Animation Presets", + "cursorMotionPresets": "Cursor Motion Presets", + "motionPresets": { + "focused": { + "label": "Focused", + "description": "Snappier motion for demos, walkthroughs, and everyday recordings." + }, + "smooth": { + "label": "Smooth", + "description": "Gentler motion for presentations, keynote-style videos, and polished reveals." + } + }, "zoomInDuration": "Durée du zoom avant", "zoomInOverlap": "Chevauchement du zoom avant", "zoomOutDuration": "Durée du zoom arrière", @@ -74,6 +93,9 @@ }, "cursorSize": "Taille du curseur", "cursorSmoothing": "Lissage du curseur", + "cursorSpringStiffness": "Cursor Spring Stiffness", + "cursorSpringDamping": "Cursor Spring Damping", + "cursorSpringMass": "Cursor Spring Mass", "off": "Désactivé", "cursorMotionBlur": "Flou de mouvement du curseur", "cursorClickBounce": "Rebond au clic du curseur", @@ -125,8 +147,6 @@ "generateFull": "Générer les sous-titres", "regenerateFull": "Régénérer les sous-titres", "clearFull": "Effacer les sous-titres", - "editCurrent": "Modifier le sous-titre actuel", - "editSaved": "Sous-titre mis à jour", "fontSettings": "Paramètres de police", "defaultFont": "Par défaut", "fontFamily": "Police", diff --git a/src/i18n/locales/ko/editor.json b/src/i18n/locales/ko/editor.json index 56eb6bc50..f9d687ddf 100644 --- a/src/i18n/locales/ko/editor.json +++ b/src/i18n/locales/ko/editor.json @@ -114,6 +114,11 @@ "project": { "untitled": "제목 없음" }, + "nativeCaptureUnavailable": { + "title": "문제가 발생한 것은 아니지만 애니메이션 커서 오버레이를 렌더링할 수 없습니다.", + "description": "이 기기는 네이티브 캡처를 지원하지 않습니다. 아직 확인하지 못한 여러 원인이 있을 수 있습니다. Recordly는 계속 작동하지만 커서 부드럽게 처리는 사용할 수 없습니다.", + "confirm": "확인" + }, "exportStatus": { "exporting": "내보내는 중", "renderingFile": "파일을 렌더링하고 있습니다.", diff --git a/src/i18n/locales/ko/settings.json b/src/i18n/locales/ko/settings.json index c528a98fb..5ff76b9ee 100644 --- a/src/i18n/locales/ko/settings.json +++ b/src/i18n/locales/ko/settings.json @@ -45,6 +45,11 @@ }, "backgroundBlur": "배경 블러", "zoomMotionBlur": "확대 모션 블러", + "temporalZoomMotionBlur": "시간 기반 줌 블러", + "temporalZoomMotionBlurDescription": "새로운 줌 블러 처리에 사용할 셔터 범위와 프레임 샘플 수를 조정합니다.", + "zoomMotionBlurSamples": "블러 샘플 수", + "zoomMotionBlurShutter": "셔터", + "auto": "자동", "connectZooms": "확대 구간 연결", "connectZoomsDescription": "연속된 확대 구간을 하나의 부드러운 카메라 이동으로 연결합니다.", "autoApplyFreshRecordingZooms": "새 녹화에 확대 자동 적용", @@ -57,6 +62,20 @@ "zoomOutDescription": "카메라가 확대 구간에서 빠져나오는 방식을 제어합니다.", "connectedZoomTitle": "확대 구간 사이", "connectedZoomDescription": "연결이 활성화된 경우, 연속된 확대 구간 사이의 이동을 조정합니다.", + "motionPresetsTitle": "모션 프리셋", + "motionPresetsZoomHint": "줌 모션 프리셋은 설정에서 사용할 수 있습니다.", + "animationPresets": "애니메이션 프리셋", + "cursorMotionPresets": "커서 모션 프리셋", + "motionPresets": { + "focused": { + "label": "집중형", + "description": "데모, 워크스루, 일반 녹화에 적합한 더 빠르고 또렷한 모션입니다." + }, + "smooth": { + "label": "부드러움", + "description": "프레젠테이션, 키노트 스타일 영상, 세련된 전환에 적합한 더 완만한 모션입니다." + } + }, "zoomInDuration": "확대 시작 시간", "zoomInOverlap": "확대 시작 겹침", "zoomOutDuration": "확대 종료 시간", @@ -74,6 +93,9 @@ }, "cursorSize": "커서 크기", "cursorSmoothing": "커서 보정", + "cursorSpringStiffness": "커서 스프링 강성", + "cursorSpringDamping": "커서 스프링 감쇠", + "cursorSpringMass": "커서 스프링 질량", "off": "끔", "cursorMotionBlur": "커서 모션 블러", "cursorClickBounce": "커서 클릭 바운스", @@ -125,8 +147,6 @@ "generateFull": "자막 생성", "regenerateFull": "자막 다시 생성", "clearFull": "자막 지우기", - "editCurrent": "현재 자막 편집", - "editSaved": "자막이 업데이트되었습니다", "fontSettings": "글꼴 설정", "defaultFont": "기본값", "fontFamily": "글꼴", diff --git a/src/i18n/locales/nl/editor.json b/src/i18n/locales/nl/editor.json index b174ca98c..1d41e2c27 100644 --- a/src/i18n/locales/nl/editor.json +++ b/src/i18n/locales/nl/editor.json @@ -114,6 +114,11 @@ "project": { "untitled": "Naamloos" }, + "nativeCaptureUnavailable": { + "title": "Er is niets kapot, maar we kunnen geen geanimeerde cursor-overlay renderen.", + "description": "Je apparaat ondersteunt geen native capture. Dat kan verschillende oorzaken hebben die we nog niet hebben vastgesteld. Recordly blijft werken, maar cursorverzachting is dan niet mogelijk.", + "confirm": "Oké" + }, "exportStatus": { "exporting": "Exporteren", "renderingFile": "Je bestand wordt gerenderd.", diff --git a/src/i18n/locales/nl/settings.json b/src/i18n/locales/nl/settings.json index d88e4278c..f480c5b4c 100644 --- a/src/i18n/locales/nl/settings.json +++ b/src/i18n/locales/nl/settings.json @@ -45,6 +45,11 @@ }, "backgroundBlur": "Achtergrondvervaging", "zoomMotionBlur": "Zoom-bewegingsonscherpte", + "temporalZoomMotionBlur": "Temporal Zoom Blur", + "temporalZoomMotionBlurDescription": "Control the shutter window and frame samples used by the newer zoom blur pass.", + "zoomMotionBlurSamples": "Blur Samples", + "zoomMotionBlurShutter": "Shutter", + "auto": "Auto", "connectZooms": "Zooms verbinden", "connectZoomsDescription": "Maak opeenvolgende zoomgebieden vloeiend tot een doorlopende camerabeweging.", "autoApplyFreshRecordingZooms": "Zooms voor nieuwe opnames automatisch toepassen", @@ -57,6 +62,20 @@ "zoomOutDescription": "Bepaal hoe de camera een zoomgebied verlaat.", "connectedZoomTitle": "Tussen zooms", "connectedZoomDescription": "Stel de overgang af tussen opeenvolgende zoomgebieden wanneer verbinding is ingeschakeld.", + "motionPresetsTitle": "Motion Presets", + "motionPresetsZoomHint": "Zoom motion presets are available in Settings.", + "animationPresets": "Animation Presets", + "cursorMotionPresets": "Cursor Motion Presets", + "motionPresets": { + "focused": { + "label": "Focused", + "description": "Snappier motion for demos, walkthroughs, and everyday recordings." + }, + "smooth": { + "label": "Smooth", + "description": "Gentler motion for presentations, keynote-style videos, and polished reveals." + } + }, "zoomInDuration": "Inzoomduur", "zoomInOverlap": "Inzoomoverlap", "zoomOutDuration": "Uitzoomduur", @@ -74,6 +93,9 @@ }, "cursorSize": "Cursorgrootte", "cursorSmoothing": "Cursorverzachting", + "cursorSpringStiffness": "Cursor Spring Stiffness", + "cursorSpringDamping": "Cursor Spring Damping", + "cursorSpringMass": "Cursor Spring Mass", "off": "Uit", "cursorMotionBlur": "Cursor-bewegingsonscherpte", "cursorClickBounce": "Cursorklikstuit", @@ -125,8 +147,6 @@ "generateFull": "Ondertiteling genereren", "regenerateFull": "Ondertiteling opnieuw genereren", "clearFull": "Ondertiteling wissen", - "editCurrent": "Huidige ondertiteling bewerken", - "editSaved": "Ondertiteling bijgewerkt", "fontSettings": "Lettertype-instellingen", "defaultFont": "Standaard", "fontFamily": "Lettertype", diff --git a/src/i18n/locales/pt-BR/editor.json b/src/i18n/locales/pt-BR/editor.json index 175bf6742..1477aced5 100644 --- a/src/i18n/locales/pt-BR/editor.json +++ b/src/i18n/locales/pt-BR/editor.json @@ -113,6 +113,11 @@ "project": { "untitled": "Sem título" }, + "nativeCaptureUnavailable": { + "title": "Nada está quebrado, mas não poderemos renderizar uma sobreposição animada do cursor.", + "description": "Seu dispositivo não é compatível com captura nativa. Isso pode acontecer por vários motivos que ainda não identificamos. O Recordly continuará funcionando, mas a suavização do cursor ficará indisponível.", + "confirm": "Entendi" + }, "exportStatus": { "exporting": "Exportando", "renderingFile": "Renderizando seu arquivo.", diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json index a34e6e4d2..74346b03a 100644 --- a/src/i18n/locales/pt-BR/settings.json +++ b/src/i18n/locales/pt-BR/settings.json @@ -45,6 +45,11 @@ }, "backgroundBlur": "Desfoque de fundo", "zoomMotionBlur": "Desfoque de movimento do zoom", + "temporalZoomMotionBlur": "Temporal Zoom Blur", + "temporalZoomMotionBlurDescription": "Control the shutter window and frame samples used by the newer zoom blur pass.", + "zoomMotionBlurSamples": "Blur Samples", + "zoomMotionBlurShutter": "Shutter", + "auto": "Auto", "connectZooms": "Conectar zooms", "connectZoomsDescription": "Suaviza regiões de zoom consecutivas em um movimento contínuo de câmera.", "autoApplyFreshRecordingZooms": "Aplicar zooms automaticamente em gravações novas", @@ -57,6 +62,20 @@ "zoomOutDescription": "Controla como a câmera sai de uma região de zoom.", "connectedZoomTitle": "Entre zooms", "connectedZoomDescription": "Ajuste o deslizamento entre regiões de zoom consecutivas quando a conexão estiver ativada.", + "motionPresetsTitle": "Motion Presets", + "motionPresetsZoomHint": "Zoom motion presets are available in Settings.", + "animationPresets": "Animation Presets", + "cursorMotionPresets": "Cursor Motion Presets", + "motionPresets": { + "focused": { + "label": "Focused", + "description": "Snappier motion for demos, walkthroughs, and everyday recordings." + }, + "smooth": { + "label": "Smooth", + "description": "Gentler motion for presentations, keynote-style videos, and polished reveals." + } + }, "zoomInDuration": "Duração do zoom in", "zoomInOverlap": "Sobreposição do zoom in", "zoomOutDuration": "Duração do zoom out", @@ -74,6 +93,9 @@ }, "cursorSize": "Tamanho do cursor", "cursorSmoothing": "Suavização do cursor", + "cursorSpringStiffness": "Cursor Spring Stiffness", + "cursorSpringDamping": "Cursor Spring Damping", + "cursorSpringMass": "Cursor Spring Mass", "off": "Desligado", "cursorMotionBlur": "Desfoque de movimento do cursor", "cursorClickBounce": "Salto de clique do cursor", @@ -125,8 +147,6 @@ "generateFull": "Gerar legendas", "regenerateFull": "Gerar legendas novamente", "clearFull": "Limpar legendas", - "editCurrent": "Editar legenda atual", - "editSaved": "Legenda atualizada", "fontSettings": "Configurações da fonte", "defaultFont": "Padrão", "fontFamily": "Fonte", diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json index 4a0c4a5d3..674a67c6c 100644 --- a/src/i18n/locales/zh-CN/editor.json +++ b/src/i18n/locales/zh-CN/editor.json @@ -113,6 +113,11 @@ "project": { "untitled": "未命名" }, + "nativeCaptureUnavailable": { + "title": "没有出错，但我们无法渲染动画光标叠加层。", + "description": "你的设备不支持原生捕获。这可能由多种我们尚未确认的原因造成。Recordly 仍可继续使用，但无法启用光标平滑。", + "confirm": "好的" + }, "exportStatus": { "exporting": "正在导出", "renderingFile": "正在渲染你的文件。", diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index a68d2a04f..7982c5d55 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -45,6 +45,11 @@ }, "backgroundBlur": "背景模糊", "zoomMotionBlur": "缩放运动模糊", + "temporalZoomMotionBlur": "Temporal Zoom Blur", + "temporalZoomMotionBlurDescription": "Control the shutter window and frame samples used by the newer zoom blur pass.", + "zoomMotionBlurSamples": "Blur Samples", + "zoomMotionBlurShutter": "Shutter", + "auto": "Auto", "connectZooms": "连接缩放", "connectZoomsDescription": "将连续的缩放区域平滑连接为一次连续的镜头移动。", "autoApplyFreshRecordingZooms": "自动为新录制应用缩放", diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json index e37768b3f..97db8e58a 100644 --- a/src/i18n/locales/zh-TW/editor.json +++ b/src/i18n/locales/zh-TW/editor.json @@ -113,6 +113,11 @@ "project": { "untitled": "未命名" }, + "nativeCaptureUnavailable": { + "title": "沒有出錯，但我們無法算繪動畫游標疊加層。", + "description": "你的裝置不支援原生擷取。這可能是由多種我們尚未確認的原因造成。Recordly 仍可繼續使用，但無法啟用游標平滑。", + "confirm": "好的" + }, "exportStatus": { "exporting": "正在匯出", "renderingFile": "正在渲染你的檔案。", @@ -134,4 +139,4 @@ "collapse": "摺疊時間軸" }, "openRecordingsFolder": "打開錄製資料夾" -} \ No newline at end of file +} diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json index db0ea7e04..e2651d86b 100644 --- a/src/i18n/locales/zh-TW/settings.json +++ b/src/i18n/locales/zh-TW/settings.json @@ -45,6 +45,11 @@ }, "backgroundBlur": "背景模糊", "zoomMotionBlur": "縮放動態模糊", + "temporalZoomMotionBlur": "Temporal Zoom Blur", + "temporalZoomMotionBlurDescription": "Control the shutter window and frame samples used by the newer zoom blur pass.", + "zoomMotionBlurSamples": "Blur Samples", + "zoomMotionBlurShutter": "Shutter", + "auto": "Auto", "connectZooms": "連接縮放", "connectZoomsDescription": "將連續的縮放區域平滑串接成一段連續的鏡頭移動。", "autoApplyFreshRecordingZooms": "自動套用新錄影的縮放", @@ -57,6 +62,20 @@ "zoomOutDescription": "控制相機如何離開縮放區域。", "connectedZoomTitle": "縮放之間", "connectedZoomDescription": "啟用連接後，調整連續縮放區域之間的滑行效果。", + "motionPresetsTitle": "Motion Presets", + "motionPresetsZoomHint": "Zoom motion presets are available in Settings.", + "animationPresets": "Animation Presets", + "cursorMotionPresets": "Cursor Motion Presets", + "motionPresets": { + "focused": { + "label": "Focused", + "description": "Snappier motion for demos, walkthroughs, and everyday recordings." + }, + "smooth": { + "label": "Smooth", + "description": "Gentler motion for presentations, keynote-style videos, and polished reveals." + } + }, "zoomInDuration": "拉近時間", "zoomInOverlap": "拉近重疊", "zoomOutDuration": "拉遠時間", @@ -74,6 +93,9 @@ }, "cursorSize": "游標大小", "cursorSmoothing": "游標平滑", + "cursorSpringStiffness": "Cursor Spring Stiffness", + "cursorSpringDamping": "Cursor Spring Damping", + "cursorSpringMass": "Cursor Spring Mass", "off": "關閉", "cursorMotionBlur": "游標動態模糊", "cursorClickBounce": "點擊彈跳", @@ -125,8 +147,6 @@ "generateFull": "產生字幕", "regenerateFull": "重新產生字幕", "clearFull": "清除字幕", - "editCurrent": "編輯目前字幕", - "editSaved": "字幕已更新", "fontSettings": "字型設定", "defaultFont": "預設", "fontFamily": "字型", diff --git a/src/index.css b/src/index.css index a5ae2368b..1f750e155 100644 --- a/src/index.css +++ b/src/index.css @@ -121,7 +121,6 @@ --slider-glow: 0 0% 100% / 0.5; } } - @layer base { * { @apply border-border; @@ -268,6 +267,3 @@ transform: translateX(270%); } } - - - diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts index 9e9deeec4..0b311538e 100644 --- a/src/lib/exporter/audioEncoder.test.ts +++ b/src/lib/exporter/audioEncoder.test.ts @@ -14,7 +14,7 @@ type OfflineRenderTestHarness = AudioProcessor & { sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath?: Record, ): Promise<{ - mainBuffer: AudioBuffer | null; + mainBufferEntry: { buffer: AudioBuffer; gain: number } | null; companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }>; }>; renderAndMuxOfflineAudio( @@ -55,7 +55,7 @@ describe("AudioProcessor offline render preparation", () => { ["/tmp/recording.mp4", "/tmp/recording.mic.wav"], ); - expect(prepared.mainBuffer).toBe(mainBuffer); + expect(prepared.mainBufferEntry?.buffer).toBe(mainBuffer); expect(prepared.companionEntries).toHaveLength(1); expect(prepared.companionEntries[0]?.buffer).toBe(micBuffer); expect(decodeAudioFromUrl).toHaveBeenCalledWith("file:///tmp/recording.mp4"); diff --git a/src/lib/simpleQr.ts b/src/lib/simpleQr.ts new file mode 100644 index 000000000..393d906b7 --- /dev/null +++ b/src/lib/simpleQr.ts @@ -0,0 +1,443 @@ +type QrSvgPath = { + path: string; + size: number; +}; + +type QrProfile = { + version: number; + dataCodewords: number; + eccCodewords: number; + alignmentCenters: number[]; +}; + +const LOW_ECC_PROFILES: QrProfile[] = [ + { version: 1, dataCodewords: 19, eccCodewords: 7, alignmentCenters: [] }, + { version: 2, dataCodewords: 34, eccCodewords: 10, alignmentCenters: [6, 18] }, + { version: 3, dataCodewords: 55, eccCodewords: 15, alignmentCenters: [6, 22] }, + { version: 4, dataCodewords: 80, eccCodewords: 20, alignmentCenters: [6, 26] }, + { version: 5, dataCodewords: 108, eccCodewords: 26, alignmentCenters: [6, 30] }, +]; + +const FORMAT_MASK = 0x5412; +const FORMAT_DIVISOR = 0x537; +const LOW_ECC_FORMAT_BITS = 1; +const QUIET_ZONE_MODULES = 4; +const PAD_CODEWORDS = [0xec, 0x11]; +const FINDER_LIKE_PATTERN = [true, false, true, true, true, false, true]; + +const GF_EXP = new Array(512); +const GF_LOG = new Array(256); + +let x = 1; +for (let i = 0; i < 255; i += 1) { + GF_EXP[i] = x; + GF_LOG[x] = i; + x <<= 1; + if (x & 0x100) { + x ^= 0x11d; + } +} +for (let i = 255; i < GF_EXP.length; i += 1) { + GF_EXP[i] = GF_EXP[i - 255]; +} + +export function createQrSvgPath(value: string): QrSvgPath | null { + const matrix = createQrMatrix(value); + if (!matrix) { + return null; + } + + const size = matrix.length + QUIET_ZONE_MODULES * 2; + const path = matrix + .flatMap((row, rowIndex) => + row.flatMap((dark, colIndex) => + dark + ? [`M${colIndex + QUIET_ZONE_MODULES} ${rowIndex + QUIET_ZONE_MODULES}h1v1h-1z`] + : [], + ), + ) + .join(""); + + return { path, size }; +} + +function createQrMatrix(value: string): boolean[][] | null { + const bytes = new TextEncoder().encode(value); + const profile = selectProfile(bytes.length); + if (!profile) { + return null; + } + + const dataCodewords = createDataCodewords(bytes, profile.dataCodewords); + const errorCodewords = createErrorCorrectionCodewords(dataCodewords, profile.eccCodewords); + const codewords = [...dataCodewords, ...errorCodewords]; + const bits = codewords.flatMap((codeword) => intToBits(codeword, 8)); + const base = createBaseMatrix(profile); + placeDataBits(base.modules, base.reserved, bits); + + let bestMatrix: boolean[][] | null = null; + let bestPenalty = Number.POSITIVE_INFINITY; + + for (let mask = 0; mask < 8; mask += 1) { + const candidate = applyMask(base.modules, base.reserved, mask); + drawFormatBits(candidate, mask); + const penalty = scorePenalty(candidate); + if (penalty < bestPenalty) { + bestPenalty = penalty; + bestMatrix = candidate; + } + } + + return bestMatrix; +} + +function selectProfile(byteLength: number): QrProfile | null { + return ( + LOW_ECC_PROFILES.find((profile) => { + const capacityBits = profile.dataCodewords * 8; + const requiredBits = 4 + 8 + byteLength * 8; + return requiredBits <= capacityBits; + }) ?? null + ); +} + +function createDataCodewords(bytes: Uint8Array, dataCodewordCount: number): number[] { + const capacityBits = dataCodewordCount * 8; + const bits = [...intToBits(0b0100, 4), ...intToBits(bytes.length, 8)]; + for (const byte of bytes) { + bits.push(...intToBits(byte, 8)); + } + + const terminatorLength = Math.min(4, capacityBits - bits.length); + for (let i = 0; i < terminatorLength; i += 1) { + bits.push(0); + } + while (bits.length % 8 !== 0) { + bits.push(0); + } + + const codewords: number[] = []; + for (let i = 0; i < bits.length; i += 8) { + codewords.push(bitsToInt(bits.slice(i, i + 8))); + } + let padIndex = 0; + while (codewords.length < dataCodewordCount) { + codewords.push(PAD_CODEWORDS[padIndex % PAD_CODEWORDS.length]); + padIndex += 1; + } + + return codewords; +} + +function createErrorCorrectionCodewords(data: number[], degree: number): number[] { + const generator = createGeneratorPolynomial(degree); + const message = [...data, ...new Array(degree).fill(0)]; + + for (let i = 0; i < data.length; i += 1) { + const factor = message[i]; + if (factor === 0) { + continue; + } + for (let j = 0; j < generator.length; j += 1) { + message[i + j] ^= gfMultiply(generator[j], factor); + } + } + + return message.slice(message.length - degree); +} + +function createGeneratorPolynomial(degree: number): number[] { + let result = [1]; + for (let i = 0; i < degree; i += 1) { + const next = new Array(result.length + 1).fill(0); + for (let j = 0; j < result.length; j += 1) { + next[j] ^= result[j]; + next[j + 1] ^= gfMultiply(result[j], GF_EXP[i]); + } + result = next; + } + return result; +} + +function gfMultiply(a: number, b: number): number { + if (a === 0 || b === 0) { + return 0; + } + return GF_EXP[GF_LOG[a] + GF_LOG[b]]; +} + +function createBaseMatrix(profile: QrProfile): { + modules: (boolean | null)[][]; + reserved: boolean[][]; +} { + const size = profile.version * 4 + 17; + const modules = Array.from({ length: size }, () => new Array(size).fill(null)); + const reserved = Array.from({ length: size }, () => new Array(size).fill(false)); + const setFunctionModule = (row: number, col: number, dark: boolean) => { + if (row < 0 || col < 0 || row >= size || col >= size) { + return; + } + modules[row][col] = dark; + reserved[row][col] = true; + }; + + drawFinderPattern(setFunctionModule, 0, 0); + drawFinderPattern(setFunctionModule, 0, size - 7); + drawFinderPattern(setFunctionModule, size - 7, 0); + drawAlignmentPatterns(profile, setFunctionModule); + drawTimingPatterns(size, setFunctionModule); + reserveFormatAreas(size, setFunctionModule); + setFunctionModule(profile.version * 4 + 9, 8, true); + + return { modules, reserved }; +} + +function drawFinderPattern( + setFunctionModule: (row: number, col: number, dark: boolean) => void, + row: number, + col: number, +) { + for (let y = -1; y <= 7; y += 1) { + for (let x = -1; x <= 7; x += 1) { + const inFinder = x >= 0 && x <= 6 && y >= 0 && y <= 6; + const dark = + inFinder && + (x === 0 || + x === 6 || + y === 0 || + y === 6 || + (x >= 2 && x <= 4 && y >= 2 && y <= 4)); + setFunctionModule(row + y, col + x, dark); + } + } +} + +function drawAlignmentPatterns( + profile: QrProfile, + setFunctionModule: (row: number, col: number, dark: boolean) => void, +) { + for (const row of profile.alignmentCenters) { + for (const col of profile.alignmentCenters) { + const overlapsFinder = + (row === 6 && col === 6) || + (row === 6 && col === profile.version * 4 + 10) || + (row === profile.version * 4 + 10 && col === 6); + if (overlapsFinder) { + continue; + } + for (let y = -2; y <= 2; y += 1) { + for (let x = -2; x <= 2; x += 1) { + setFunctionModule(row + y, col + x, Math.max(Math.abs(x), Math.abs(y)) !== 1); + } + } + } + } +} + +function drawTimingPatterns( + size: number, + setFunctionModule: (row: number, col: number, dark: boolean) => void, +) { + for (let i = 8; i < size - 8; i += 1) { + const dark = i % 2 === 0; + setFunctionModule(6, i, dark); + setFunctionModule(i, 6, dark); + } +} + +function reserveFormatAreas( + size: number, + setFunctionModule: (row: number, col: number, dark: boolean) => void, +) { + for (let i = 0; i < 9; i += 1) { + if (i !== 6) { + setFunctionModule(8, i, false); + setFunctionModule(i, 8, false); + } + } + for (let i = 0; i < 8; i += 1) { + setFunctionModule(size - 1 - i, 8, false); + setFunctionModule(8, size - 1 - i, false); + } +} + +function placeDataBits(modules: (boolean | null)[][], reserved: boolean[][], bits: number[]) { + const size = modules.length; + let bitIndex = 0; + let upward = true; + + for (let right = size - 1; right >= 1; right -= 2) { + if (right === 6) { + right -= 1; + } + for (let vertical = 0; vertical < size; vertical += 1) { + const row = upward ? size - 1 - vertical : vertical; + for (let offset = 0; offset < 2; offset += 1) { + const col = right - offset; + if (reserved[row][col]) { + continue; + } + modules[row][col] = bitIndex < bits.length ? bits[bitIndex] === 1 : false; + bitIndex += 1; + } + } + upward = !upward; + } +} + +function applyMask( + modules: (boolean | null)[][], + reserved: boolean[][], + mask: number, +): boolean[][] { + return modules.map((row, rowIndex) => + row.map((value, colIndex) => { + const dark = value === true; + if (reserved[rowIndex][colIndex]) { + return dark; + } + return shouldMask(mask, rowIndex, colIndex) ? !dark : dark; + }), + ); +} + +function shouldMask(mask: number, row: number, col: number): boolean { + switch (mask) { + case 0: + return (row + col) % 2 === 0; + case 1: + return row % 2 === 0; + case 2: + return col % 3 === 0; + case 3: + return (row + col) % 3 === 0; + case 4: + return (Math.floor(row / 2) + Math.floor(col / 3)) % 2 === 0; + case 5: + return ((row * col) % 2) + ((row * col) % 3) === 0; + case 6: + return (((row * col) % 2) + ((row * col) % 3)) % 2 === 0; + case 7: + return (((row + col) % 2) + ((row * col) % 3)) % 2 === 0; + default: + return false; + } +} + +function drawFormatBits(matrix: boolean[][], mask: number) { + const size = matrix.length; + const bits = createFormatBits(mask); + const getBit = (index: number) => ((bits >> index) & 1) !== 0; + + for (let i = 0; i <= 5; i += 1) { + matrix[i][8] = getBit(i); + } + matrix[7][8] = getBit(6); + matrix[8][8] = getBit(7); + matrix[8][7] = getBit(8); + for (let i = 9; i < 15; i += 1) { + matrix[8][14 - i] = getBit(i); + } + + for (let i = 0; i < 8; i += 1) { + matrix[8][size - 1 - i] = getBit(i); + } + for (let i = 8; i < 15; i += 1) { + matrix[size - 15 + i][8] = getBit(i); + } + matrix[size - 8][8] = true; +} + +function createFormatBits(mask: number): number { + const data = (LOW_ECC_FORMAT_BITS << 3) | mask; + let remainder = data; + for (let i = 0; i < 10; i += 1) { + remainder = (remainder << 1) ^ (((remainder >> 9) & 1) === 1 ? FORMAT_DIVISOR : 0); + } + return ((data << 10) | (remainder & 0x3ff)) ^ FORMAT_MASK; +} + +function scorePenalty(matrix: boolean[][]): number { + let penalty = 0; + const size = matrix.length; + + for (let row = 0; row < size; row += 1) { + penalty += scoreLine(matrix[row]); + penalty += scoreFinderLikePatterns(matrix[row]); + } + for (let col = 0; col < size; col += 1) { + const column = matrix.map((row) => row[col]); + penalty += scoreLine(column); + penalty += scoreFinderLikePatterns(column); + } + for (let row = 0; row < size - 1; row += 1) { + for (let col = 0; col < size - 1; col += 1) { + const color = matrix[row][col]; + if ( + color === matrix[row][col + 1] && + color === matrix[row + 1][col] && + color === matrix[row + 1][col + 1] + ) { + penalty += 3; + } + } + } + + const darkCount = matrix.flat().filter(Boolean).length; + const percent = (darkCount * 100) / (size * size); + penalty += Math.floor(Math.abs(percent - 50) / 5) * 10; + + return penalty; +} + +function scoreFinderLikePatterns(line: boolean[]): number { + let penalty = 0; + for (let i = 0; i <= line.length - FINDER_LIKE_PATTERN.length; i += 1) { + const matches = FINDER_LIKE_PATTERN.every((value, index) => line[i + index] === value); + if (!matches) { + continue; + } + + const hasLightBefore = i >= 4 && line.slice(i - 4, i).every((value) => value === false); + const afterStart = i + FINDER_LIKE_PATTERN.length; + const hasLightAfter = + afterStart + 4 <= line.length && + line.slice(afterStart, afterStart + 4).every((value) => value === false); + if (hasLightBefore || hasLightAfter) { + penalty += 40; + } + } + return penalty; +} + +function scoreLine(line: boolean[]): number { + let penalty = 0; + let runColor = line[0]; + let runLength = 1; + + for (let i = 1; i < line.length; i += 1) { + if (line[i] === runColor) { + runLength += 1; + continue; + } + if (runLength >= 5) { + penalty += runLength - 2; + } + runColor = line[i]; + runLength = 1; + } + + if (runLength >= 5) { + penalty += runLength - 2; + } + + return penalty; +} + +function intToBits(value: number, length: number): number[] { + return Array.from({ length }, (_, index) => (value >> (length - 1 - index)) & 1); +} + +function bitsToInt(bits: number[]): number { + return bits.reduce((value, bit) => (value << 1) | bit, 0); +}