diff --git a/packages/dashboard/src/dashboard.tsx b/packages/dashboard/src/dashboard.tsx index c2f1e643eaf25..27e966c1973ed 100644 --- a/packages/dashboard/src/dashboard.tsx +++ b/packages/dashboard/src/dashboard.tsx @@ -19,6 +19,9 @@ import './dashboard.css'; import { ChevronLeftIcon, ChevronRightIcon, LockIcon, LockOpenIcon, ReloadIcon, ScreenshotRegionIcon } from './icons'; import { clientToViewport, getImageLayout } from './imageLayout'; import { Recording } from './recording'; +import { HistoryPlayer } from './historyPlayer'; +import type { HistoryPlayback } from './historyPlayback'; +import { HistoryScrubber } from './historyScrubber'; import { AnnotateSidebar, AnnotateOverlay } from './annotateView'; import { ToolbarButton } from '@web/components/toolbarButton'; @@ -66,7 +69,7 @@ export const Dashboard: React.FC = ({ model }) => { const [, setRevision] = React.useState(0); React.useEffect(() => model.subscribe(() => setRevision(r => r + 1)), [model]); - const { tabs, mode, recording, liveFrame, annotateSession, pendingCapture } = model.state; + const { tabs, mode, recording, liveFrame, annotateSession, pendingCapture, history } = model.state; const interactive = mode === 'interactive'; const annotateActive = !!annotateSession; const selectedFrame = annotateSession?.frames.find(f => f.id === annotateSession.selectedFrameId) ?? null; @@ -79,6 +82,8 @@ export const Dashboard: React.FC = ({ model }) => { const viewportMainRef = React.useRef(null); const browserChromeRef = React.useRef(null); const interactiveBtnRef = React.useRef(null); + const historyVideoRef = React.useRef(null); + const historyPlaybackRef = React.useRef(null); const moveThrottleRef = React.useRef(0); const aspect = liveFrame && liveFrame.viewportWidth && liveFrame.viewportHeight @@ -259,7 +264,9 @@ export const Dashboard: React.FC = ({ model }) => { title={annotateActive ? 'Add screenshot' : 'Take screenshot'} disabled={!ready || pendingCapture} onClick={() => { - if (annotateActive) + if (history.scrubMode) + void model.addHistoryAnnotateFrame(historyVideoRef.current); + else if (annotateActive) model.addAnnotateFrame(); else model.enterAnnotate('user'); @@ -368,6 +375,8 @@ export const Dashboard: React.FC = ({ model }) => { /> {overlayText &&
{overlayText}
} + {history.scrubMode && } + {history.enabled && } diff --git a/packages/dashboard/src/dashboardChannel.ts b/packages/dashboard/src/dashboardChannel.ts index b09fc47e66e20..2106eb2bf8e37 100644 --- a/packages/dashboard/src/dashboardChannel.ts +++ b/packages/dashboard/src/dashboardChannel.ts @@ -34,21 +34,39 @@ export type AnnotationData = { x: number; y: number; width: number; height: numb export type SubmittedAnnotationFrame = { data: string; - ariaSnapshot: string; + ariaSnapshot?: string; annotations: AnnotationData[]; - sessionTitle: string; - title: string; - url: string; + sessionTitle?: string; + title?: string; + url?: string; viewportWidth: number; viewportHeight: number; + timestamp: number; +}; + +export type HistorySnapshot = + | { enabled: false } + | { enabled: true; init: string }; + +// Cluster manifest entry shipped per cluster. Coverage end is derived +// on the frontend as `clusters[idx+1]?.startWallMs ?? Date.now()` — +// the live cluster grows up to "now" as frames are written. +export type HistoryCluster = { + // Wallclock ms of this cluster's first frame. + startWallMs: number; + // Byte position of the cluster's first byte within the recorder webm. + fileOffset: number; + byteLen: number; }; export type DashboardChannelEvents = { sessions: { sessions: SessionStatus[]; clientInfo: ClientInfo }; tabs: { tabs: Tab[] }; - frame: { data: string; viewportWidth: number; viewportHeight: number }; + frame: { data: string; viewportWidth: number; viewportHeight: number; timestamp: number }; annotate: {}; cancelAnnotate: {}; + history: HistorySnapshot; + historyCluster: HistoryCluster; }; export type MouseButton = 'left' | 'middle' | 'right'; @@ -73,6 +91,7 @@ export interface DashboardChannel { startRecording(): Promise; stopRecording(): Promise<{ streamId: string }>; readStream(params: { streamId: string }): Promise<{ data: string; eof: boolean }>; + recorderReadBytes(params: { offset: number; length: number }): Promise<{ data: string }>; screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number; ariaSnapshot: string }>; submitAnnotation(params: { frames: SubmittedAnnotationFrame[]; feedback: string }): Promise; cancelAnnotation(): Promise; diff --git a/packages/dashboard/src/dashboardModel.ts b/packages/dashboard/src/dashboardModel.ts index 30f4c38bf04b9..67471ef28be68 100644 --- a/packages/dashboard/src/dashboardModel.ts +++ b/packages/dashboard/src/dashboardModel.ts @@ -16,9 +16,10 @@ import { buildAnnotatedImage, saveAnnotationAsDownload } from './annotationImage'; import { buildAnnotationZip } from './annotationZip'; +import { captureVideoFrameAsPng } from './historyPlayback'; import type { Annotation } from './annotations'; -import type { DashboardChannel, DashboardChannelEvents, MouseButton, SessionStatus, SubmittedAnnotationFrame, Tab } from './dashboardChannel'; +import type { DashboardChannel, DashboardChannelEvents, HistoryCluster, HistorySnapshot, MouseButton, SessionStatus, SubmittedAnnotationFrame, Tab } from './dashboardChannel'; import type { ClientInfo } from '../../playwright-core/src/tools/cli-client/registry'; import type { BrowserDescriptor } from '../../playwright-core/src/serverRegistry'; @@ -35,10 +36,13 @@ export type AnnotateFrame = { data: string; viewportWidth: number; viewportHeight: number; - ariaSnapshot: string; - sessionTitle: string; - title: string; - url: string; + // Wall-clock ms since epoch — same coordinate space as live frame + // timestamps, so live and historical frames are directly comparable. + timestamp: number; + ariaSnapshot?: string; + sessionTitle?: string; + title?: string; + url?: string; annotations: Annotation[]; }; @@ -50,6 +54,28 @@ export type AnnotateSession = { feedback: string; }; +export type HistoryState = { + enabled: boolean; + // True when the user is in scrub mode (history player visible). + scrubMode: boolean; + // Currently scrubbed time, in wall-clock ms. + scrubTime: number; + // While scrubbing, the bar's right edge is frozen here so the cursor + // doesn't drift left as the wall clock keeps advancing. + // null = use the live edge. + scrubFrozenEndMs: number | null; + // Cluster manifest pushed by the daemon. Each entry describes a + // wallclock start and an exact byte range in the webm, so the frontend + // knows precisely what's available and where to fetch it from. Reset + // on every `history` event; rebuilt from subsequent `historyCluster` + // events. + clusters: HistoryCluster[]; + // Webm init segment bytes for the current recording, decoded once + // from the base64 in the `history` event. null when no recording is + // active. The MSE pipeline appends this verbatim before any cluster. + init: Uint8Array | null; +}; + export type DashboardState = { // Session model state. sessions: SessionStatus[]; @@ -62,6 +88,7 @@ export type DashboardState = { pendingCapture: boolean; mode: Mode; recording: RecordingState | null; + history: HistoryState; }; type Listener = () => void; @@ -76,8 +103,26 @@ const initialState: DashboardState = { pendingCapture: false, mode: 'readonly', recording: null, + history: { enabled: false, scrubMode: false, scrubTime: 0, scrubFrozenEndMs: null, clusters: [], init: null }, }; +// Returns the wall-clock range visible to the history scrubber. `startMs` is +// the first cluster's wallclock; `endMs` is the frozen scrub-entry time while +// scrubbing (so the cursor doesn't drift as wall-clock advances), or the +// live edge otherwise (max of latest frame and wall clock now). +export function historyTimeRange(history: HistoryState, liveFrame: { timestamp: number } | undefined): { startMs: number; endMs: number } { + const liveEdge = Math.max(liveFrame?.timestamp ?? 0, Date.now()); + const end = history.scrubFrozenEndMs ?? liveEdge; + const start = history.clusters[0]?.startWallMs ?? end; + return { startMs: start, endMs: Math.max(start, end) }; +} + +function clamp(v: number, min: number, max: number): number { + if (max < min) + return min; + return Math.min(Math.max(v, min), max); +} + export class DashboardModel { state: DashboardState = initialState; @@ -86,6 +131,7 @@ export class DashboardModel { // Monotonic token to invalidate in-flight screenshot requests when // pendingAnnotate is cleared or replaced. private _requestId = 0; + private _bytesCache = new Map>(); constructor(client: DashboardChannel) { this._client = client; @@ -94,6 +140,76 @@ export class DashboardModel { client.on('frame', params => this._emit({ liveFrame: params })); client.on('annotate', () => this.enterAnnotate('cli')); client.on('cancelAnnotate', () => this.cancelAnnotate(false)); + client.on('history', snapshot => this._onHistorySnapshot(snapshot)); + client.on('historyCluster', params => this._onHistoryCluster(params)); + } + + private _onHistorySnapshot(snapshot: HistorySnapshot) { + this._bytesCache.clear(); + const initialState = { + enabled: false, + scrubMode: false, + scrubTime: 0, + scrubFrozenEndMs: null, + clusters: [], + init: null + }; + if (!snapshot.enabled) { + this._emit({ history: initialState }); + return; + } + this._emit({ history: { ...initialState, enabled: true, init: Uint8Array.fromBase64(snapshot.init) } }); + } + + private _onHistoryCluster(c: HistoryCluster) { + const { history } = this.state; + if (!history.enabled) + return; + history.clusters.push(c); + this._emit({ history }); + } + + enterScrub(timeMs?: number) { + const h = this.state.history; + if (!h.enabled || h.clusters.length === 0) + return; + // Freeze the bar's right edge so the cursor stays where the user + // put it as wall time keeps advancing. + const frozenEnd = Math.max(this.state.liveFrame?.timestamp ?? 0, Date.now()); + const start = h.clusters[0].startWallMs; + const t = clamp(timeMs ?? frozenEnd, start, frozenEnd); + this._emit({ history: { ...h, scrubMode: true, scrubTime: t, scrubFrozenEndMs: frozenEnd } }); + } + + exitScrub() { + const h = this.state.history; + if (!h.scrubMode) + return; + this._emit({ history: { ...h, scrubMode: false, scrubFrozenEndMs: null } }); + } + + setScrubTime(timeMs: number) { + const h = this.state.history; + if (!h.scrubMode) + return; + const range = historyTimeRange(h, this.state.liveFrame); + const t = clamp(timeMs, range.startMs, range.endMs); + if (t === h.scrubTime) + return; + this._emit({ history: { ...h, scrubTime: t } }); + } + + readBytes(offset: number, length: number): Promise { + const cached = this._bytesCache.get(offset); + if (cached) + return cached; + const p = (async () => { + const result = await this._client.recorderReadBytes({ offset, length }); + return Uint8Array.fromBase64(result.data); + })(); + this._bytesCache.set(offset, p); + p.catch(() => this._bytesCache.delete(offset)); + return p; } subscribe(listener: Listener): () => void { @@ -296,6 +412,7 @@ export class DashboardModel { url: frame.url, viewportWidth: frame.viewportWidth, viewportHeight: frame.viewportHeight, + timestamp: frame.timestamp, }); } if (session.initiator === 'cli') { @@ -359,12 +476,36 @@ export class DashboardModel { data: frameData.data, viewportWidth: frameData.viewportWidth, viewportHeight: frameData.viewportHeight, + timestamp: Date.now(), ariaSnapshot: frameData.ariaSnapshot, sessionTitle, title: selectedTab?.title ?? '', url: selectedTab?.url ?? '', annotations: [], }; + this._pushAnnotateFrame(frame, initiator); + } + + async addHistoryAnnotateFrame(video: HTMLVideoElement | null) { + if (!video || this.state.pendingCapture) + return; + const captured = await captureVideoFrameAsPng(video); + if (!captured) + return; + if (!this.state.annotateSession) + await this._discardRecording(); + const frame: AnnotateFrame = { + id: 'frm-' + Math.random().toString(36).slice(2, 10), + data: captured.data, + viewportWidth: captured.viewportWidth, + viewportHeight: captured.viewportHeight, + timestamp: this.state.history.scrubTime, + annotations: [], + }; + this._pushAnnotateFrame(frame, 'user'); + } + + private _pushAnnotateFrame(frame: AnnotateFrame, initiator?: 'cli' | 'user') { const existing = this.state.annotateSession; const session: AnnotateSession = existing ? { ...existing, frames: [...existing.frames, frame], selectedFrameId: frame.id, focusAnnotationId: null } diff --git a/packages/dashboard/src/historyPlayback.ts b/packages/dashboard/src/historyPlayback.ts new file mode 100644 index 0000000000000..a7a2a677016a2 --- /dev/null +++ b/packages/dashboard/src/historyPlayback.ts @@ -0,0 +1,363 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { HistoryCluster } from './dashboardChannel'; +import type { DashboardModel } from './dashboardModel'; + +const MIME = 'video/webm; codecs="vp8"'; +const SEEK_TOLERANCE_SEC = 0.05; +const SUPPORTS_MSE = typeof window !== 'undefined' && 'MediaSource' in window && MediaSource.isTypeSupported(MIME); + +// Owns the per-session MSE playback pipeline: the