microsoft · Skn0tt · May 20, 2026
diff --git a/packages/dashboard/src/dashboard.tsx b/packages/dashboard/src/dashboard.tsx
@@ -19,6 +19,9 @@ import './dashboard.css';
 import { ChevronLeftIcon, ChevronRightIcon, LockIcon, LockOpenIcon, ReloadIcon, ScreenshotRegionIcon } from './icons';
 import { clientToViewport, getImageLayout } from './imageLayout';
 import { Recording } from './recording';
+import { HistoryPlayer } from './historyPlayer';
+import type { HistoryPlayback } from './historyPlayback';
+import { HistoryScrubber } from './historyScrubber';
 import { AnnotateSidebar, AnnotateOverlay } from './annotateView';
 
 import { ToolbarButton } from '@web/components/toolbarButton';
@@ -66,7 +69,7 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
   const [, setRevision] = React.useState(0);
   React.useEffect(() => model.subscribe(() => setRevision(r => r + 1)), [model]);
 
-  const { tabs, mode, recording, liveFrame, annotateSession, pendingCapture } = model.state;
+  const { tabs, mode, recording, liveFrame, annotateSession, pendingCapture, history } = model.state;
   const interactive = mode === 'interactive';
   const annotateActive = !!annotateSession;
   const selectedFrame = annotateSession?.frames.find(f => f.id === annotateSession.selectedFrameId) ?? null;
@@ -79,6 +82,8 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
   const viewportMainRef = React.useRef<HTMLDivElement>(null);
   const browserChromeRef = React.useRef<HTMLDivElement>(null);
   const interactiveBtnRef = React.useRef<HTMLButtonElement>(null);
+  const historyVideoRef = React.useRef<HTMLVideoElement>(null);
+  const historyPlaybackRef = React.useRef<HistoryPlayback | null>(null);
   const moveThrottleRef = React.useRef(0);
 
   const aspect = liveFrame && liveFrame.viewportWidth && liveFrame.viewportHeight
@@ -259,7 +264,9 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
             title={annotateActive ? 'Add screenshot' : 'Take screenshot'}
             disabled={!ready || pendingCapture}
             onClick={() => {
-              if (annotateActive)
+              if (history.scrubMode)
+                void model.addHistoryAnnotateFrame(historyVideoRef.current);
+              else if (annotateActive)
                 model.addAnnotateFrame();
               else
                 model.enterAnnotate('user');
@@ -368,6 +375,8 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
                 />
               </div>
               {overlayText && <div className={'screen-overlay' + (liveFrame ? ' has-frame' : '')}><span>{overlayText}</span></div>}
+              {history.scrubMode && <HistoryPlayer videoRef={historyVideoRef} playbackRef={historyPlaybackRef} model={model} time={history.scrubTime} />}
+              {history.enabled && <HistoryScrubber model={model} history={history} videoRef={historyVideoRef} playbackRef={historyPlaybackRef} />}
             </div>
           </div>
         </div>

diff --git a/packages/dashboard/src/dashboardChannel.ts b/packages/dashboard/src/dashboardChannel.ts
@@ -34,21 +34,39 @@ export type AnnotationData = { x: number; y: number; width: number; height: numb
 
 export type SubmittedAnnotationFrame = {
   data: string;
-  ariaSnapshot: string;
+  ariaSnapshot?: string;
   annotations: AnnotationData[];
-  sessionTitle: string;
-  title: string;
-  url: string;
+  sessionTitle?: string;
+  title?: string;
+  url?: string;
   viewportWidth: number;
   viewportHeight: number;
+  timestamp: number;
+};
+
+export type HistorySnapshot =
+  | { enabled: false }
+  | { enabled: true; init: string };
+
+// Cluster manifest entry shipped per cluster. Coverage end is derived
+// on the frontend as `clusters[idx+1]?.startWallMs ?? Date.now()` —
+// the live cluster grows up to "now" as frames are written.
+export type HistoryCluster = {
+  // Wallclock ms of this cluster's first frame.
+  startWallMs: number;
+  // Byte position of the cluster's first byte within the recorder webm.
+  fileOffset: number;
+  byteLen: number;
 };
 
 export type DashboardChannelEvents = {
   sessions: { sessions: SessionStatus[]; clientInfo: ClientInfo };
   tabs: { tabs: Tab[] };
-  frame: { data: string; viewportWidth: number; viewportHeight: number };
+  frame: { data: string; viewportWidth: number; viewportHeight: number; timestamp: number };
   annotate: {};
   cancelAnnotate: {};
+  history: HistorySnapshot;
+  historyCluster: HistoryCluster;
 };
 
 export type MouseButton = 'left' | 'middle' | 'right';
@@ -73,6 +91,7 @@ export interface DashboardChannel {
   startRecording(): Promise<void>;
   stopRecording(): Promise<{ streamId: string }>;
   readStream(params: { streamId: string }): Promise<{ data: string; eof: boolean }>;
+  recorderReadBytes(params: { offset: number; length: number }): Promise<{ data: string }>;
   screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number; ariaSnapshot: string }>;
   submitAnnotation(params: { frames: SubmittedAnnotationFrame[]; feedback: string }): Promise<void>;
   cancelAnnotation(): Promise<void>;

diff --git a/packages/dashboard/src/dashboardModel.ts b/packages/dashboard/src/dashboardModel.ts
@@ -16,9 +16,10 @@
 
 import { buildAnnotatedImage, saveAnnotationAsDownload } from './annotationImage';
 import { buildAnnotationZip } from './annotationZip';
+import { captureVideoFrameAsPng } from './historyPlayback';
 
 import type { Annotation } from './annotations';
-import type { DashboardChannel, DashboardChannelEvents, MouseButton, SessionStatus, SubmittedAnnotationFrame, Tab } from './dashboardChannel';
+import type { DashboardChannel, DashboardChannelEvents, HistoryCluster, HistorySnapshot, MouseButton, SessionStatus, SubmittedAnnotationFrame, Tab } from './dashboardChannel';
 import type { ClientInfo } from '../../playwright-core/src/tools/cli-client/registry';
 import type { BrowserDescriptor } from '../../playwright-core/src/serverRegistry';
 
@@ -35,10 +36,13 @@ export type AnnotateFrame = {
   data: string;
   viewportWidth: number;
   viewportHeight: number;
-  ariaSnapshot: string;
-  sessionTitle: string;
-  title: string;
-  url: string;
+  // Wall-clock ms since epoch — same coordinate space as live frame
+  // timestamps, so live and historical frames are directly comparable.
+  timestamp: number;
+  ariaSnapshot?: string;
+  sessionTitle?: string;
+  title?: string;
+  url?: string;
   annotations: Annotation[];
 };
 
@@ -50,6 +54,28 @@ export type AnnotateSession = {
   feedback: string;
 };
 
+export type HistoryState = {
+  enabled: boolean;
+  // True when the user is in scrub mode (history player visible).
+  scrubMode: boolean;
+  // Currently scrubbed time, in wall-clock ms.
+  scrubTime: number;
+  // While scrubbing, the bar's right edge is frozen here so the cursor
+  // doesn't drift left as the wall clock keeps advancing.
+  // null = use the live edge.
+  scrubFrozenEndMs: number | null;
+  // Cluster manifest pushed by the daemon. Each entry describes a
+  // wallclock start and an exact byte range in the webm, so the frontend
+  // knows precisely what's available and where to fetch it from. Reset
+  // on every `history` event; rebuilt from subsequent `historyCluster`
+  // events.
+  clusters: HistoryCluster[];
+  // Webm init segment bytes for the current recording, decoded once
+  // from the base64 in the `history` event. null when no recording is
+  // active. The MSE pipeline appends this verbatim before any cluster.
+  init: Uint8Array | null;
+};
+
 export type DashboardState = {
   // Session model state.
   sessions: SessionStatus[];
@@ -62,6 +88,7 @@ export type DashboardState = {
   pendingCapture: boolean;
   mode: Mode;
   recording: RecordingState | null;
+  history: HistoryState;
 };
 
 type Listener = () => void;
@@ -76,8 +103,26 @@ const initialState: DashboardState = {
   pendingCapture: false,
   mode: 'readonly',
   recording: null,
+  history: { enabled: false, scrubMode: false, scrubTime: 0, scrubFrozenEndMs: null, clusters: [], init: null },
 };
 
+// Returns the wall-clock range visible to the history scrubber. `startMs` is
+// the first cluster's wallclock; `endMs` is the frozen scrub-entry time while
+// scrubbing (so the cursor doesn't drift as wall-clock advances), or the
+// live edge otherwise (max of latest frame and wall clock now).
+export function historyTimeRange(history: HistoryState, liveFrame: { timestamp: number } | undefined): { startMs: number; endMs: number } {
+  const liveEdge = Math.max(liveFrame?.timestamp ?? 0, Date.now());
+  const end = history.scrubFrozenEndMs ?? liveEdge;
+  const start = history.clusters[0]?.startWallMs ?? end;
+  return { startMs: start, endMs: Math.max(start, end) };
+}
+
+function clamp(v: number, min: number, max: number): number {
+  if (max < min)
+    return min;
+  return Math.min(Math.max(v, min), max);
+}
+
 export class DashboardModel {
   state: DashboardState = initialState;
 
@@ -86,6 +131,7 @@ export class DashboardModel {
   // Monotonic token to invalidate in-flight screenshot requests when
   // pendingAnnotate is cleared or replaced.
   private _requestId = 0;
+  private _bytesCache = new Map<number, Promise<Uint8Array>>();
 
   constructor(client: DashboardChannel) {
     this._client = client;
@@ -94,6 +140,76 @@ export class DashboardModel {
     client.on('frame', params => this._emit({ liveFrame: params }));
     client.on('annotate', () => this.enterAnnotate('cli'));
     client.on('cancelAnnotate', () => this.cancelAnnotate(false));
+    client.on('history', snapshot => this._onHistorySnapshot(snapshot));
+    client.on('historyCluster', params => this._onHistoryCluster(params));
+  }
+
+  private _onHistorySnapshot(snapshot: HistorySnapshot) {
+    this._bytesCache.clear();
+    const initialState = {
+      enabled: false,
+      scrubMode: false,
+      scrubTime: 0,
+      scrubFrozenEndMs: null,
+      clusters: [],
+      init: null
+    };
+    if (!snapshot.enabled) {
+      this._emit({ history: initialState });
+      return;
+    }
+    this._emit({ history: { ...initialState, enabled: true, init: Uint8Array.fromBase64(snapshot.init) } });
+  }
+
+  private _onHistoryCluster(c: HistoryCluster) {
+    const { history } = this.state;
+    if (!history.enabled)
+      return;
+    history.clusters.push(c);
+    this._emit({ history });
+  }
+
+  enterScrub(timeMs?: number) {
+    const h = this.state.history;
+    if (!h.enabled || h.clusters.length === 0)
+      return;
+    // Freeze the bar's right edge so the cursor stays where the user
+    // put it as wall time keeps advancing.
+    const frozenEnd = Math.max(this.state.liveFrame?.timestamp ?? 0, Date.now());
+    const start = h.clusters[0].startWallMs;
+    const t = clamp(timeMs ?? frozenEnd, start, frozenEnd);
+    this._emit({ history: { ...h, scrubMode: true, scrubTime: t, scrubFrozenEndMs: frozenEnd } });
+  }
+
+  exitScrub() {
+    const h = this.state.history;
+    if (!h.scrubMode)
+      return;
+    this._emit({ history: { ...h, scrubMode: false, scrubFrozenEndMs: null } });
+  }
+
+  setScrubTime(timeMs: number) {
+    const h = this.state.history;
+    if (!h.scrubMode)
+      return;
+    const range = historyTimeRange(h, this.state.liveFrame);
+    const t = clamp(timeMs, range.startMs, range.endMs);
+    if (t === h.scrubTime)
+      return;
+    this._emit({ history: { ...h, scrubTime: t } });
+  }
+
+  readBytes(offset: number, length: number): Promise<Uint8Array | null> {
+    const cached = this._bytesCache.get(offset);
+    if (cached)
+      return cached;
+    const p = (async () => {
+      const result = await this._client.recorderReadBytes({ offset, length });
+      return Uint8Array.fromBase64(result.data);
+    })();
+    this._bytesCache.set(offset, p);
+    p.catch(() => this._bytesCache.delete(offset));
+    return p;
   }
 
   subscribe(listener: Listener): () => void {
@@ -296,6 +412,7 @@ export class DashboardModel {
         url: frame.url,
         viewportWidth: frame.viewportWidth,
         viewportHeight: frame.viewportHeight,
+        timestamp: frame.timestamp,
       });
     }
     if (session.initiator === 'cli') {
@@ -359,12 +476,36 @@ export class DashboardModel {
       data: frameData.data,
       viewportWidth: frameData.viewportWidth,
       viewportHeight: frameData.viewportHeight,
+      timestamp: Date.now(),
       ariaSnapshot: frameData.ariaSnapshot,
       sessionTitle,
       title: selectedTab?.title ?? '',
       url: selectedTab?.url ?? '',
       annotations: [],
     };
+    this._pushAnnotateFrame(frame, initiator);
+  }
+
+  async addHistoryAnnotateFrame(video: HTMLVideoElement | null) {
+    if (!video || this.state.pendingCapture)
+      return;
+    const captured = await captureVideoFrameAsPng(video);
+    if (!captured)
+      return;
+    if (!this.state.annotateSession)
+      await this._discardRecording();
+    const frame: AnnotateFrame = {
+      id: 'frm-' + Math.random().toString(36).slice(2, 10),
+      data: captured.data,
+      viewportWidth: captured.viewportWidth,
+      viewportHeight: captured.viewportHeight,
+      timestamp: this.state.history.scrubTime,
+      annotations: [],
+    };
+    this._pushAnnotateFrame(frame, 'user');
+  }
+
+  private _pushAnnotateFrame(frame: AnnotateFrame, initiator?: 'cli' | 'user') {
     const existing = this.state.annotateSession;
     const session: AnnotateSession = existing
       ? { ...existing, frames: [...existing.frames, frame], selectedFrameId: frame.id, focusAnnotationId: null }