Persist frontmatter quarantine diagnostics

Author: kavinsood

engineering/frontmatter-integrity-rfc.md

@@ -21,10 +21,15 @@ P0 and the first P1 containment pass are implemented:
   in both disk-to-CRDT and CRDT-to-disk directions
 - blocked transitions are traced, surfaced with a throttled user notice, and can
   be opted out from Advanced settings for troubleshooting
+- blocked paths persist bounded diagnostic-only quarantine metadata in plugin
+  state for restart-safe debugging
+- parser-backed validation now complements the cheap extractor on changed
+  frontmatter slices, and schema-lite field policies are used for
+  classification only
 
 Still proposed:
 
-- persisted quarantine state and full recovery UI
+- full recovery UI
 - structure-aware frontmatter sidecar
 - canonical YAML rendering
 
@@ -280,13 +285,14 @@ guards and diagnostics.
 
 ### 5. Add frontmatter validation
 
-The first containment pass uses a cheap structural classifier. It blocks only
-obvious hazards such as duplicate top-level keys, repeated bare-key bursts,
+The first containment pass uses a cheap structural classifier plus parser-backed
+validation on the extracted frontmatter slice. It blocks only obvious hazards
+such as duplicate top-level keys, repeated bare-key bursts, parser failures,
 malformed frontmatter fences, and isolated frontmatter growth bursts.
 
-That first pass is an emergency brake, not a complete YAML policy. The durable
-implementation should use a real YAML parser rather than ad hoc string
-manipulation.
+That first pass is still an emergency brake, not a complete YAML policy. The
+durable implementation should keep parser-backed validation while avoiding
+premature merge or rewrite semantics.
 
 The validator should detect:
 
@@ -353,8 +359,9 @@ The quarantine should be clearable when:
 - the user disables the guard for the path
 
 The first implementation is skip behavior plus trace/log diagnostics, a
-throttled notice, and a global opt-out. Persisted per-file quarantine state and
-explicit accept/keep recovery controls should follow.
+throttled notice, a global opt-out, and bounded persisted quarantine metadata
+for debugging. Explicit accept/keep recovery controls should follow only if they
+remain consistent with YAOS snapshot and Obsidian File Recovery policy.
 
 ## P2: Structure-aware frontmatter sync
 

package-lock.json

@@ -9,7 +9,7 @@
 		"build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
 		"build:server-release": "node build-server-release.mjs",
 		"test:server-update-local": "npm run build:server-release && node tests/server-update-local.mjs",
-		"test:regressions": "node --import jiti/register tests/diff-regressions.mjs && node --import jiti/register tests/bound-recovery-regressions.mjs && node --import jiti/register tests/frontmatter-guard-regressions.mjs && node tests/disk-mirror-regressions.mjs && node tests/markdown-ingest-regressions.mjs && node --import jiti/register tests/closed-file-mirror.ts && node --import jiti/register tests/folder-rename.ts && node --import jiti/register tests/chunked-doc-store.ts && node --import jiti/register tests/trace-store.ts && node --import jiti/register tests/server-hardening.ts && node --import jiti/register tests/v2-offline-rename-regressions.mjs",
+		"test:regressions": "node --import jiti/register tests/diff-regressions.mjs && node --import jiti/register tests/bound-recovery-regressions.mjs && node --import jiti/register tests/frontmatter-guard-regressions.mjs && node --import jiti/register tests/frontmatter-quarantine-regressions.mjs && node tests/disk-mirror-regressions.mjs && node tests/markdown-ingest-regressions.mjs && node --import jiti/register tests/closed-file-mirror.ts && node --import jiti/register tests/folder-rename.ts && node --import jiti/register tests/chunked-doc-store.ts && node --import jiti/register tests/trace-store.ts && node --import jiti/register tests/server-hardening.ts && node --import jiti/register tests/v2-offline-rename-regressions.mjs",
 		"test:integration:worker": "node tests/worker-integration.mjs",
 		"test:e2e:obsidian": "wdio run wdio.conf.mts",
 		"test:ci": "npm run test:regressions && npm run test:integration:worker",
@@ -35,6 +35,7 @@
 	"dependencies": {
 		"fast-diff": "^1.3.0",
 		"fflate": "^0.8.2",
+		"js-yaml": "^4.1.1",
 		"obsidian": "1.8.7",
 		"partyserver": "0.3.2",
 		"qrcode": "^1.5.4",

package.json

@@ -0,0 +1,9 @@
+declare module "js-yaml" {
+	export function load(yaml: string): unknown;
+
+	const yaml: {
+		load: typeof load;
+	};
+
+	export default yaml;
+}

src/js-yaml.d.ts

@@ -25,8 +25,16 @@ import { applyDiffToYText } from "./sync/diff";
 import {
 	isFrontmatterBlocked,
 	validateFrontmatterTransition,
+	extractFrontmatter,
 	type FrontmatterValidationResult,
 } from "./sync/frontmatterGuard";
+import {
+	buildFrontmatterQuarantineDebugLines,
+	clearFrontmatterQuarantinePath,
+	readPersistedFrontmatterQuarantine,
+	upsertFrontmatterQuarantineEntry,
+	type FrontmatterQuarantineEntry,
+} from "./sync/frontmatterQuarantine";
 import {
 	type DiskIndex,
 	collectFileStats,
@@ -77,6 +85,7 @@ type PersistedPluginState = Partial<VaultSyncSettings> & {
 	_blobQueue?: BlobQueueSnapshot;
 	_serverCapabilitiesCache?: PersistedServerCapabilitiesCache;
 	_updateManifestCache?: PersistedUpdateManifestCache;
+	_frontmatterQuarantine?: FrontmatterQuarantineEntry[];
 };
 
 /** Minimum interval between reconcile runs (prevents rapid reconnect churn). */
@@ -286,6 +295,7 @@ export default class VaultCrdtSyncPlugin extends Plugin {
 	private commandsRegistered = false;
 	private idbDegradedHandled = false;
 	private frontmatterGuardNoticeAt = new Map<string, number>();
+	private frontmatterQuarantineEntries: FrontmatterQuarantineEntry[] = [];
 
 	/**
 	 * True when startup timed out waiting for provider sync.
@@ -449,7 +459,15 @@ export default class VaultCrdtSyncPlugin extends Plugin {
 				this.settings.debug,
 				(source, msg, details) => this.trace(source, msg, details),
 				() => this.settings.frontmatterGuardEnabled,
-				(path, direction) => this.showFrontmatterGuardNotice(path, direction),
+				(path, direction, reason, validation, previousContent, nextContent) =>
+					this.handleFrontmatterValidation(
+						path,
+						direction,
+						reason,
+						validation,
+						previousContent,
+						nextContent,
+					),
 			);
 			this.diskMirror.startMapObservers();
 
@@ -2167,24 +2185,49 @@ export default class VaultCrdtSyncPlugin extends Plugin {
 		if (!this.settings.frontmatterGuardEnabled) return false;
 
 		const validation = validateFrontmatterTransition(previousContent, nextContent);
-		if (!isFrontmatterBlocked(validation)) return false;
-
-		this.traceFrontmatterQuarantine(
+		this.handleFrontmatterValidation(
 			path,
 			"disk-to-crdt",
 			reason,
 			validation,
-			previousContent?.length ?? null,
-			nextContent.length,
+			previousContent,
+			nextContent,
 		);
+		if (!isFrontmatterBlocked(validation)) return false;
 		this.log(
 			`Frontmatter ingest blocked for "${path}" ` +
 			`(${validation.reasons.join(", ") || validation.risk})`,
 		);
-		this.showFrontmatterGuardNotice(path, "disk-to-crdt");
 		return true;
 	}
 
+	private handleFrontmatterValidation(
+		path: string,
+		direction: "disk-to-crdt" | "crdt-to-disk",
+		reason: string,
+		validation: FrontmatterValidationResult,
+		previousContent: string | null,
+		nextContent: string,
+	): void {
+		if (validation.risk === "ok") {
+			void this.clearFrontmatterQuarantine(path, `${direction}:${reason}`);
+			return;
+		}
+
+		if (!isFrontmatterBlocked(validation)) return;
+
+		this.traceFrontmatterQuarantine(
+			path,
+			direction,
+			reason,
+			validation,
+			previousContent?.length ?? null,
+			nextContent.length,
+		);
+		this.showFrontmatterGuardNotice(path, direction);
+		void this.persistFrontmatterQuarantine(path, direction, validation, previousContent, nextContent);
+	}
+
 	private showFrontmatterGuardNotice(
 		path: string,
 		direction: "disk-to-crdt" | "crdt-to-disk",
@@ -2223,6 +2266,44 @@ export default class VaultCrdtSyncPlugin extends Plugin {
 		});
 	}
 
+	private async persistFrontmatterQuarantine(
+		path: string,
+		direction: "disk-to-crdt" | "crdt-to-disk",
+		validation: FrontmatterValidationResult,
+		previousContent: string | null,
+		nextContent: string,
+	): Promise<void> {
+		const now = Date.now();
+		const prevHash = await this.hashFrontmatterContent(previousContent);
+		const nextHash = await this.hashFrontmatterContent(nextContent);
+		this.frontmatterQuarantineEntries = upsertFrontmatterQuarantineEntry(
+			this.frontmatterQuarantineEntries,
+			{
+				path,
+				firstSeenAt: now,
+				lastSeenAt: now,
+				direction,
+				reasons: validation.reasons,
+				prevHash,
+				nextHash,
+				count: 1,
+			},
+		);
+		await this.persistPluginState();
+	}
+
+	private async clearFrontmatterQuarantine(path: string, reason: string): Promise<void> {
+		if (this.frontmatterQuarantineEntries.length === 0) return;
+		const nextEntries = clearFrontmatterQuarantinePath(this.frontmatterQuarantineEntries, path);
+		if (nextEntries.length === this.frontmatterQuarantineEntries.length) return;
+		this.frontmatterQuarantineEntries = nextEntries;
+		this.trace("trace", "frontmatter-quarantine-cleared", {
+			path,
+			reason,
+		});
+		await this.persistPluginState();
+	}
+
 	private async updateDiskIndexForPath(path: string): Promise<void> {
 		try {
 			const stat = await this.app.vault.adapter.stat(path);
@@ -2667,6 +2748,7 @@ export default class VaultCrdtSyncPlugin extends Plugin {
 			this.updateManifest = null;
 			this.updateManifestFetchedAt = 0;
 		}
+		this.frontmatterQuarantineEntries = readPersistedFrontmatterQuarantine(data?._frontmatterQuarantine);
 		this.refreshPersistedState();
 		if (migratedSettings) {
 			await this.persistPluginState();
@@ -3515,6 +3597,11 @@ export default class VaultCrdtSyncPlugin extends Plugin {
 		} else {
 			delete nextState._updateManifestCache;
 		}
+		if (this.frontmatterQuarantineEntries.length > 0) {
+			nextState._frontmatterQuarantine = this.frontmatterQuarantineEntries;
+		} else {
+			delete nextState._frontmatterQuarantine;
+		}
 		this.persistedState = nextState;
 	}
 
@@ -3572,6 +3659,7 @@ export default class VaultCrdtSyncPlugin extends Plugin {
 			`Open files: ${this.openFilePaths.size}`,
 			`Server trace events: ${this.recentServerTrace.length}`,
 			`Remote cursors: ${this.settings.showRemoteCursors ? "shown" : "hidden"}`,
+			...buildFrontmatterQuarantineDebugLines(this.frontmatterQuarantineEntries),
 		].join("\n");
 	}
 
@@ -3595,6 +3683,13 @@ export default class VaultCrdtSyncPlugin extends Plugin {
 		return arrayBufferToHex(digest);
 	}
 
+	private async hashFrontmatterContent(content: string | null): Promise<string | undefined> {
+		if (content == null) return undefined;
+		const block = extractFrontmatter(content);
+		if (block.kind !== "present") return undefined;
+		return await this.sha256Hex(block.frontmatterText);
+	}
+
 	private async exportDiagnostics(): Promise<void> {
 		if (!this.vaultSync) {
 			new Notice("Sync not initialized");

src/main.ts

@@ -104,10 +104,13 @@ export class DiskMirror {
 		debug: boolean,
 		private trace?: TraceRecord,
 		private frontmatterGuardEnabled: () => boolean = () => true,
-		private onFrontmatterBlocked?: (
+		private onFrontmatterValidated?: (
 			path: string,
 			direction: "crdt-to-disk",
+			reason: "flush-write",
 			validation: FrontmatterValidationResult,
+			previousContent: string | null,
+			nextContent: string,
 		) => void,
 	) {
 		this.debug = debug;
@@ -489,6 +492,14 @@ export class DiskMirror {
 		if (!this.frontmatterGuardEnabled()) return false;
 
 		const validation = validateFrontmatterTransition(previousContent, nextContent);
+		this.onFrontmatterValidated?.(
+			path,
+			"crdt-to-disk",
+			"flush-write",
+			validation,
+			previousContent,
+			nextContent,
+		);
 		if (!isFrontmatterBlocked(validation)) return false;
 
 		this.trace?.("trace", "frontmatter-quarantined", {
@@ -506,7 +517,6 @@ export class DiskMirror {
 			`frontmatter write blocked for "${path}" ` +
 			`(${validation.reasons.join(", ") || validation.risk})`,
 		);
-		this.onFrontmatterBlocked?.(path, "crdt-to-disk", validation);
 		return true;
 	}