diff --git a/.gitattributes b/.gitattributes index f254f0cc31..f175a8e192 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,5 @@ * text=auto eol=lf *.bin binary poc/evidence/**/*.typescript -text +# Sealed fixtures are hashed byte-exactly by tests; no line-ending conversion. +core/crates/layerfs-content/tests/fixtures/filesystem/** -text diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bad85750f6..a275eb7875 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,3 +123,39 @@ jobs: # shellcheck disable=SC2086 # one --test flag per word timeout 120 cargo +1.85.1 nextest run --manifest-path Cargo.toml --locked \ --workspace --exclude layerfs-node $targets --profile ci -E "$filter" + + # The Windows-native Store route's receipts, re-runnable on every push: + # the cfg(windows) suites the Linux lanes compile out — the store + # serialization and abandoned-mutex lanes, the WTF-8 name round trip and + # its end-to-end Init, the alias/dangling-reparse refusals, the overlay + # identity/allocation/deletion, the kernel guard, the owner-only DACLs, + # and the content name grammar. Report-only like every job here. The Node + # binding stays outside CI (owner, 2026-10-23), as above. + windows: + name: windows store suites + runs-on: windows-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v7.0.1 + - run: rustup toolchain install 1.85.1 --profile minimal + - uses: Swatinem/rust-cache@v2.9.2 + with: + workspaces: . -> target + save-if: ${{ github.ref == 'refs/heads/main' }} + # Build first so compile time is never counted against the test ceiling; + # only the five crates carrying Windows-gated suites are built. + - name: Build test binaries + run: > + cargo +1.85.1 test --manifest-path Cargo.toml --locked --no-run + -p layerfs-content -p layerfs-persistence -p layerfs-project + -p layerfs-overlay -p layerfs-store-service + - name: Run windows store suites (each crate capped at 120 s) + shell: bash + run: | + status=0 + for crate in layerfs-content layerfs-persistence layerfs-project \ + layerfs-overlay layerfs-store-service; do + echo "## $crate" + timeout 120 cargo +1.85.1 test --manifest-path Cargo.toml --locked -p "$crate" || status=1 + done + exit $status diff --git a/.gitignore b/.gitignore index 70c937bd24..7387221bc0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ target/ # The Node addon that api/scripts/build-native.mjs builds beside its crate. /engine/crates/layerfs-node/layerfs.node +# Dead post-restructure paths where only local build output ever lands +# (K3-R3 F1: a stale addon copy and pnpm trees entered a merge once). +/sdk/js/native/layerfs.node +/dsh/ +# A hardened-probe run directory that a mistaken CWD once committed whole. +/api/agentharden-run/ /target-linux/ /eval/runs/ /eval/datasets/ diff --git a/AGENTS.md b/AGENTS.md index 7ec8a92359..2bde3e0383 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,6 +23,7 @@ simplification. | Cluster one | Core Content/Storage/History/Telemetry algorithms; Engine Persistence and Project Init | [Cluster-one handbook](docs/general/cluster_one_handbook.md), [CAS/CDC/delta handbook](docs/general/cas_cdc_deltaencoding_handbook.md) | | Cluster two | Engine Workspace/FUSE/daemon/Sandbox; API execution and Commit/runtime integration | [Current design index](core/docs/issues/303/README.md) and its seven primary contracts | | Legacy reference | v0.1.6, retired from `main` by R9c; read at the recovery tag only | [R9c record](core/docs/issues/307/R9C-COMPLETION-20261010.md) | +| Windows-native Store | Store engine and the SDK's in-process Store face on native Windows (owner, 2026-10-25); FUSE, daemon and Workspace stay Linux-only | [Direct Store contract](core/docs/issues/352/DIRECT_STORE.md) | **The LayerFS UI is the dsh LayerFS plugin** (owner, 2026-10-19): the LayerFS tab in DeepSeek Harness. It lives in its own repository, @@ -154,6 +155,29 @@ remains a separate MEMORY/OFF/EXCLUSIVE database file. native handles and project discovery. This is independent of the macOS adapter and changes neither the product SDK nor the daemon. A lost reply is never replayed, and an uncertain Init retains its Linux home and volume. +- **Windows-native Store route (owner direction 2026-10-25).** The Store + engine (SQLite, the cross-process writer lane, the kernel guard) and the + SDK's in-process Store face (fs, branch, project, operator) also run + natively on Windows — the sandbox family, including + `project.list_sandboxes`, still refuses `Unsupported` there: local NTFS + volumes only, a named kernel mutex as the + cross-process writer lane, real volume-serial + file-index identity for + acquisition rows, and bundled SQLite, each in audited FFI modules + (engine/AGENTS.md). No WSL2 and no daemon are involved. This supersedes the + Linux-only Store sentence above for this route only; Workspace, FUSE, Exec + and daemon calls still refuse `Unsupported` on Windows, and the dsh macOS + and WSL2 exceptions are unchanged. Recorded limitations: Windows Init has + no hard-link membership and synthesizes the mode from the read-only + attribute; names travel as lossless WTF-8 — **owner-ratified 2026-10-31 + ("可以合并") as the long-term content grammar** (an unpaired surrogate's + three-byte form is accepted, a pair spelled apart is refused; JS callers + cannot spell such names — listings render them empty and a JS write names + an U+FFFD sibling; the earlier + lossy-names limitation is superseded) — the backing-inside-source check + walks the audited volume-serial + file-index identity, and overlay + allocation is the real on-disk footprint; homes, discovery records and + Stores carry owner-only DACLs. See + [the Windows-native route](core/docs/issues/352/DIRECT_STORE.md). - **Plugin-neutral LayerFS (owner, 2026-10-24).** LayerFS is agnostic to dsh and every other plugin. The macOS and Windows worker routes above are open to any integration: `@layerfs/runtime` starts the worker on those platforms @@ -291,7 +315,8 @@ remain binding; moving them out of this file does not weaken them. failure/uncertainty and custody; no guessed resend, deletion or success. - **CI reports; it never gates (owner, 2026-10-23).** GitHub Actions runs a fast lane after each push or pull request (`.github/workflows/ci.yml`: fmt, - guards, Clippy and the fast tests) and a nightly lane (`nightly.yml`: slow and + guards, Clippy, the fast tests and — from the Windows-native route — a + report-only windows store-suites job) and a nightly lane (`nightly.yml`: slow and FUSE mount tests); `format.yml` checks the Ruff formatting below. [tools/ci_tests.py](tools/ci_tests.py) decides which tests go in which lane. None is a required check, and nothing runs before a push. Do diff --git a/Cargo.lock b/Cargo.lock index ca39be0e98..f054fccae9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -636,6 +636,8 @@ dependencies = [ "napi", "napi-build", "napi-derive", + "sha2 0.11.0", + "windows-sys", ] [[package]] @@ -645,6 +647,7 @@ dependencies = [ "blake3", "nix", "rusqlite", + "windows-sys", ] [[package]] @@ -658,6 +661,8 @@ dependencies = [ "layerfs-telemetry", "nix", "rusqlite", + "sha2 0.11.0", + "windows-sys", ] [[package]] @@ -671,6 +676,7 @@ dependencies = [ "layerfs-storage", "layerfs-telemetry", "sha2 0.11.0", + "windows-sys", ] [[package]] @@ -704,6 +710,7 @@ dependencies = [ "layerfs-telemetry", "layerfs-workspace", "nix", + "windows-sys", ] [[package]] diff --git a/api/README.md b/api/README.md index 9b7cd26445..20fab01b6d 100644 --- a/api/README.md +++ b/api/README.md @@ -32,10 +32,19 @@ the others as it grows: them, and on macOS and Windows starts the worker: add [`@layerfs/runtime`](docs/runtime.md). -Store calls run in the calling process on Linux only. On macOS and Windows they -go through the LayerFS home's worker, which `@layerfs/runtime` starts; see +Store calls run in the calling process on Linux, and on Windows by default +(the Windows-native route below) when no worker endpoint is set. On macOS — +and on Windows when a worker endpoint is set — they go through the LayerFS +home's worker, which `@layerfs/runtime` starts; see [where calls run](docs/agent.md#where-calls-run) and [platforms and the worker](docs/runtime.md#platforms-and-the-worker). +On Windows the native direct route (owner direction 2026-10-25) is the +default: the binding built from this tree opens a local NTFS Store itself — +fs, branch, project and operator calls in this process (on Windows, +`project.list_sandboxes` and the sandbox family still refuse `Unsupported`), +no worker and no daemon; workspace, exec and daemon calls refuse +`Unsupported`. See +[the Windows-native route](../core/docs/issues/352/DIRECT_STORE.md). ## The folder @@ -75,7 +84,14 @@ addon. It uses Cargo +1.85.1, `--offline --locked --release`, and the repository ARM64 inputs. `-- --debug` selects dev. The artifact is atomically renamed into place, avoiding stale macOS executable-page signature state. `CARGO_TARGET_DIR` must stay inside this worktree; `LAYERFS_NATIVE_OUTPUT` can select a separate -platform artifact. Keep Linux addons separate from the host addon. +platform artifact. Keep Linux addons separate from the host addon. The same +command builds on Windows: the script runs Cargo `+1.85.1 --release --locked` +against the repository-root manifest and stages +`target//layerfs_node.dll` to the addon path. A first build needs +the cargo registry warmed (`--offline` cannot fetch); a clean machine +installs rustup, the 1.85.1 toolchain and VS Build Tools (link.exe) first. +Node ≥ 20 is the SDK's runtime floor (`api/package.json` engines); the +build script itself needs no newer Node than that. At runtime, the Docker Engine server selected by `LAYERFS_DOCKER` must speak Engine API v1.48 (Docker 28.0) or newer. Before its first request the SDK reads diff --git a/api/agent/src/errors.js b/api/agent/src/errors.js index 46a2321a35..5e21fdbe81 100644 --- a/api/agent/src/errors.js +++ b/api/agent/src/errors.js @@ -22,6 +22,11 @@ export const CODES = Object.freeze([ "NotText", "NoMatch", "AmbiguousMatch", "Exists", "NotEmpty", "NameTaken", "ReservedName", "ReadOnly", "MountsActive", "TooLarge", "InvalidInput", "NotMounted", "NoSuchSandbox", "NoSuchProcess", "Capacity", "NotComparable", + // The Windows-native Store route (and the shared refusals under it) also + // emits: a privacy-boundary or platform refusal, a failed operation with + // custody detail, a missing Store path, an unknown method, and the store + // file refusing to open as itself (link substitution or widened DACL). + "Unauthorized", "Unsupported", "Failed", "NoStore", "UnknownMethod", ]); /** True when `error` is a LayerFS refusal with `code`. */ diff --git a/api/agent/src/transport/inprocess.js b/api/agent/src/transport/inprocess.js index 6d8989530e..7c74d165cf 100644 --- a/api/agent/src/transport/inprocess.js +++ b/api/agent/src/transport/inprocess.js @@ -1,14 +1,16 @@ // The in-process transport: the real native addon in this process, from // LAYERFS_NATIVE, else engine/crates/layerfs-node/layerfs.node. Store calls open the -// project's Store here (Linux only, #352 F-020). +// project's Store here: on Linux and on Windows's native route (#352 +// F-020); on macOS through the worker instead. import { createRequire } from "node:module"; import { fileURLToPath } from "node:url"; const require = createRequire(import.meta.url); -// Every binding call runs on Node's worker pool, four threads unless the -// process sets UV_THREADPOOL_SIZE. A Sandbox exec holds its thread for the -// command's whole run, so with four agents' commands running every other call +// Every binding call runs on Node's worker pool, sized 64 by default here +// (WORKERS below) unless the process sets UV_THREADPOOL_SIZE itself. A +// Sandbox exec holds its thread for the +// command's whole run, so with many agents' commands running every other call // queued behind them (#352 F-017). Node reads the size when the pool first // starts; a process that set it keeps its own value. export const WORKERS = 64; diff --git a/api/agent/test/init_refusals.mjs b/api/agent/test/init_refusals.mjs index 57ed60d3ee..7c51946303 100644 --- a/api/agent/test/init_refusals.mjs +++ b/api/agent/test/init_refusals.mjs @@ -20,6 +20,14 @@ import { import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; +if (process.platform === "win32") { + // Every refusal this test drives is a unix mode-bit or filesystem-type + // contract (setgid, setuid, sticky, overlayfs, FIFO); Windows carries + // none of them. The Windows Store-face lifecycle is store_lifecycle.mjs. + console.log("init_refusals: unix-only contracts, not run on Windows"); + process.exit(0); +} + const OVERLAYFS = 0x794c7630; const work = mkdtempSync(join(tmpdir(), "layerfs-init-refusals-")); const home = join(work, "home"); diff --git a/api/agent/test/reinit.mjs b/api/agent/test/reinit.mjs index e4b92b22ce..a8cdb3fa81 100644 --- a/api/agent/test/reinit.mjs +++ b/api/agent/test/reinit.mjs @@ -39,14 +39,18 @@ try { }; const layerfs = client(); - // A set-group-ID source directory has a mode LayerFS cannot store. - chmodSync(source, 0o2755); - await assert.rejects(layerfs.project.init(source), { - code: "InvalidInput", - detail: { reason: "Mode", path: source, mode: "2755" }, - }); - assert.ok(!existsSync(join(work, "home", "website")), "a refused Init leaves no entry"); - chmodSync(source, 0o755); + // A set-group-ID source directory has a mode LayerFS cannot store. The + // bit does not exist on Windows, whose chmod maps only the read-only + // attribute; the refusal contract is checked on unix platforms. + if (process.platform !== "win32") { + chmodSync(source, 0o2755); + await assert.rejects(layerfs.project.init(source), { + code: "InvalidInput", + detail: { reason: "Mode", path: source, mode: "2755" }, + }); + assert.ok(!existsSync(join(work, "home", "website")), "a refused Init leaves no entry"); + chmodSync(source, 0o755); + } const first = await layerfs.project.init(source); assert.equal(first.kind, "Initialized"); diff --git a/api/agent/test/store_concurrent.mjs b/api/agent/test/store_concurrent.mjs new file mode 100644 index 0000000000..e93f5e0881 --- /dev/null +++ b/api/agent/test/store_concurrent.mjs @@ -0,0 +1,93 @@ +// Cross-process Store writers on the real native binding (owner direction +// 2026-10-25, route 2): the parent initializes a project and forks one +// branch, then four re-exec'd child processes write distinct files to that +// branch concurrently, each through its own client of the same Store - the +// direct route's cross-process writer lane and the discovery record guard +// serialize them; every write must land. The parent reads all four files +// back and counts the Commits. No Docker, no daemon, no WSL2; on Linux the +// Store sits on the E2E launcher's volume (LAYERFS_STORE_PATH): +// node api/agent/test/store_concurrent.mjs # parent run +// node api/agent/test/store_concurrent.mjs --child +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + +// One child writer: open the project's Store and write its own file once. +if (process.argv[2] === "--child") { + const [, , , home, store, name] = process.argv; + process.env.LAYERFS_HOME = home; + const { LayerFS } = await import("@layerfs/agent"); + process.env.LAYERFS_STORE_PATH = store; + const writer = await LayerFS.open("concurrent"); + const body = `written by ${name} in pid ${process.pid}\n`; + const done = await writer.fs.write("load", `/${name}.md`, body, `write ${name}`); + await writer.close(); + assert.equal(done.kind, "Committed"); + process.exit(0); +} + +const WRITERS = 4; +const work = mkdtempSync(join(tmpdir(), "layerfs-concurrent-")); +const volume = process.env.LAYERFS_STORE_PATH + ? join(dirname(dirname(process.env.LAYERFS_STORE_PATH)), `concurrent-${process.pid}`) + : join(work, "volume"); +const store = join(volume, "global", "store.sqlite"); +const home = join(work, "home"); +const retryRemove = (path) => { + for (let attempt = 0; attempt < 5; attempt++) { + try { rmSync(path, { recursive: true, force: true }); return; } catch { } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 200); + } +}; +try { + const source = join(work, "source"); + mkdirSync(source); + writeFileSync(join(source, "README.md"), "concurrent writers\n"); + process.env.LAYERFS_HOME = home; + mkdirSync(join(volume, "global"), { recursive: true }); + const { LayerFS } = await import("@layerfs/agent"); + process.env.LAYERFS_STORE_PATH = store; + const parent = await LayerFS.init("concurrent", source); + await parent.branch.fork("load"); + const base = (await parent.branch.list_commits("load")).items.length; + + const children = []; + for (let n = 1; n <= WRITERS; n++) { + children.push(spawn(process.execPath, [fileURLToPath(import.meta.url), "--child", home, store, `w${n}`], { + stdio: ["ignore", "pipe", "pipe"], + env: { ...process.env, LAYERFS_STORE_PATH: store }, + })); + } + const reports = await Promise.all(children.map(async (child, index) => { + const [code, output] = await new Promise((resolve) => { + let captured = ""; + child.stdout.on("data", (chunk) => { captured += chunk; }); + child.stderr.on("data", (chunk) => { captured += chunk; }); + const timer = setTimeout(() => { + child.kill(); + resolve([-1, `${captured} (killed: 30s ceiling)`]); + }, 30_000); + child.on("exit", (exitCode) => { clearTimeout(timer); resolve([exitCode, captured]); }); + }); + return { index, code, output }; + })); + for (const { index, code, output } of reports) { + assert.equal(code, 0, `writer w${index + 1} failed (${code}): ${output.slice(0, 400)}`); + } + + // Every writer's file is committed and readable; the branch carries them all. + for (let n = 1; n <= WRITERS; n++) { + const read = await parent.fs.read("load", `/w${n}.md`); + assert.ok(read.text.startsWith(`written by w${n} in pid `), read.text); + } + const commits = await parent.branch.list_commits("load", 50); + assert.equal(commits.items.length, base + WRITERS, `${commits.items.length} commits, expected ${base + WRITERS}`); + await parent.close(); + console.log(`store_concurrent: ok (${WRITERS} writers, ${commits.items.length} commits)`); +} finally { + retryRemove(work); + retryRemove(volume); +} diff --git a/api/agent/test/store_lifecycle.mjs b/api/agent/test/store_lifecycle.mjs new file mode 100644 index 0000000000..b0be999cc4 --- /dev/null +++ b/api/agent/test/store_lifecycle.mjs @@ -0,0 +1,48 @@ +// The Store-face lifecycle on the real native binding (owner direction +// 2026-10-25, route 2): project init, fork, a one-Commit fs.write, reading +// back on the new branch and on main, stat, the Commit listing and close — +// all in this process through the SDK's direct Store route. No Docker, no +// daemon, no WSL2. The Store's volume is LAYERFS_STORE_PATH's parent when +// the E2E launcher provides one (Linux); otherwise this machine's temporary +// volume, which the Windows route guards as NTFS: +// node api/agent/test/store_lifecycle.mjs +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + +const work = mkdtempSync(join(tmpdir(), "layerfs-lifecycle-")); +const volume = process.env.LAYERFS_STORE_PATH + ? join(dirname(dirname(process.env.LAYERFS_STORE_PATH)), `lifecycle-${process.pid}`) + : join(work, "volume"); +const store = join(volume, "global", "store.sqlite"); +try { + const source = join(work, "source"); + mkdirSync(source); + writeFileSync(join(source, "a.txt"), "hello windows-native store\n"); + process.env.LAYERFS_HOME = join(work, "home"); + mkdirSync(join(volume, "global"), { recursive: true }); + const { LayerFS } = await import("@layerfs/agent"); + process.env.LAYERFS_STORE_PATH = store; + const layerfs = await LayerFS.init("lifecycle", source); + + assert.equal(layerfs.created.kind, "Initialized"); + await layerfs.branch.fork("dev"); + + const body = "written by the native Store face\n"; + assert.equal((await layerfs.fs.write("dev", "/windows.md", body, "lifecycle write")).kind, "Committed"); + + assert.equal((await layerfs.fs.read("dev", "/windows.md")).text, body); + assert.equal((await layerfs.fs.read("main", "/a.txt")).text, "hello windows-native store\n"); + assert.equal((await layerfs.fs.stat("dev", "/windows.md")).size, Buffer.byteLength(body)); + + const commits = await layerfs.branch.list_commits("dev"); + const records = commits.records ?? commits.items ?? commits; + assert.ok(Array.isArray(records) && records.length >= 1, JSON.stringify(commits).slice(0, 200)); + + await layerfs.close(); + console.log(`store_lifecycle: ok (${records.length} commit listed)`); +} finally { + rmSync(work, { recursive: true, force: true }); + rmSync(volume, { recursive: true, force: true }); +} diff --git a/api/agent/test/store_matrix.mjs b/api/agent/test/store_matrix.mjs new file mode 100644 index 0000000000..11fa4d311b --- /dev/null +++ b/api/agent/test/store_matrix.mjs @@ -0,0 +1,130 @@ +// The full Store method matrix on the real native binding (owner direction +// 2026-10-25, route 2): every fs, branch and project operation the direct +// route serves — writes (write, edit, mkdir, cp, mv, rm, apply), reads +// (read with a line range, list, stat, grep, glob, blame), comparisons +// (diff_files, diff_commits, diff_branches) and branch combination +// (merge, publish) — plus the Windows refusal of the daemon-only workspace +// surface. No Docker, no daemon, no WSL2; on Linux the Store sits on the +// E2E launcher's volume (LAYERFS_STORE_PATH): +// node api/agent/test/store_matrix.mjs +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + +const work = mkdtempSync(join(tmpdir(), "layerfs-matrix-")); +const volume = process.env.LAYERFS_STORE_PATH + ? join(dirname(dirname(process.env.LAYERFS_STORE_PATH)), `matrix-${process.pid}`) + : join(work, "volume"); +const store = join(volume, "global", "store.sqlite"); +const retryRemove = (path) => { + for (let attempt = 0; attempt < 5; attempt++) { + try { rmSync(path, { recursive: true, force: true }); return; } catch { } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 200); + } +}; +try { + const source = join(work, "source"); + mkdirSync(join(source, "lib"), { recursive: true }); + writeFileSync(join(source, "README.md"), "one\ntwo\nthree\n"); + writeFileSync(join(source, "lib/app.js"), "const a = 1;\nconst b = 2;\n"); + process.env.LAYERFS_HOME = join(work, "home"); + mkdirSync(join(volume, "global"), { recursive: true }); + const { LayerFS } = await import("@layerfs/agent"); + process.env.LAYERFS_STORE_PATH = store; + const layerfs = await LayerFS.init("matrix", source); + + assert.equal(layerfs.created.kind, "Initialized"); + const branches = await layerfs.project.list_branches(); + assert.deepEqual(branches.items.map((item) => item.name), ["main"]); + await layerfs.branch.fork("work"); + + // Writes: each call is one Commit; the surface changes shape together. + await layerfs.fs.mkdir("work", "/docs", "make docs"); + await layerfs.fs.write("work", "/docs/note.md", "alpha\n", "first note"); + const second = await layerfs.fs.write("work", "/docs/note.md", "alpha\nbeta\n", "second note"); + assert.equal(second.kind, "Committed"); + const edited = await layerfs.fs.edit("work", "/docs/note.md", "alpha", "gamma", "reword"); + assert.equal(edited.kind, "Committed"); + assert.notEqual(edited.head, second.head, "each write call is one new Commit"); + await layerfs.fs.cp("work", "/docs/note.md", "/docs/copy.md", "copy"); + await layerfs.fs.mv("work", "/docs/copy.md", "/docs/moved.md", "move"); + await layerfs.fs.rm("work", "/docs/moved.md", "remove"); + const listing = await layerfs.fs.list("work", "/docs"); + assert.deepEqual(listing.items.map((item) => item.name).sort(), ["note.md"]); + + // Reads: content, shape and search over the written tree. + assert.equal((await layerfs.fs.read("work", "/docs/note.md")).text, "gamma\nbeta\n"); + const ranged = await layerfs.fs.read("work", "/README.md", 1, 2); + assert.equal(ranged.text, "one\ntwo\n"); + assert.equal((await layerfs.fs.stat("work", "/docs/note.md")).size, Buffer.byteLength("gamma\nbeta\n")); + const grep = await layerfs.fs.grep("work", "gamma"); + assert.deepEqual(grep.items.map((hit) => hit.path), ["/docs/note.md"]); + assert.deepEqual((await layerfs.fs.glob("work", "**/*.js")).items, ["/lib/app.js"]); + const blame = await layerfs.fs.blame("work", "/docs/note.md"); + assert.equal(blame.lines.length, 2); + assert.ok(blame.lines.every((line) => typeof line.commit === "string")); + + // Comparisons: file, commit and branch. + assert.equal((await layerfs.fs.diff_files("main", "/README.md", "work", "/README.md")).kind, "Equal"); + const changed = await layerfs.fs.diff_files("main", "/README.md", "work", "/docs/note.md"); + assert.notEqual(changed.kind, "Equal"); + const commits = await layerfs.branch.list_commits("work"); + assert.ok(commits.items.length >= 6, `${commits.items.length} commits listed`); + // main is the project's main line, not a Branch: it moves only through + // branch.publish and is refused by the Branch name index. + await assert.rejects(layerfs.branch.list_commits("main"), { code: "NoSuchBranch" }); + const oldest = commits.items[commits.items.length - 1]; + const commitDiff = await layerfs.branch.diff_commits(oldest.id, commits.items[0].id); + assert.ok(JSON.stringify(commitDiff).includes("/docs/note.md"), JSON.stringify(commitDiff).slice(0, 200)); + const branchDiff = await layerfs.branch.diff_branches("main", "work"); + assert.equal(branchDiff.kind, "ok"); + + // Combination: merge reads both lines; publish moves main's head. + const merged = await layerfs.branch.merge("main", "work", "merged", "combine"); + assert.equal(merged.kind, "Merged"); + assert.equal((await layerfs.fs.read("merged", "/docs/note.md")).text, "gamma\nbeta\n"); + const published = await layerfs.branch.publish("work"); + assert.ok(published.kind, JSON.stringify(published).slice(0, 200)); + assert.equal((await layerfs.fs.read("main", "/docs/note.md")).text, "gamma\nbeta\n"); + // A divergent merge: both sides advanced past their common base, so this + // is a real merge Commit (two parents), and both sides' files must be + // readable on the merged Branch and on main after its publish. + await layerfs.fs.write("work", "/later.md", "later\n", "work advanced"); + await layerfs.branch.fork("feat2"); + await layerfs.fs.write("feat2", "/feat2.md", "feat2 line\n", "feat2 advanced"); + const combined = await layerfs.branch.merge("work", "feat2", "combined", "combine diverged"); + assert.equal(combined.kind, "Merged"); + assert.equal(combined.fastForward, false, "diverged sources cannot fast-forward"); + assert.equal((await layerfs.fs.read("combined", "/later.md")).text, "later\n"); + assert.equal((await layerfs.fs.read("combined", "/feat2.md")).text, "feat2 line\n"); + const combinedHead = (await layerfs.branch.list_commits("combined", 1)).items[0]; + assert.ok(combinedHead.secondParent, "a divergent merge Commit carries two parents"); + await layerfs.branch.publish("combined"); + assert.equal((await layerfs.fs.read("main", "/later.md")).text, "later\n"); + assert.equal((await layerfs.fs.read("main", "/feat2.md")).text, "feat2 line\n"); + + const after = await layerfs.project.list_branches(); + assert.deepEqual(after.items.map((item) => item.name).sort(), ["combined", "feat2", "main", "merged", "work"]); + + // The daemon-only surface refuses Unsupported on the Windows route; on + // Linux the workspace route is the daemon's and is not exercised here. + if (process.platform === "win32") { + // The daemon-only surface refuses Unsupported at the native layer; a + // sandbox.create reaches it with valid arguments on every API shape. + process.env.LAYERFS_IMAGE = "fake-image"; + process.env.LAYERFS_VOLUME = "fake-volume"; + process.env.LAYERFS_DAEMON = "/fake/daemon"; + await assert.rejects( + layerfs.sandbox.create({ image: "fake-image" }), + { code: "Unsupported" }, + "the Windows route refuses the daemon surface", + ); + } + + await layerfs.close(); + console.log(`store_matrix: ok (${commits.items.length} commits, ${after.items.length} branches)`); +} finally { + retryRemove(work); + retryRemove(volume); +} diff --git a/api/agent/test/trace.mjs b/api/agent/test/trace.mjs index bd43e47f9e..23a307ea24 100644 --- a/api/agent/test/trace.mjs +++ b/api/agent/test/trace.mjs @@ -5,9 +5,11 @@ import assert from "node:assert/strict"; import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; const FAKE = fileURLToPath(new URL("./fixture/fake-native.cjs", import.meta.url)); +// import() needs a file:// URL; a bare Windows path is not one. +const FAKE_URL = pathToFileURL(FAKE).href; Object.assign(process.env, { LAYERFS_NATIVE: FAKE, LAYERFS_HOME: "/private/trace-home", LAYERFS_DOCKER: "/fake.sock", LAYERFS_IMAGE: "fake-image", LAYERFS_VOLUME: "fake-volume", LAYERFS_DAEMON: "/fake/daemon", @@ -85,7 +87,7 @@ const selected = process.argv[2]; try { if (selected === "on") { const reference = await script(await LayerFS.open("website")); - const native = (await import(FAKE)).default; + const native = (await import(FAKE_URL)).default; const before = native.calls.length; assert.ok(native.observed.every((scope) => scope === null), "untraced calls carry no scope"); process.env.LAYERFS_TRACE = file; @@ -249,7 +251,7 @@ try { } else if (selected === "store") { // Store calls open the client's Store in this process once, and close() // releases it; no Sandbox is started for them (#352 F-020). - const native = (await import(FAKE)).default; + const native = (await import(FAKE_URL)).default; const client = await LayerFS.open("website"); await client.project.list_branches(); await client.fs.read("main", "/README.md"); @@ -292,7 +294,7 @@ try { // A client in a container dials its daemonHost on every call that // reaches a Sandbox, backend calls included: the binding reopens or // starts the Sandbox with the call's deploy. - const native = (await import(FAKE)).default; + const native = (await import(FAKE_URL)).default; process.env.LAYERFS_DAEMON_HOST = "host.docker.internal"; await script(await LayerFS.open("website")); const named = native.calls.filter((_, index) => "sandbox" in native.args[index]); diff --git a/api/docs/agent.md b/api/docs/agent.md index 330fcab914..a1a8dec97a 100644 --- a/api/docs/agent.md +++ b/api/docs/agent.md @@ -98,14 +98,21 @@ or reinstalling over live state. Store calls (`fs`, `branch`, `project.list_branches`) run in this process against the installed Store file at `storePath` (else `LAYERFS_STORE_PATH`), -which the factory opened; no Sandbox is involved. That +which the factory opened; no Sandbox is involved. The project family's +Sandbox-related calls (`project.list_sandboxes`, `sandbox.*`) still need the +daemon and refuse `Unsupported` on the Windows-native route (on Linux they +are served in this process). That file is `global/store.sqlite` in the project's volume, so the process runs where -that volume is mounted (`/layerfs-store` in a Sandbox). This is Linux only, with -the Store on a local filesystem the daemons share (ext4, xfs, btrfs); anywhere -else opening fails `Unsupported`, and without a `storePath` it fails -`InvalidInput`. On macOS and Windows, any process that names its LayerFS +that volume is mounted (`/layerfs-store` in a Sandbox). On Linux the Store sits +on a local filesystem the daemons share (ext4, xfs, btrfs); on **Windows the +Store sits on a local NTFS volume this process opens directly** (the +Windows-native route, owner 2026-10-25); anywhere else opening fails +`Unsupported`, without a `storePath` it fails +`InvalidInput`, and a `storePath` naming no installed file fails +`NoStore`. On macOS, and on Windows when a worker is preferred, any process that names its LayerFS home's worker (`LAYERFS_WORKER_ENDPOINT`, or the endpoint `@layerfs/runtime` -hands over with `useWorker`) sends the calls there instead. The route is open to any integration (owner, +hands over with `useWorker`) sends the calls there instead; on Windows the +in-process native route is the default when no worker endpoint is set. The route is open to any integration (owner, 2026-10-24); LayerFS names none. A client keeps the binding it was made with: the in-process addon, or the worker if one was set when it was made, so a client made before `@layerfs/runtime` hands over its worker stays in-process and @@ -135,14 +142,14 @@ Keep each agent's operations on a Workspace sequential. | Environment | Meaning | | --- | --- | -| `LAYERFS_HOME` | Absolute private directory for discovery records (projects, Sandboxes) and Init's temporary sealed Store, deleted once its install is recorded. Created as 0700; existing directories must be owned by the current effective user with no group/other access. Records are 0600, not symlinks. | +| `LAYERFS_HOME` | Absolute private directory for discovery records (projects, Sandboxes) and Init's temporary sealed Store, deleted once its install is recorded. Its parent directory must already exist (creation is one level, like the store directory's parent); on Windows it carries an owner-only DACL and existing entries granting broader access refuse `Unauthorized`. Created as 0700 on unix; existing directories must be owned by the current effective user with no group/other access. Records are 0600, not symlinks. | | `LAYERFS_DOCKER` | Local Engine Unix socket; `/var/run/docker.sock` is the default. On the macOS worker route the worker and Init's container use the engine's socket inside its VM instead (the endpoint's `docker`). | | `LAYERFS_IMAGE` | Exact immutable runtime `sha256:...` digest a new Sandbox runs. A running Sandbox keeps the image it started with. | | `LAYERFS_VOLUME` | Existing named local VM volume shared by the project's daemons. | | `LAYERFS_DAEMON` | Host path to the built Linux daemon executable. | | `LAYERFS_NATIVE` | Optional path of the real addon; defaults to `engine/crates/layerfs-node/layerfs.node`. | | `LAYERFS_WORKER_ENDPOINT` | macOS and Windows: the endpoint file of the LayerFS home's worker, which then makes every binding call. Any integration may set it. | -| `LAYERFS_STORE_PATH` | The installed Store file this client's Store calls open: `global/store.sqlite` in the project's volume. On Linux this process opens it, so it must be on a local filesystem the daemons share, such as `/layerfs-store/global/store.sqlite` where the volume is mounted at `/layerfs-store`; otherwise Store calls fail `Unsupported`. On macOS and Windows the worker opens it, so it is the path inside the worker's Linux, `/var/lib/docker/volumes//_data/global/store.sqlite`. On any other platform Store calls fail `Unsupported`. Unset, they fail `InvalidInput` everywhere. | +| `LAYERFS_STORE_PATH` | The installed Store file this client's Store calls open: `global/store.sqlite` in the project's volume. On Linux this process opens it, so it must be on a local filesystem the daemons share, such as `/layerfs-store/global/store.sqlite` where the volume is mounted at `/layerfs-store`; otherwise Store calls fail `Unsupported`. On Windows the native route opens it in this process, so it is a Windows path on a local NTFS volume, such as `D:\layerfs\home\\volume\global\store.sqlite`; a Store on a share, a junction to one, or a non-NTFS volume fails `Unsupported`. On macOS (and Windows with a worker) the worker opens it, so it is the path inside the worker's Linux, `/var/lib/docker/volumes//_data/global/store.sqlite`. On any other platform Store calls fail `Unsupported`. Unset, they fail `InvalidInput` everywhere. | | `LAYERFS_CONTAINER_PREFIX` | Optional. Names every container this client creates `-` (default prefix `layerfs-`); another LayerFS home on the same Docker engine sets its own, so names never collide. Docker's name rule applies (an ASCII letter or digit, then letters, digits, `_`, `.`, `-`, at most 128). A name another container holds refuses the create before any effect. | | `LAYERFS_DAEMON_HOST` | Optional. The host this process dials for a daemon's port. Each daemon publishes its port on the Docker host's `127.0.0.1` only, and that stays so. A client inside a container does not share that loopback, so it names the host that reaches it, `host.docker.internal` on Docker Desktop. One resolution, its first address, one connect; nothing is tried again. Unset, the client dials `127.0.0.1`. | | `LAYERFS_TRACE` | Optional per-call trace file (see [the call trace](backend.md#the-call-trace)); unset means no trace. | @@ -307,6 +314,15 @@ identity/timestamps. Destinations are exact and must not exist. Same-path/subtre moves or copies are InvalidInput. Mkdir creates parents and an existing directory is a no-op. Edit requires exactly one match and streams large files. +Names must be spellable by JavaScript: a path with an unpaired surrogate +(which the content grammar keeps byte-exact on Windows) cannot cross the JS +string boundary — a write by such a name lands as an U+FFFD sibling file +rather than being refused, and listings render the original empty. A merge +or diff whose walk must build a path over such a name fails `Failed` with +the cause in the message, and `fs.rm`/`fs.cp` over a subtree holding one +refuses the same way. The +byte-carrying path is recorded follow-up work. + A file's or folder's permission bits are an integer `mode` such as `0o755`. A `write` (in `fs.write`'s options or an `fs.apply` change) or `mkdir` may carry one: a new file gets `mode`, else 0o644; an existing file keeps its mode unless @@ -405,14 +421,16 @@ makes exactly one SDK call through the SDK's public API, prints the result and closes the client. It adds no capability, check or retry of its own: the SDK checks the inputs and makes its one attempt, and the command reports what it answered. Settings come from the environment exactly as for the SDK, and the -same platform rules apply (Store calls run in this process on Linux, through -the LayerFS home's worker on macOS and Windows, and fail `Unsupported` anywhere else). +same platform rules apply (Store calls run in this process on Linux and, on the +native route, on Windows; through the LayerFS home's worker on macOS and on +Windows when one is set; and fail `Unsupported` anywhere else). ### Install and run The command is the `api/cli/` folder of a LayerFS checkout; it isn't published to npm. It needs Node 20 or newer and the SDK's native addon (`npm run -build:native` in `api`, or `LAYERFS_NATIVE`). +build:native` in `api`, or `LAYERFS_NATIVE`; the script builds on Linux, macOS +and Windows, placing the addon at `engine/crates/layerfs-node/layerfs.node`). ```bash npm --prefix "$LAYERFS_REPO/api" install diff --git a/api/docs/runtime.md b/api/docs/runtime.md index 6ce3e1e341..a28900014b 100644 --- a/api/docs/runtime.md +++ b/api/docs/runtime.md @@ -126,7 +126,12 @@ Each platform folder implements one interface (`endpoint`, `status`, - **Windows** runs the worker as a Node process in a WSL2 distribution (`server.js --stdin`); Init runs in the worker, which maps the folder with `wslpath`. This process owns the worker through its stdin: `close()` drains - and ends it. + and ends it. This is the worker route; it coexists with the **Windows-native + Store route** (owner, 2026-10-25), where the in-process addon opens a local + NTFS Store directly and no worker is involved. The native route is the + default when no worker endpoint is set (`LAYERFS_WORKER_ENDPOINT` or + `useWorker`): setting a worker sends the calls to the worker instead, and + the worker's paths are then inside its WSL2 Linux. The runtime starts a worker only on macOS and Windows. The worker route is open to any integration (owner, 2026-10-24): on those platforms the runtime diff --git a/api/scripts/build-native.mjs b/api/scripts/build-native.mjs index 3e591f31a9..2e09d2a447 100644 --- a/api/scripts/build-native.mjs +++ b/api/scripts/build-native.mjs @@ -7,7 +7,8 @@ import { fileURLToPath } from "node:url"; const root = fileURLToPath(new URL("../../", import.meta.url)); const suffix = { darwin: "dylib", linux: "so" }[process.platform]; -if (!suffix) throw new Error(`native addon builds require macOS or Linux, got ${process.platform}`); +const windows = process.platform === "win32"; +if (!suffix && !windows) throw new Error(`native addon builds require macOS, Linux or Windows, got ${process.platform}`); const flags = process.argv.slice(2); if (flags.some((flag) => flag !== "--debug")) throw new Error("usage: build-native.mjs [--debug]"); const profile = flags.includes("--debug") ? "debug" : "release"; @@ -15,14 +16,18 @@ const configured = process.env.CARGO_TARGET_DIR; const target = configured ? resolve(root, configured) : join(root, "target"); const outside = relative(root, target); if (outside.startsWith("..") || isAbsolute(outside)) throw new Error("Cargo target must stay in this worktree"); -if (process.env.CARGO_BUILD_TARGET) throw new Error("build on the actual macOS/Linux host without CARGO_BUILD_TARGET"); +if (process.env.CARGO_BUILD_TARGET) throw new Error("build on the actual host without CARGO_BUILD_TARGET"); const result = spawnSync("cargo", [ "+1.85.1", "build", "--manifest-path", "Cargo.toml", "--offline", "--locked", ...(profile === "release" ? ["--release"] : []), "-p", "layerfs-node", ], { cwd: root, stdio: "inherit", env: { ...process.env, CARGO_TARGET_DIR: target } }); if (result.error) throw result.error; if (result.status !== 0) process.exit(result.status ?? 1); -const artifact = join(target, profile, `liblayerfs_node.${suffix}`); +// Windows names its cdylib layerfs_node.dll; the other hosts use the +// lib-prefixed spelling. Both land at the addon path the SDK loads. +const artifact = windows + ? join(target, profile, "layerfs_node.dll") + : join(target, profile, `liblayerfs_node.${suffix}`); const output = process.env.LAYERFS_NATIVE_OUTPUT ? resolve(root, process.env.LAYERFS_NATIVE_OUTPUT) : fileURLToPath(new URL("../../engine/crates/layerfs-node/layerfs.node", import.meta.url)); diff --git a/core/crates/layerfs-content/src/compare/changes.rs b/core/crates/layerfs-content/src/compare/changes.rs index 5844f4ec23..6da70ec351 100644 --- a/core/crates/layerfs-content/src/compare/changes.rs +++ b/core/crates/layerfs-content/src/compare/changes.rs @@ -23,7 +23,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; use super::sorted::DifferenceWork; -use super::tree::{binding_changes, inode_differences, join, EntryFacts, TreeTable}; +use super::tree::{binding_changes, inode_differences, join_exact, EntryFacts, TreeTable}; use crate::error::{ContentError, ContentResult}; use crate::filesystem::directory::read::{list_after, lookup, DirectoryReadWork}; use crate::filesystem::path::PathName; @@ -328,7 +328,7 @@ impl<'a> ChangeWalk<'a> { let Some((name, [left, right])) = frame.ready.pop_front() else { continue; }; - let path = join(&frame.path, &name); + let path = join_exact(&frame.path, &name)?; self.stack.push(frame); return Ok(Some((path, left, right))); } diff --git a/core/crates/layerfs-content/src/compare/named.rs b/core/crates/layerfs-content/src/compare/named.rs index 187dafaaa7..10d692345d 100644 --- a/core/crates/layerfs-content/src/compare/named.rs +++ b/core/crates/layerfs-content/src/compare/named.rs @@ -18,7 +18,7 @@ use std::rc::Rc; use crate::compare::tree::TreeTable; use crate::compare::BindingHints; -use crate::error::ContentResult; +use crate::error::{ContentError, ContentResult}; use crate::filesystem::path::PathName; use crate::object::{AuthenticatedObjects, ObjectId}; use layerfs_telemetry::timer::TimingScope; @@ -186,11 +186,20 @@ fn name_all( let path = if first.is_empty() { "/".to_owned() } else { - first.iter().fold(String::new(), |mut path, name| { + // A name outside UTF-8 (the WTF-8 grammar) cannot ride a String + // path: rendering it empty would collapse distinct entries and + // every decision built on the path would be wrong. Refuse until + // the byte-carrying merge/compare path lands (the grammar is + // owner-ratified; the carriage is follow-up work). + let mut path = String::new(); + for name in first { + let text = std::str::from_utf8(name.as_bytes()).map_err(|_| { + ContentError::InvalidRecord("a name outside UTF-8 has no String path") + })?; path.push('/'); - path.push_str(name.as_str()); - path - }) + path.push_str(text); + } + path }; named.insert(*serial, path); } diff --git a/core/crates/layerfs-content/src/compare/tree.rs b/core/crates/layerfs-content/src/compare/tree.rs index 268ca406ba..d18f330c7d 100644 --- a/core/crates/layerfs-content/src/compare/tree.rs +++ b/core/crates/layerfs-content/src/compare/tree.rs @@ -263,7 +263,9 @@ pub fn list_directory( } } -/// Joins a parent path and one name; the root is "/". +/// Joins a parent path and one name; the root is "/". This is the +/// **display** spelling: a name outside UTF-8 renders empty (the SDK +/// listing contract), so its result must never key a decision. pub fn join(parent: &str, name: &PathName) -> String { if parent == "/" { format!("/{}", name.as_str()) @@ -272,6 +274,22 @@ pub fn join(parent: &str, name: &PathName) -> String { } } +/// Joins for a **decision**: a name outside UTF-8 cannot ride a String +/// path — rendering it empty would collapse distinct entries under one +/// key and silently drop the second change of a directory (round-6 D1 +/// F2), so this fence refuses exactly as `name_all` does until the +/// byte-carrying merge/compare path lands. Every keying, comparison or +/// merge site uses this twin, never `join`. +pub fn join_exact(parent: &str, name: &PathName) -> ContentResult { + let text = std::str::from_utf8(name.as_bytes()) + .map_err(|_| ContentError::InvalidRecord("a name outside UTF-8 has no String path"))?; + Ok(if parent == "/" { + format!("/{text}") + } else { + format!("{parent}/{text}") + }) +} + /// Paths of the wanted serials, breadth first from the root, first path only. /// /// The walk ends when every wanted serial is named or `limit` directories have @@ -298,7 +316,7 @@ pub fn locate( }; let mut children = Vec::new(); list_directory(reader, value.content_root, |name, serial| { - children.push((join(&path, name), serial)); + children.push((join_exact(&path, name)?, serial)); Ok(true) })?; let serials = children @@ -333,7 +351,7 @@ pub fn subtree( }; let mut children = Vec::new(); list_directory(reader, value.content_root, |name, child| { - children.push((join(&path, name), child)); + children.push((join_exact(&path, name)?, child)); Ok(true) })?; let serials = children @@ -455,7 +473,7 @@ pub fn root_changes( Some(after.content_root), &mut out.work, |name, was, now| { - bindings.push((join(path, name), was, now)); + bindings.push((join_exact(path, name)?, was, now)); Ok(()) }, )?; diff --git a/core/crates/layerfs-content/src/filesystem/path.rs b/core/crates/layerfs-content/src/filesystem/path.rs index dad84fa64b..bfb2a20d43 100644 --- a/core/crates/layerfs-content/src/filesystem/path.rs +++ b/core/crates/layerfs-content/src/filesystem/path.rs @@ -1,10 +1,13 @@ //! Canonical names and paths: byte grammar, ordering and declared bounds. //! -//! A name is one nonempty, at-most-255-byte UTF-8 component without NUL, `/` or -//! `\`, and never `.` or `..`. A path is either the empty root or a sequence of -//! such components joined by single separators, at most 4,096 bytes and 256 -//! components. Ordering is byte order, which is what the sorted tree engine -//! requires and what the reference uses. +//! A name is one nonempty, at-most-255-byte component without NUL, `/` or +//! `\`, and never `.` or `..`; its bytes are well-formed UTF-8 or, for a +//! Windows host name holding an unpaired surrogate, well-formed WTF-8 (the +//! same encoding with surrogates as their three-byte forms). A path is +//! either the empty root or a sequence of such components joined by single +//! separators, at most 4,096 bytes and 256 components. Ordering is byte +//! order, which is what the sorted tree engine requires and what the +//! reference uses. use std::cmp::Ordering; @@ -36,7 +39,8 @@ impl PathName { &self.bytes } - /// The name as text; construction proved it is UTF-8. + /// The name as text; a WTF-8 name outside UTF-8 answers the empty + /// string rather than a lossy spelling — `as_bytes` carries it whole. pub fn as_str(&self) -> &str { std::str::from_utf8(&self.bytes).map_or("", |value| value) } @@ -97,7 +101,8 @@ impl LogicalPath { &self.bytes } - /// The path as text; construction proved it is UTF-8. + /// The path as text; a WTF-8 path outside UTF-8 answers the empty + /// string rather than a lossy spelling — `as_bytes` carries it whole. pub fn as_str(&self) -> &str { std::str::from_utf8(&self.bytes).map_or("", |value| value) } @@ -150,7 +155,7 @@ fn validate_path(bytes: &[u8]) -> ContentResult<()> { if bytes.is_empty() { return Ok(()); } - if std::str::from_utf8(bytes).is_err() { + if !is_wtf8(bytes) { return Err(ContentError::InvalidUtf8); } let mut components = 0_usize; @@ -173,7 +178,7 @@ fn validate_name(bytes: &[u8]) -> ContentResult<()> { if bytes.len() > MAXIMUM_NAME_BYTES { return Err(ContentError::PathLimitExceeded); } - if std::str::from_utf8(bytes).is_err() { + if !is_wtf8(bytes) { return Err(ContentError::InvalidUtf8); } if bytes == b"." @@ -186,3 +191,59 @@ fn validate_name(bytes: &[u8]) -> ContentResult<()> { } Ok(()) } + +/// True for well-formed UTF-8, and for well-formed WTF-8 — the same +/// grammar with an unpaired surrogate as its three-byte form, which a +/// Windows host name can hold. A pair spelled as two surrogate sequences +/// is not canonical WTF-8 and is refused, so one name has one byte +/// spelling on every platform. Overlong forms, 5-byte leads, truncated +/// tails and out-of-range values stay refused, so every strict UTF-8 +/// name keeps its exact acceptance. +fn is_wtf8(bytes: &[u8]) -> bool { + let mut index = 0; + let mut pending_high_surrogate = false; + while index < bytes.len() { + let lead = bytes[index]; + if lead < 0x80 { + index += 1; + pending_high_surrogate = false; + continue; + } + let (width, minimum, maximum) = match lead { + 0xC2..=0xDF => (2_usize, 0x80_u32, 0x7FF), + 0xE0..=0xEF => (3, 0x800, 0xFFFF), + 0xF0..=0xF4 => (4, 0x10000, 0x10FFFF), + _ => return false, + }; + if index + width > bytes.len() { + return false; + } + let mut value = match width { + 2 => u32::from(lead & 0x1F), + 3 => u32::from(lead & 0x0F), + _ => u32::from(lead & 0x07), + }; + for continuation in &bytes[index + 1..index + width] { + if continuation & 0b1100_0000 != 0b1000_0000 { + return false; + } + value = (value << 6) | u32::from(continuation & 0b0011_1111); + } + if value < minimum || value > maximum { + return false; + } + // A high surrogate immediately followed by its low half as a second + // sequence is a pair spelled apart — not canonical WTF-8. + if (0xD800..=0xDBFF).contains(&value) { + pending_high_surrogate = true; + } else if (0xDC00..=0xDFFF).contains(&value) { + if pending_high_surrogate { + return false; + } + } else { + pending_high_surrogate = false; + } + index += width; + } + true +} diff --git a/core/crates/layerfs-content/src/merge/names.rs b/core/crates/layerfs-content/src/merge/names.rs index b13974c999..e85c4656ad 100644 --- a/core/crates/layerfs-content/src/merge/names.rs +++ b/core/crates/layerfs-content/src/merge/names.rs @@ -20,7 +20,7 @@ use super::serial::{is_directory, listing, same, Ctx}; use super::spill::{ single, ABSORB, CANCEL_MOVE, CANCEL_REMOVE, DONE, DROP, EXTRA, MOVED, NAMED, REVIVE_ALL, TARGET, }; -use crate::compare::tree::{binding_changes, join, EntryFacts}; +use crate::compare::tree::{binding_changes, join_exact, EntryFacts}; use crate::compare::DifferenceWork; use crate::error::{ContentError, ContentResult}; use crate::filesystem::directory::read::lookup; @@ -306,7 +306,7 @@ impl Ctx<'_, '_> { let base = Values::entry(&values.b, b); let ours = Values::entry(&values.f, f); let theirs = Values::entry(&values.i, i); - let path = join(&self.slot_path(listing)?, name); + let path = join_exact(&self.slot_path(listing)?, name)?; let key = named_key(listing.target, name); if let (Some(fe), Some(ie)) = (ours, theirs) { let absorb = is_directory(Some(fe.value)) diff --git a/core/crates/layerfs-content/src/merge/places.rs b/core/crates/layerfs-content/src/merge/places.rs index bada72ceab..3c8dcf2c23 100644 --- a/core/crates/layerfs-content/src/merge/places.rs +++ b/core/crates/layerfs-content/src/merge/places.rs @@ -22,7 +22,7 @@ use super::spill::{ REVIVE_ALL, SURVIVE, TARGET, }; use super::{Conflict, ConflictKind}; -use crate::compare::tree::{join, list_directory, locate}; +use crate::compare::tree::{join_exact, list_directory, locate}; use crate::error::{ContentError, ContentResult}; use crate::filesystem::path::PathName; use crate::object::inode_leaf::InodeValue; @@ -308,7 +308,7 @@ impl Ctx<'_, '_> { continue; } let parent_path = self.path_anywhere(parent)?; - let target = join(&parent_path, &PathName::new(&name)?); + let target = join_exact(&parent_path, &PathName::new(&name)?)?; let base_path = self.path(BASE, serial)?; let into_path = self.path(self.into, serial)?; let cancelled = if into_path != base_path { diff --git a/core/crates/layerfs-content/tests/compare_join_exact.rs b/core/crates/layerfs-content/tests/compare_join_exact.rs new file mode 100644 index 0000000000..96d802ddef --- /dev/null +++ b/core/crates/layerfs-content/tests/compare_join_exact.rs @@ -0,0 +1,32 @@ +//! The decision-spelling twin of `join` (round-6 D1 F2): a name outside +//! UTF-8 refuses `InvalidRecord` instead of collapsing to an empty +//! segment, while the display `join` keeps the listing contract — and +//! the collapsed display spelling is exactly what must never key a +//! decision. +use layerfs_content::compare::tree::{join, join_exact}; +use layerfs_content::{ContentError, PathName}; + +#[test] +fn utf8_names_join_identically_on_both_twins() { + let name = PathName::from_bytes("中文🚀.txt".as_bytes()).unwrap(); + assert_eq!(join_exact("/dir", &name).unwrap(), join("/dir", &name)); + assert_eq!(join_exact("/", &name).unwrap(), join("/", &name)); +} + +#[test] +fn a_name_outside_utf8_refuses_the_decision_spelling() { + let lone = PathName::from_bytes(&[0xED, 0xA0, 0x80][..]).unwrap(); // unpaired high + // Display keeps the contract: the segment renders empty. + assert_eq!(join("/dir", &lone), "/dir/"); + let error = join_exact("/dir", &lone).unwrap_err(); + assert!( + matches!(error, ContentError::InvalidRecord(_)), + "the fence refuses, got {error:?}" + ); + // The harm the fence closes: two distinct names collapse to one + // display key, so the display spelling can never decide anything. + let other = PathName::from_bytes(&[0xED, 0xB2, 0x80][..]).unwrap(); // unpaired low + assert_ne!(lone.as_bytes(), other.as_bytes()); + assert_eq!(join("/dir", &lone), join("/dir", &other)); + assert!(join_exact("/dir", &other).is_err()); +} diff --git a/core/crates/layerfs-content/tests/filesystem_name_grammar.rs b/core/crates/layerfs-content/tests/filesystem_name_grammar.rs new file mode 100644 index 0000000000..88e30bfbbb --- /dev/null +++ b/core/crates/layerfs-content/tests/filesystem_name_grammar.rs @@ -0,0 +1,45 @@ +//! The name grammar accepts well-formed UTF-8 and the canonical WTF-8 +//! spelling of an unpaired surrogate (a Windows host name); a pair spelled +//! as two surrogate sequences and every broken encoding are refused, so +//! one name has one byte spelling on every platform. +use layerfs_content::PathName; + +#[test] +fn utf8_names_and_unpaired_surrogates_are_accepted() { + for bytes in [ + "ascii.txt".as_bytes(), + "中文".as_bytes(), + // The supplementary plane keeps its UTF-8 scalar spelling. + "🚀-🧬".as_bytes(), + // An unpaired high (U+D800) and an unpaired low (U+DC80) surrogate, + // each in its three-byte WTF-8 form. + &[0xED, 0xA0, 0x80][..], + &[0xED, 0xB2, 0x80][..], + // An unpaired surrogate beside ordinary text. + &[b'A', 0xED, 0xA0, 0x80, b'B'][..], + // Two unpaired high surrogates in a row are each unpaired. + &[0xED, 0xA0, 0x80, 0xED, 0xA0, 0xBF][..], + ] { + assert!(PathName::from_bytes(bytes).is_ok(), "{bytes:02x?}"); + } +} + +#[test] +fn split_pairs_and_broken_encodings_are_refused() { + for bytes in [ + // 🚀 (U+1F680) spelled as D83D DE80 in two surrogate sequences. + &[0xED, 0xA0, 0xBD, 0xED, 0xBA, 0x80][..], + // Overlong encodings of '/' and 'A'. + &[0xE0, 0x80, 0xAF][..], + &[0xC0, 0xAF][..], + // Beyond U+10FFFF and a bare continuation byte. + &[0xF5, 0x80, 0x80, 0x80][..], + &[0x80][..], + // A truncated four-byte tail. + &[0xF0, 0x9F][..], + // A malformed continuation. + &[0xE4, 0xB8, 0x00][..], + ] { + assert!(PathName::from_bytes(bytes).is_err(), "{bytes:02x?}"); + } +} diff --git a/core/docs/architecture/83-store-concurrency-model.md b/core/docs/architecture/83-store-concurrency-model.md index b2e6e8cdd0..14d9fb2db6 100644 --- a/core/docs/architecture/83-store-concurrency-model.md +++ b/core/docs/architecture/83-store-concurrency-model.md @@ -31,11 +31,21 @@ | Layer | Scope | Why it exists | | --- | --- | --- | | SQLite file lock (WAL) | the Store file | SQLite allows one write transaction per database; readers proceed alongside | -| Directory lock (Linux) | all writer processes of one Store | Writers in different processes (each Sandbox's daemon, the SDK process) take turns instead of meeting `SQLITE_BUSY` | +| Directory lock (Linux) / named kernel mutex (Windows) | all writer processes of one Store | Writers in different processes (each Sandbox's daemon, the SDK process) take turns instead of meeting `SQLITE_BUSY`; on Windows the mutex is named per Store directory and an abandoned mutex is taken by the next waiter | | Writer admission queue | threads of one process | Threads take turns on the process's one writer connection, first come first served, before touching SQLite | | Publication lane | one Commit's publish | Head check, merge and publish are one atomic step, so two Commits never both publish on the same old head | +| Open/teardown holds | one Store's connect, writable setup and close | The connect-time statement block (profile pragmas, format reads — where a cleaned WAL `-shm` is rebuilt under an exclusive recovery hold), the writable setup block (held publication lane, nested turns take no second lock) and a coordinated close (`StoreLane`/`Handles::shutdown`; the service's `close` stops overlays first, then drops the sessions under the lane) run under the same cross-process lane: SQLite's close-time checkpoint is writer work like any other and must never run beside another opener's first attach (no-retry rule) | | Workspace lock | one Workspace's overlay database | Separate concern: guards that Workspace's own overlay connection (root AGENTS.md) | +The lane's Windows name is SHA-256 of the resolved directory's encoded +bytes truncated to 64 bits — a version-stable digest, never +`DefaultHasher`, whose algorithm may change between Rust releases and +silently split one directory onto two lanes across builds (K3 review F1). +The writer lane lives in the `Global\` namespace, so one lane covers every +login session (moved from `Local\` on the Windows-native route, round 3 — +DIRECT_STORE.md §6); the discovery mutexes stay `Local\` and only coordinate +one session's discovery. + The queue and the directory lock replace failure, not add serialization: the repository forbids automatic retries and busy handlers (one attempt per operation, exact failure), so a writer waits its turn before its single SQL diff --git a/core/docs/issues/352/DIRECT_STORE.md b/core/docs/issues/352/DIRECT_STORE.md index 765b326d0b..bef9a37315 100644 --- a/core/docs/issues/352/DIRECT_STORE.md +++ b/core/docs/issues/352/DIRECT_STORE.md @@ -11,6 +11,8 @@ > **One route, Linux only (owner, 2026-10-18):** Store calls run in-process > only; the daemon stops serving them, and an unsupported setup fails > `Unsupported`. There is no daemon route to choose or fall back to. +> **Windows-native route (owner, 2026-10-25):** the Store engine and this +> in-process Store face also run natively on Windows; see §6. ## 1. Architecture @@ -88,3 +90,159 @@ Disposable/WAL/OFF only; page size 4 KiB; one attempt per operation; FUSE and Workspaces stay in the daemon; the daemon keeps opening the Store directly; no `layerfs-server`, no host/server adapter in the data path (the SDK is a direct opener, not a proxy). + +## 6. Windows-native route (owner direction 2026-10-25) + +The one-kernel safety condition is met natively on Windows: every opener of a +Windows Store shares one Windows kernel and one local NTFS volume. No WSL2 and +no daemon are involved. Implemented on branch `windows-native-store`: + +- The cross-process writer lane is a named kernel mutex + (`Global\layerfs-writer-lane-` — one lane per directory across + login sessions); an abandoned mutex is taken by the + next waiter, so a dead writer does not hold the lane. +- The kernel guard allows local NTFS volumes only: it opens the Store + directory's own handle, reads its final path (a spelling that lands on + `\\?\UNC\` — a share behind a junction or mapped drive — is refused) and + the filesystem name with `GetVolumeInformationByHandleW`; anything else + fails `Unsupported` before the Store opens. +- SQLite is bundled (rusqlite) on Windows targets; Disposable/WAL/OFF is the + only profile, unchanged. +- Acquisition identity is the real volume serial and file index + (`GetFileInformationByHandle`), in audited FFI modules listed in + engine/AGENTS.md. +- The SDK addon serves fs, branch, project and operator calls in-process + exactly as on Linux (on Windows, `project.list_sandboxes` and the sandbox + family also refuse `Unsupported` — the daemon owns them). Workspace, exec + and daemon calls refuse `Unsupported` + on Windows: FUSE, the daemon and Workspaces remain Linux-only, and the dsh + macOS and WSL2 exceptions are unchanged. + +Recorded limitations of this route: the writer lane lives in the +`Global\` namespace, so one login session's scheduled task and another +session's interactive process share one lane (round-3 A2 F4 closed the +`Local\` per-session split recorded earlier; a mutant needs no +create-global privilege); the discovery mutexes remain `Local\` — they +coordinate one session's discovery only and are no integrity boundary. +The named mutexes can in principle be squatted by another process of the +machine before this product creates them (the directory ACL is not the +capability there, as Linux flock's is); the discovery turn waits bounded +(60s `Busy`) and the writer lane's wait is the readiness wait of the lock +model, as Linux's blocking flock is. Windows Init carries names as +lossless WTF-8 end to end (an unpaired surrogate included, byte-exact +round trip through the codec and the audited identity probes) and +synthesizes the mode from the read-only attribute, and has no hard-link +membership. **The WTF-8 name grammar is owner-ratified 2026-10-31 +("可以合并") as the long-term content grammar**; until the byte-carrying +merge/compare path lands as follow-up, a merge or compare that must build +a String path over a name outside UTF-8 is refused (`InvalidRecord`) at +every entrance — the recorded-name path and the walked path alike +(round-6 D1 F2 fenced `join`, whose collapsed key silently dropped the +second changed entry of one directory) +rather than decided on a collapsed path, and SDK listings render such +names empty; a JS write naming one lands as an U+FFFD sibling file +rather than being refused (the JS string boundary cannot carry an +unpaired surrogate — the same byte-carrying follow-up). The +backing-inside-source check walks the audited +volume-serial + file-index identity of every ancestor (a subst drive or +junction into the source is recognised and refused, as on Linux), and +overlay allocation is the entry's real on-disk footprint. Last-owner +overlay-file deletion applies the same audited identity on Windows as on +Linux (volume serial + file index, sole link, checked on both the path's +entry and the retained handle), opening the path itself, never a +substituted reparse point. Homes, discovery records, the install volume +directory and Store files carry owner-only protected DACLs (the audited +ACL module): what this route creates is restricted to its owner, and an +existing entry whose DACL grants Everyone, Users or Authenticated Users +is refused — the parity of unix's 0700/0600 discipline; the installed +Store file answers its own entry check at every open, its directory is +restricted when the route creates it (a caller-pre-created directory is +accepted as the caller shaped it — hence the sidecar note below), and a +link planted at the file's spelling is refused, never followed. A home that +lost its owner (restored from backup, moved between machines) answers +`Unauthorized`; an administrator recovers it from an **elevated** console +(runbook verified end to end, round 6): first +`takeown /f /r /d y` — and the same recursively on the volume +tree, when the volume sits outside the home; taking ownership rewrites +the files' OWNER RIGHTS ace into an inherit-only form, so every file +must be re-granted explicitly — then +`icacls /inheritance:r /grant:r ':(OI)(CI)F'` and the same +grant on the volume tree, and finally a plain +`icacls /grant:r ':F'` on the installed Store file and on +`//project.record` (an `(OI)(CI)` grant gives a *file* +nothing) — before the route accepts it again. + +Operational notes from the 2026-10-31 production-readiness round: the +audited Win32 calls spell paths verbatim (`\\?\`), so trees deeper than +MAX_PATH initialize — but the host process's own plain-spelled +operations still need the host's long-path relaxation (a +`longPathAware` manifest, or an OS build that no longer requires it — +round-3 A1 measured plain 308-character paths working on one such +host); keep sources and homes within ~260 characters on hosts whose +policy is not confirmed. +Windows lacks the 64 MiB disk-headroom pre-admission macOS and Linux run +(a filling disk surfaces mid-transaction as the typed `Capacity` +refusal instead); SQLite's `-wal`/`-shm` sidecars inherit the +directory's DACL, and every open tightens the ones it finds back to +owner-only (a crash's leftovers wait for the next open); a writer-lane +squatter parks every Store call for as long as it holds the mutex (the +readiness wait has no bound, exactly as Linux's blocking flock) — round +5 added two neighbours of that local-adversary availability class: a +pre-planted lane object carrying a foreign DACL makes the next open +fail (currently typed `Malformed`, recorded for a later typing pass), +and any local user can enumerate lane names in +`\BaseNamedObjects` (a digest of the directory that reveals write +activity, not the path); a caller-pre-created wide directory also lets +another user pre-place `-wal`/`-shm` the per-open tightening cannot +re-own (create the volume owner-only, as the route's own layout does); +the lane's name is the canonicalized spelling, so a subst-drive spelling +that does not canonicalize to the real path can hold a second lane for +one directory — bounded by SQLite's own WAL write lock, which refuses +the overlapping writer rather than corrupting (round-6 D1 F3); +a killed process leaves one +overlay directory under `%TEMP%\layerfs-store-*` until the OS cleans +it (schema-only and never blocking the next open — accumulation is the +machine's TEMP policy); the overlay crate's two Win32 path probes +(entry identity, free space) take raw wide spellings of TEMP-local +paths — the product-source boundary keeps layerfs-overlay free of +first-party dependencies, so the shared verbatim helper is not imported +there and a TEMP tree beyond MAX_PATH fails those probes cleanly; and +a `fs.rm`/`fs.cp` over a subtree holding a name outside UTF-8 +refuses with a message naming that entry (the SDK cannot spell such a +name until the byte-carrying path lands — the same follow-up as the +merge/compare fence above, which surfaces at the SDK as code `Failed` +with the cause in the message, not as the content-layer error name). +Upstream-family records pending an owner decision (round 7): the Linux +FUSE invalidation keys `Stale::Entry` on the display spelling, so a +non-UTF-8 name rebound by a merge misses its kernel dentry invalidation +(untouched by this branch); the 256-component path limit is enforced on +the workspace/rename route but not the SDK fs/init routes (live-accepted +at 257-301 components); and deep single-chain operations grow +near-quadratically (a d=250 `mkdir` measures ~1.8 s) — the engine guide's +reject-quadratic rule makes that a decision, not a footnote. + +**Obtaining the addon**: `npm run build:native` in `api/` builds on +Linux, macOS and Windows (cargo `+1.85.1 --release --offline --locked -p +layerfs-node`, artifact staged to `engine/crates/layerfs-node/layerfs.node`; +Node ≥ 20 is the runtime floor); `LAYERFS_NATIVE` names another copy. + +When the owner merges this branch, the observation period runs by a +fixed checklist (each step reads a mechanism that already exists, none +needs a new decision): + +1. **Merge lands** — the observation job records `merged` with the merge + commit; the tech-tree gate `pr_merged_and_stable` flips on that fact. +2. **Stability window** — the owner bounds it (the ruling names no + length); its evidence is the absence of new findings on this PR plus + the standing suites staying green on `main`. +3. **WinFsp independent period starts contract-first** (SSO pattern): + the mount face's contracts are written and reviewed before any + implementation, under the `winfsp-independent-period` tree node whose + precondition is exactly step 1 plus step 2. + +Proof receipts: the cross-process lane verifier +(`layerfs-persistence` `tests/windows_store_native.rs`, two re-exec'd child +processes, bounded waits); `cargo check --locked` of the five touched crates +on Windows and in the `rust:1.85.1-bookworm` container; the lifecycle smoke +(init → fork → one-Commit write → read back → list_commits → close) with no +WSL2 and no daemon. diff --git a/docs/general/cluster_one_handbook.md b/docs/general/cluster_one_handbook.md index c9849ba7d5..0713933c6e 100644 --- a/docs/general/cluster_one_handbook.md +++ b/docs/general/cluster_one_handbook.md @@ -57,7 +57,7 @@ The active Core workspace contains six crates: | `layerfs-content` | Canonical identity, file construction/editing, filesystem trees, logical reads | Turn bytes/edits/namespace rows into canonical objects; read logical data | | `layerfs-storage` | Reuse, physical encoding, delta selection, packing, bounded Save and Reader | Store canonical objects and acquire authenticated canonical bytes | | `layerfs-history` | Typed history records and `HistoryCatalog` semantic operations | Inode reservation, genesis, branches, Commits, Layers | -| `layerfs-persistence` | Shared SQLite provider for storage and history on macOS/Linux | Create/open the Store and execute provider-owned transactions | +| `layerfs-persistence` | Shared SQLite provider for storage and history on macOS/Linux/Windows (Windows: local NTFS volumes; owner direction 2026-10-25) | Create/open the Store and execute provider-owned transactions | | `layerfs-project` | Native namespace Init orchestration | Import a regular directory and publish its genesis LayerStack | | `layerfs-telemetry` | Explicit timing scopes and reports | Observe the same production operations with timing enabled/disabled | diff --git a/engine/AGENTS.md b/engine/AGENTS.md index 058b9f669e..e4129d981e 100644 --- a/engine/AGENTS.md +++ b/engine/AGENTS.md @@ -593,3 +593,27 @@ No other Persistence module may contain unsafe Rust; the guard and its tests enforce the exact path. This does not authorize third-party patches, dependency changes, manual sidecar deletion or other FFI. See the [decision and diagnostic](../core/docs/issues/307/SEAL-PERSIST-WAL-DECISION-20261007.md). + +### Audited Windows FFI modules + +Owner direction 2026-10-25 (route 2, Windows-native Store; root AGENTS.md) +adds audited Windows FFI beside the macOS seal wrapper. The guard +(`tools/check_product_boundary.py`) and its tests enforce the exact paths: + +- Persistence keeps `deny(unsafe_code)` with two allowed modules: + `src/backend/sqlite/file_control.rs` (the macOS seal wrapper, unchanged) + and `src/backend/sqlite/windows_lane.rs` (the named kernel mutex writer + lane and `GetVolumeInformationByHandleW` volume filesystem). +- layerfs-project moves forbid→deny with one allowed module, + `src/import/windows_identity.rs` (`GetFileInformationByHandle`: volume + serial, file index, creation time). +- layerfs-overlay moves forbid→deny with one allowed module, + `src/database/windows_identity.rs` (handle/path identity, + `GetFileInformationByHandleEx` allocation, `GetDiskFreeSpaceExW`). +- layerfs-node declares deny with two allowed discovery modules: + `src/discovery.rs` (unix uid/flock, upstream's) and + `src/discovery_windows.rs` (the named kernel mutex discovery turn). +- layerfs-store-service moves forbid→deny with one allowed module, + `src/windows_acl.rs` (owner-only DACLs for homes, discovery records and + Stores; `SetEntriesInAclW`/`SetNamedSecurityInfoW`/`GetNamedSecurityInfoW`). +- Storage's `src/encoding/codec.rs` authorization is unchanged. diff --git a/engine/crates/layerfs-daemon/tests/support/held_writer.rs b/engine/crates/layerfs-daemon/tests/support/held_writer.rs index b7099eeb16..482efbeabc 100644 --- a/engine/crates/layerfs-daemon/tests/support/held_writer.rs +++ b/engine/crates/layerfs-daemon/tests/support/held_writer.rs @@ -9,7 +9,10 @@ use std::{ pub struct HeldWriter(Child); impl HeldWriter { pub fn acquire(database: &std::path::Path) -> Self { - let script = "import sqlite3,sys,select\nc=sqlite3.connect(sys.argv[1],timeout=0)\nc.execute('BEGIN IMMEDIATE')\nprint('held',flush=True)\nselect.select([sys.stdin],[],[],5)\nc.rollback()\nc.close()\n"; + // The writer holds BEGIN IMMEDIATE, signals readiness and stands by + // for one line. select() is not portable to a Windows pipe stdin; + // a blocking readline is, and Drop's kill bounds a lost releaser. + let script = "import sqlite3,sys\nc=sqlite3.connect(sys.argv[1],timeout=0)\nc.execute('BEGIN IMMEDIATE')\nprint('held',flush=True)\nsys.stdin.readline()\nc.rollback()\nc.close()\n"; let child = Command::new("python3") .arg("-c") .arg(script) @@ -26,10 +29,10 @@ impl HeldWriter { let result = output.read_line(&mut line).map(|_| line); let _ = tx.send(result); }); - assert_eq!( - rx.recv_timeout(Duration::from_secs(3)).unwrap().unwrap(), - "held\n" - ); + // The external writer's readiness line, whatever end-of-line the + // host's python prints. + let line = rx.recv_timeout(Duration::from_secs(3)).unwrap().unwrap(); + assert_eq!(line.trim_end(), "held"); reader.join().unwrap(); held } diff --git a/engine/crates/layerfs-node/Cargo.toml b/engine/crates/layerfs-node/Cargo.toml index 44fca52204..557c287ef9 100644 --- a/engine/crates/layerfs-node/Cargo.toml +++ b/engine/crates/layerfs-node/Cargo.toml @@ -27,3 +27,9 @@ napi-derive = "=2.16.13" [build-dependencies] napi-build = "=2.1.6" + +[target.'cfg(target_os = "windows")'.dependencies] +# The discovery guard's mutex-name digest (K3 F1), the locked sha2. +sha2 = { version = "=0.11.0", default-features = false } +# The Windows discovery guard's named kernel mutex; already in the locked graph. +windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_System_Threading"] } diff --git a/engine/crates/layerfs-node/examples/restore_record.rs b/engine/crates/layerfs-node/examples/restore_record.rs index 540f12084d..8c7dec1545 100644 --- a/engine/crates/layerfs-node/examples/restore_record.rs +++ b/engine/crates/layerfs-node/examples/restore_record.rs @@ -13,6 +13,7 @@ use layerfs_bridge::control::{Answer, Reply, SdkValue}; use layerfs_bridge::provision::{ProviderKind, StoreManifest, StoreProfile}; use std::io::Write; +#[cfg(unix)] use std::os::unix::fs::OpenOptionsExt; fn hex(text: &str) -> [u8; N] { @@ -23,6 +24,7 @@ fn hex(text: &str) -> [u8; N] { bytes.try_into().expect("length") } +#[cfg(unix)] fn main() { let args: Vec = std::env::args().skip(1).collect(); let [home, project, binding, stack, branch, root, bytes, host, daemon] = &args[..] else { @@ -78,3 +80,10 @@ fn main() { file.write_all(&record).expect("write project.record"); println!("{}", path.display()); } + +// The record format this writes (0o600 mode, unix paths) belongs to the +// Linux restore runbook; Windows discovery saves its own records. +#[cfg(not(unix))] +fn main() { + eprintln!("restore_record is the Linux restore runbook; not run here"); +} diff --git a/engine/crates/layerfs-node/src/direct.rs b/engine/crates/layerfs-node/src/direct.rs index 0b47e3dae1..b3f12025ad 100644 --- a/engine/crates/layerfs-node/src/direct.rs +++ b/engine/crates/layerfs-node/src/direct.rs @@ -33,10 +33,29 @@ struct Opened { static STORES: LazyLock>> = LazyLock::new(Default::default); -/// The Store path a call names. +/// The Store path a call names — absolute, like the home it is set beside: +/// a relative spelling would resolve against the process's working +/// directory and silently name another file. fn path(args: &SdkValue) -> Result { match args.get_text("storePath") { - Some(path) if !path.is_empty() => Ok(PathBuf::from(path)), + Some(text) if !text.is_empty() => { + // NUL would silently truncate at the Win32 path boundary and + // name a different file (round-5 C3 F4). + if text.contains('\0') { + return Err(refused( + "InvalidInput", + "the client's storePath must not contain NUL", + )); + } + let path = PathBuf::from(text); + if !path.is_absolute() { + return Err(refused( + "InvalidInput", + "the client's storePath must be absolute (the installed Store file)", + )); + } + Ok(path) + } _ => Err(refused( "InvalidInput", "Store calls need the client's storePath (the installed Store file)", @@ -110,6 +129,8 @@ pub(crate) fn release(args: &SdkValue) -> Outcome { } let opened = stores.remove(&store).expect("present above"); drop(stores); + // The service's close coordinates its sessions' teardown under the + // Store's writer lane itself. let closed = match Arc::try_unwrap(opened.service) { Ok(service) => service.close().map_err(failed), Err(_) => Ok(()), @@ -174,6 +195,7 @@ pub(crate) fn call(method: &str, args: SdkValue, backend: bool) -> Outcome { } /// The Branch id of `name` in the project a call names, read in this process. +#[cfg(unix)] pub(crate) fn branch_id( args: &SdkValue, name: &str, @@ -184,6 +206,7 @@ pub(crate) fn branch_id( } /// The newest Commit of `branch`, read in this process; `None` when it has none. +#[cfg(unix)] pub(crate) fn head( args: &SdkValue, branch: layerfs_history::BranchId, diff --git a/engine/crates/layerfs-node/src/discovery.rs b/engine/crates/layerfs-node/src/discovery.rs index c0e058d87c..221e948094 100644 --- a/engine/crates/layerfs-node/src/discovery.rs +++ b/engine/crates/layerfs-node/src/discovery.rs @@ -1,4 +1,7 @@ //! Private, permission-authorized project and Sandbox discovery. No mutation replay. +// One audited unsafe surface with the Windows twin: uid/flock calls std does +// not expose; the boundary guard admits only the two discovery modules. +#![allow(unsafe_code)] use crate::{ job::{failed, refused}, registry::{self, Mount, Process, Project, SandboxState}, diff --git a/engine/crates/layerfs-node/src/discovery_windows.rs b/engine/crates/layerfs-node/src/discovery_windows.rs new file mode 100644 index 0000000000..d2cc55635b --- /dev/null +++ b/engine/crates/layerfs-node/src/discovery_windows.rs @@ -0,0 +1,286 @@ +//! Private, permission-authorized project discovery on Windows: the Store +//! face of the unix `discovery` module. Sandbox discovery and daemon +//! operation are refused `Unsupported`; the native Store route serves +//! project, fs and branch calls in-process. +//! +//! Windows cannot express the unix entry's owner/mode privacy check; a +//! symlink and an unexpected entry kind are still refused, and the audited +//! ACL module (layerfs-store-service) both restricts what this module +//! creates to its owner and refuses an existing entry whose DACL grants +//! access beyond the owner. The discovery turn keeps the +//! unix discipline: this process's calls take turns on one directory, and +//! between processes one named kernel mutex per directory is held, released +//! by the kernel when its holder exits. +#![allow(unsafe_code)] +use crate::{ + job::{failed, refused}, + registry::{self, Project}, +}; +use layerfs_bridge::{ + control::{Reply, SdkRefusal, SdkValue}, + provision::StoreManifest, +}; +use layerfs_persistence::{SealedStore, SqlitePersistenceProfile}; +use std::{ + collections::BTreeSet, + fs::{self, OpenOptions}, + io::{Read, Write}, + path::{Path, PathBuf}, + sync::{Condvar, Mutex}, + time::{Duration, Instant}, +}; + +type Result = std::result::Result; + +pub(crate) fn home(args: &SdkValue) -> Result { + let home = args + .get_text("home") + .map(PathBuf::from) + .or_else(|| std::env::var_os("LAYERFS_HOME").map(PathBuf::from)) + .ok_or_else(|| refused("InvalidInput", "LAYERFS_HOME is required"))?; + if !home.is_absolute() { + return Err(refused("InvalidInput", "LAYERFS_HOME must be absolute")); + } + directory(&home)?; + Ok(home) +} + +/// A private entry on Windows: no symlink, the expected kind, and a DACL +/// that grants nothing beyond its owner — the unix arm reads the same +/// intent from group/other mode bits. +fn private(path: &Path, dir: bool) -> Result<()> { + let meta = fs::symlink_metadata(path).map_err(failed)?; + if meta.file_type().is_symlink() || meta.is_dir() != dir { + return Err(refused( + "Unauthorized", + format!( + "{} must be a private {} entry", + path.display(), + if dir { "directory" } else { "file" } + ), + )); + } + if let Err(error) = layerfs_store_service::windows_acl::require_owner_only(path) { + return Err(refused("Unauthorized", error.to_string())); + } + Ok(()) +} + +pub(crate) fn directory(path: &Path) -> Result<()> { + match fs::DirBuilder::new().create(path) { + Ok(()) => layerfs_store_service::windows_acl::restrict_to_owner(path).map_err(failed), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(()), + Err(e) => Err(failed(e)), + }?; + private(path, true) +} + +pub(crate) fn project_dir(args: &SdkValue) -> Result { + let name = registry::text(args, "project")?; + layerfs_history::HistoryName::new(name).map_err(failed)?; + let path = home(args)?.join(name); + private(&path, true)?; + Ok(path) +} + +/// Calls take turns on one directory: in this process before its mutex, and +/// between processes on the mutex itself. The wait is readiness, bounded, +/// and nothing has been attempted before it ends. +const IN_PROCESS_WAIT: Duration = Duration::from_secs(60); +static HELD: Mutex> = Mutex::new(BTreeSet::new()); +static FREED: Condvar = Condvar::new(); + +/// One discovery mutation's ownership of a directory: this process's turn +/// and the named kernel mutex. Dropping it releases both. +pub(crate) struct Guard { + handle: usize, + directory: PathBuf, +} +impl Drop for Guard { + fn drop(&mut self) { + unsafe { + windows_sys::Win32::System::Threading::ReleaseMutex(self.handle as _); + windows_sys::Win32::Foundation::CloseHandle(self.handle as _); + } + let mut held = HELD.lock().unwrap_or_else(|error| error.into_inner()); + held.remove(&self.directory); + FREED.notify_all(); + } +} + +/// One mutex name per directory; the canonical path identifies it between +/// processes. SHA-256 truncated, not `DefaultHasher`: the hasher's algorithm +/// may change between Rust releases and silently split one directory onto +/// two guards across builds (K3 review F1, with the persistence lane name). +fn mutex_name(directory: &Path) -> String { + use sha2::{Digest, Sha256}; + let resolved = fs::canonicalize(directory).unwrap_or_else(|_| directory.to_path_buf()); + let digest = Sha256::digest(resolved.as_os_str().as_encoded_bytes()); + let mut tag = [0u8; 8]; + tag.copy_from_slice(&digest[..8]); + format!("Local\\layerfs-discovery-{:016x}", u64::from_be_bytes(tag)) +} + +pub(crate) fn guard(directory: &Path) -> Result { + let deadline = Instant::now() + IN_PROCESS_WAIT; + let mut held = registry::lock(&HELD)?; + while held.contains(directory) { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(refused( + "Busy", + "another call of this process owns this discovery mutation", + )); + } + held = FREED + .wait_timeout(held, remaining) + .map_err(|_| refused("Failed", "discovery turn poisoned"))? + .0; + } + held.insert(directory.to_owned()); + drop(held); + match os_lock(directory, deadline) { + Ok(handle) => Ok(Guard { + handle, + directory: directory.to_owned(), + }), + Err(refusal) => { + let mut held = HELD.lock().unwrap_or_else(|error| error.into_inner()); + held.remove(directory); + FREED.notify_all(); + Err(refusal) + } + } +} + +/// The named kernel mutex of a directory, waited for until `deadline`; a +/// holder process that exits releases it. +fn os_lock(directory: &Path, deadline: Instant) -> Result { + use windows_sys::Win32::Foundation::{ + CloseHandle, WAIT_ABANDONED, WAIT_OBJECT_0, WAIT_TIMEOUT, + }; + use windows_sys::Win32::System::Threading::{CreateMutexW, WaitForSingleObject}; + let name: Vec = mutex_name(directory) + .encode_utf16() + .chain(std::iter::once(0)) + .collect(); + let handle = unsafe { CreateMutexW(std::ptr::null(), 0, name.as_ptr()) }; + if handle.is_null() { + return Err(refused("Failed", "the discovery mutex cannot open")); + } + // One bounded attempt, not a retry loop: every exit below is final. + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + unsafe { CloseHandle(handle) }; + return Err(refused( + "Busy", + "another process owns this discovery mutation", + )); + } + let waited = + unsafe { WaitForSingleObject(handle, remaining.as_millis().min(u32::MAX as u128) as u32) }; + match waited { + WAIT_OBJECT_0 | WAIT_ABANDONED => Ok(handle as usize), + WAIT_TIMEOUT => { + unsafe { CloseHandle(handle) }; + Err(refused( + "Busy", + "another process owns this discovery mutation", + )) + } + _ => { + unsafe { CloseHandle(handle) }; + Err(refused("Failed", "the discovery mutex cannot be held")) + } + } +} + +pub(crate) fn save(path: &Path, value: SdkValue) -> Result<()> { + let bytes = layerfs_bridge::control::Answer { + id: 1, + reply: Reply::Sdk(value), + } + .encode() + .map_err(failed)?; + let temp = path.with_extension(format!( + "{}-{}", + std::process::id(), + layerfs_bridge::sdk_values::hex(®istry::random()?[..8]) + )); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temp) + .map_err(failed)?; + file.write_all(&bytes).map_err(failed)?; + drop(file); + layerfs_store_service::windows_acl::restrict_to_owner(&temp).map_err(failed)?; + fs::rename(temp, path).map_err(failed) +} + +pub(crate) fn load(path: &Path) -> Result { + private(path, false)?; + let file = OpenOptions::new().read(true).open(path).map_err(failed)?; + if file.metadata().map_err(failed)?.len() > 65535 { + return Err(refused("InvalidInput", "discovery record too large")); + } + let mut bytes = Vec::new(); + file.take(65536).read_to_end(&mut bytes).map_err(failed)?; + match layerfs_bridge::control::Answer::decode(&bytes) + .map_err(failed)? + .reply + { + Reply::Sdk(value) => Ok(value), + _ => Err(refused("InvalidInput", "discovery record kind")), + } +} + +pub(crate) fn save_project(path: &Path, project: &Project) -> Result<()> { + save( + &path.join("project.record"), + SdkValue::map([ + ( + "manifest", + SdkValue::Bytes(project.manifest.encode().map_err(failed)?), + ), + ( + "installed", + project + .installed + .as_ref() + .map(|m| m.encode().map(SdkValue::Bytes)) + .transpose() + .map_err(failed)? + .unwrap_or_default(), + ), + ]), + ) +} + +pub(crate) fn project(args: &SdkValue) -> Result { + let path = project_dir(args)?; + let value = load(&path.join("project.record"))?; + let decode = |name| match value.get(name) { + Some(SdkValue::Bytes(bytes)) => StoreManifest::decode(bytes).map_err(failed), + _ => Err(refused("NoSuchProject", "no complete project manifest")), + }; + let manifest = decode("manifest")?; + if manifest.profile != layerfs_bridge::provision::StoreProfile::Disposable { + return Err(refused("InvalidInput", "Disposable required")); + } + let installed = if matches!(value.get("installed"), Some(SdkValue::Bytes(_))) { + Some(decode("installed")?) + } else { + None + }; + Ok(Project { + store: SealedStore { + path: path.join("sealed.sqlite"), + profile: SqlitePersistenceProfile::Disposable, + sqlite_version: manifest.host_sqlite.clone(), + bytes: manifest.bytes, + }, + manifest, + installed, + }) +} diff --git a/engine/crates/layerfs-node/src/dispatch.rs b/engine/crates/layerfs-node/src/dispatch.rs index 99c80b12b6..5102c80d35 100644 --- a/engine/crates/layerfs-node/src/dispatch.rs +++ b/engine/crates/layerfs-node/src/dispatch.rs @@ -2,8 +2,12 @@ //! public table (`call`: project, branch, fs, sandbox, workspace) and the //! backend table (`backendCall`: `backend.store.*`, `backend.daemon.*` and the //! runtime's `runtime.*` reads and actions). +#[cfg(unix)] +use crate::job; use crate::job::{refused, Job, Outcome}; -use crate::{convert, direct, exec, job, output, project, registry, workspace}; +use crate::{convert, direct, project}; +#[cfg(unix)] +use crate::{exec, output, registry, workspace}; use layerfs_bridge::control::SdkValue; use napi::{bindgen_prelude::AsyncTask, JsUnknown}; @@ -28,33 +32,63 @@ fn dispatch(op: &str, args: SdkValue) -> Outcome { .map_err(|error| refused("InvalidInput", error.to_string()))?; } } - match op { - "project.init" => project::init(&args), - "project.list_sandboxes" => workspace::sandboxes(&args), - "sandbox.list_workspaces" => listed(daemon_backend(op, args)?), - "sandbox.create" => crate::sandbox::create(&args), - "sandbox.close" => crate::sandbox::close(&args), - "workspace.mount" => workspace::mount(&args), - "workspace.commit" => workspace::commit(&args), - "workspace.unmount" => workspace::unmount(&args), - "workspace.exec" if args.get_bool("background") => Err(refused( - "InvalidInput", - "background launch is backend.daemon.process.start", - )), - "workspace.exec" => exec::exec(&args), - "workspace.exec_release" => output::release(&args), - _ if op.starts_with("backend.") || op.starts_with("runtime.") => Err(refused( - "UnknownMethod", - "backend and runtime methods require backendCall", - )), - // Store calls run in this process; there is no other route. - _ if op.contains('.') => direct::call(op, args, false), - _ => Err(refused("UnknownMethod", format!("no operation {op}"))), + #[cfg(unix)] + { + match op { + "project.init" => project::init(&args), + "project.list_sandboxes" => workspace::sandboxes(&args), + "sandbox.list_workspaces" => listed(daemon_backend(op, args)?), + "sandbox.create" => crate::sandbox::create(&args), + "sandbox.close" => crate::sandbox::close(&args), + "workspace.mount" => workspace::mount(&args), + "workspace.commit" => workspace::commit(&args), + "workspace.unmount" => workspace::unmount(&args), + "workspace.exec" if args.get_bool("background") => Err(refused( + "InvalidInput", + "background launch is backend.daemon.process.start", + )), + "workspace.exec" => exec::exec(&args), + "workspace.exec_release" => output::release(&args), + _ if op.starts_with("backend.") || op.starts_with("runtime.") => Err(refused( + "UnknownMethod", + "backend and runtime methods require backendCall", + )), + // Store calls run in this process; there is no other route. + _ if op.contains('.') => direct::call(op, args, false), + _ => Err(refused("UnknownMethod", format!("no operation {op}"))), + } + } + #[cfg(windows)] + { + // Workspace and Sandbox operations need a daemon; the Windows-native + // Store route serves project, fs and branch calls in-process. + match op { + "project.init" => project::init(&args), + "project.list_sandboxes" + | "sandbox.list_workspaces" + | "sandbox.create" + | "sandbox.close" + | "workspace.mount" + | "workspace.commit" + | "workspace.unmount" + | "workspace.exec" + | "workspace.exec_release" => Err(refused( + "Unsupported", + "Workspace and Sandbox operations need a daemon; the Windows-native Store route serves project, fs and branch calls in-process", + )), + _ if op.starts_with("backend.") || op.starts_with("runtime.") => Err(refused( + "UnknownMethod", + "backend and runtime methods require backendCall", + )), + _ if op.contains('.') => direct::call(op, args, false), + _ => Err(refused("UnknownMethod", format!("no operation {op}"))), + } } } /// `sandbox.list_workspaces`: the daemon's read model without its history /// half, `{ kind, version, workspaces }`; the SDK pages and shapes them. +#[cfg(unix)] fn listed(answer: SdkValue) -> Outcome { let SdkValue::Map(fields) = answer else { return Err(refused( @@ -72,6 +106,7 @@ fn listed(answer: SdkValue) -> Outcome { /// A daemon backend call through the named Sandbox of the project: its /// Workspaces and the read model that lists them. `deploy` only selects how this process dials +#[cfg(unix)] /// the daemon, so it is not sent on. fn daemon_backend(op: &str, args: SdkValue) -> Outcome { let state = registry::sandbox_live(&args)?; @@ -100,8 +135,21 @@ pub(crate) fn exec_output( args: JsUnknown, page: napi::bindgen_prelude::Buffer, ) -> napi::Result> { - let args = convert::from_js(args)?; - Ok(AsyncTask::new(Job::new(move || output::page(&args, page)))) + #[cfg(windows)] + { + let _ = (&args, &page); + Ok(AsyncTask::new(Job::new(move || { + Err(refused( + "Unsupported", + "exec output needs a daemon Workspace; the Windows-native Store route serves project, fs and branch calls in-process", + )) + }))) + } + #[cfg(unix)] + { + let args = convert::from_js(args)?; + Ok(AsyncTask::new(Job::new(move || output::page(&args, page)))) + } } /// Opens or releases a Store of this process. @@ -118,27 +166,57 @@ pub(crate) fn backend_call(op: String, args: JsUnknown) -> napi::Result workspace::status(&args), - "backend.daemon.process.start" => exec::exec(&args), - "backend.daemon.process.list" => exec::processes(&args), - "backend.daemon.process.cancel" => exec::cancel(&args), - "runtime.project.get_init" => crate::project::progress(&args), - "runtime.sandbox.reclaim" => workspace::reclaim(&args), - "backend.daemon.sandbox.get_log" => crate::sandbox::get_log(&args), - "backend.store.project.get_history" - | "backend.store.project.get_storage" - | "backend.store.project.list_overwritten" - | "backend.store.branch.list_commit_storage" - | "backend.store.fs.get_storage" - | "backend.store.fs.list_revisions" => direct::call(&op, args, true), - "backend.daemon.workspace.get_storage" | "backend.daemon.workspace.list_changes" => { - daemon_backend(&op, args) + Job::new(move || { + #[cfg(unix)] + { + match op.as_str() { + "backend.daemon.workspace.status" => workspace::status(&args), + "backend.daemon.process.start" => exec::exec(&args), + "backend.daemon.process.list" => exec::processes(&args), + "backend.daemon.process.cancel" => exec::cancel(&args), + "runtime.project.get_init" => crate::project::progress(&args), + "runtime.sandbox.reclaim" => workspace::reclaim(&args), + "backend.daemon.sandbox.get_log" => crate::sandbox::get_log(&args), + "backend.store.project.get_history" + | "backend.store.project.get_storage" + | "backend.store.project.list_overwritten" + | "backend.store.branch.list_commit_storage" + | "backend.store.fs.get_storage" + | "backend.store.fs.list_revisions" => direct::call(&op, args, true), + "backend.daemon.workspace.get_storage" + | "backend.daemon.workspace.list_changes" => daemon_backend(&op, args), + _ => Err(refused( + "UnknownMethod", + format!("no backend operation {op}"), + )), + } + } + #[cfg(windows)] + { + match op.as_str() { + "runtime.project.get_init" => crate::project::progress(&args), + "backend.store.project.get_history" + | "backend.store.project.get_storage" + | "backend.store.project.list_overwritten" + | "backend.store.branch.list_commit_storage" + | "backend.store.fs.get_storage" + | "backend.store.fs.list_revisions" => direct::call(&op, args, true), + // Daemon-only operations say so by name; an unknown + // operation is still unknown here, as on unix (K3-R3 F13). + "backend.daemon.workspace.status" + | "backend.daemon.process.start" + | "backend.daemon.process.list" + | "backend.daemon.process.cancel" + | "backend.daemon.sandbox.get_log" + | "backend.daemon.workspace.get_storage" + | "backend.daemon.workspace.list_changes" + | "runtime.sandbox.reclaim" => Err(refused( + "Unsupported", + "this backend call needs a daemon; the Windows-native Store route serves Store calls in-process", + )), + _ => Err(refused("UnknownMethod", format!("no backend operation {op}"))), + } } - _ => Err(refused( - "UnknownMethod", - format!("no backend operation {op}"), - )), }) .traced(scope), )) diff --git a/engine/crates/layerfs-node/src/lib.rs b/engine/crates/layerfs-node/src/lib.rs index 298d5020af..d80a486dc3 100644 --- a/engine/crates/layerfs-node/src/lib.rs +++ b/engine/crates/layerfs-node/src/lib.rs @@ -1,16 +1,26 @@ //! Node entry points for the public SDK and its backend table (#428). +#![deny(unsafe_code)] mod convert; mod direct; +#[cfg(unix)] +mod discovery; +#[cfg(windows)] +#[path = "discovery_windows.rs"] mod discovery; mod dispatch; +#[cfg(unix)] mod exec; mod job; +#[cfg(unix)] mod output; +#[cfg(unix)] mod pool; mod project; mod registry; +#[cfg(unix)] mod sandbox; mod trace; +#[cfg(unix)] mod workspace; use job::Job; diff --git a/engine/crates/layerfs-node/src/project.rs b/engine/crates/layerfs-node/src/project.rs index bcfeec94d3..409a09f26e 100644 --- a/engine/crates/layerfs-node/src/project.rs +++ b/engine/crates/layerfs-node/src/project.rs @@ -41,17 +41,59 @@ pub(crate) fn init(args: &SdkValue) -> Outcome { // checked first: a refused policy, timeout or Store path leaves no project entry let policy = policy(args)?; let deadline = deadline(args)?; - let store_path = - match args.get_text("storePath") { - Some(path) if !path.is_empty() => PathBuf::from(path), - _ => return Err(refused( + let store_path = match args.get_text("storePath") { + Some(path) if !path.is_empty() => { + // NUL would silently truncate at the Win32 path boundary and + // name a different file (round-5 C3 F4). + if path.contains('\0') { + return Err(refused( + "InvalidInput", + "the client's storePath must not contain NUL", + )); + } + let path = PathBuf::from(path); + // Absolute, like every Store call's storePath: a relative + // spelling would resolve against this process's working + // directory and leave a half-initialized project entry + // before the install path check refuses it (round-3 A1 F2). + if !path.is_absolute() { + return Err(refused( + "InvalidInput", + "the client's storePath must be absolute (project.init installs the Store file)", + )); + } + path + } + _ => { + return Err(refused( "InvalidInput", "project.init installs the Store at the client's storePath (LAYERFS_STORE_PATH)", - )), - }; - // the Store's directory is made by the install; its volume must carry the Store's locks - let global = store_path.parent().unwrap_or(&store_path); - layerfs_store_service::kernel_guard::check(global) + )) + } + }; + // The Store's directory is created first (owner-only, like the install + // does): check takes the Store file and examines its parent's volume, + // and the SDK layout may not have made it yet. + let store_directory = store_path.parent().unwrap_or(&store_path).to_path_buf(); + if !store_directory.as_os_str().is_empty() && !store_directory.exists() { + #[cfg(unix)] + { + // Owner-only from the first instant, as the install does: a + // create-then-chmod pair leaves a umask window (round-5 C1 F3). + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new() + .mode(0o700) + .create(&store_directory) + .map_err(failed)?; + } + #[cfg(windows)] + { + std::fs::create_dir(&store_directory).map_err(failed)?; + layerfs_store_service::windows_acl::restrict_to_owner(&store_directory) + .map_err(failed)?; + } + } + layerfs_store_service::kernel_guard::check(&store_path) .map_err(|reason| refused("Unsupported", reason.0))?; let directory = home.join(name); // The home entry is the project: creating it is the one existence check, @@ -63,11 +105,14 @@ pub(crate) fn init(args: &SdkValue) -> Outcome { failed(error) } })?; + #[cfg(unix)] std::fs::set_permissions(&directory, { use std::os::unix::fs::PermissionsExt; std::fs::Permissions::from_mode(0o700) }) .map_err(failed)?; + #[cfg(windows)] + layerfs_store_service::windows_acl::restrict_to_owner(&directory).map_err(failed)?; let _guard = crate::discovery::guard(&directory)?; let history = |text: &str| HistoryName::new(text).map_err(failed); let request = InitRequest { diff --git a/engine/crates/layerfs-node/src/registry.rs b/engine/crates/layerfs-node/src/registry.rs index aeeb25bc47..d9a4fda1a9 100644 --- a/engine/crates/layerfs-node/src/registry.rs +++ b/engine/crates/layerfs-node/src/registry.rs @@ -8,9 +8,15 @@ //! binding) also keeps the background processes it started; the daemon holds //! no process record. use crate::job::refused; +#[cfg(unix)] +use crate::pool::Controls; +#[cfg(unix)] use layerfs_api::MountedWorkspace; -use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken}; +#[cfg(unix)] +use layerfs_bridge::control::WorkspaceToken; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_bridge::provision::StoreManifest; +#[cfg(unix)] use layerfs_sandbox::{backend::docker::Sandbox, CommandIdentity}; use std::collections::BTreeMap; use std::sync::{Arc, LazyLock, Mutex, MutexGuard}; @@ -24,12 +30,14 @@ pub(crate) struct Project { } /// One mounted Workspace. +#[cfg(unix)] pub(crate) struct Mount { pub(crate) workspace: MountedWorkspace, pub(crate) branch: String, } /// One background process started in a Workspace. +#[cfg(unix)] pub(crate) struct Process { pub(crate) path: String, pub(crate) command: String, @@ -39,6 +47,7 @@ pub(crate) struct Process { } /// One live Sandbox: Sandbox `id` (its number) of `project`. +#[cfg(unix)] pub(crate) struct SandboxState { pub(crate) id: String, pub(crate) project: String, @@ -47,7 +56,7 @@ pub(crate) struct SandboxState { pub(crate) directory: std::path::PathBuf, pub(crate) runtime: Sandbox, pub(crate) identity: CommandIdentity, - pub(crate) controls: crate::pool::Controls, + pub(crate) controls: Controls, pub(crate) mounts: Mutex>, pub(crate) processes: Mutex>, } @@ -56,6 +65,7 @@ pub(crate) struct SandboxState { pub(crate) static INITS: LazyLock>>> = LazyLock::new(Default::default); /// This process's live Sandboxes, by their record directory. +#[cfg(unix)] pub(crate) static SANDBOXES: LazyLock>>> = LazyLock::new(Default::default); @@ -77,6 +87,7 @@ pub(crate) fn text<'a>(args: &'a SdkValue, name: &str) -> Result<&'a str, SdkRef /// mount checks just that mount (`SandboxState::token`, `directory`); one /// that needs every mount or process another process may have changed takes /// [`sandbox_refreshed`]. +#[cfg(unix)] pub(crate) fn sandbox(args: &SdkValue) -> Result, SdkRefusal> { let directory = crate::discovery::sandbox_dir(args, text(args, "sandbox")?)?; let existing = lock(&SANDBOXES)?.get(&directory).cloned(); @@ -95,6 +106,7 @@ pub(crate) fn sandbox(args: &SdkValue) -> Result, SdkRefusal> /// records (which inspects too). A container found gone or stopped (an idle /// Sandbox's daemon ends itself, #373) is the refusal of /// `discovery::gone`, and this process forgets the state. +#[cfg(unix)] pub(crate) fn sandbox_live(args: &SdkValue) -> Result, SdkRefusal> { let directory = crate::discovery::sandbox_dir(args, text(args, "sandbox")?)?; let Some(state) = lock(&SANDBOXES)?.get(&directory).cloned() else { @@ -116,6 +128,7 @@ pub(crate) fn sandbox_live(args: &SdkValue) -> Result, SdkRefu } /// `args` naming Sandbox `id` of their project. +#[cfg(unix)] pub(crate) fn naming(args: &SdkValue, id: &str) -> SdkValue { let mut fields = match args { SdkValue::Map(fields) => fields @@ -131,12 +144,14 @@ pub(crate) fn naming(args: &SdkValue, id: &str) -> SdkValue { /// The Sandbox an operation names, its tables read again from the shared /// records and the daemon (one status call per mount). +#[cfg(unix)] pub(crate) fn sandbox_refreshed(args: &SdkValue) -> Result, SdkRefusal> { let state = sandbox(args)?; crate::discovery::refresh(&state)?; Ok(state) } +#[cfg(unix)] impl SandboxState { /// The token of the Workspace mounted at `path`. pub(crate) fn token(&self, path: &str) -> Result { diff --git a/engine/crates/layerfs-node/src/trace.rs b/engine/crates/layerfs-node/src/trace.rs index 8fad42e91f..3c7ae58d47 100644 --- a/engine/crates/layerfs-node/src/trace.rs +++ b/engine/crates/layerfs-node/src/trace.rs @@ -2,10 +2,17 @@ //! makes travels as a traced observation under the call's scope, and the //! telemetry of the replies is collected for its trace line. An untraced call //! sends exactly what it did before. Telemetry never changes a result. +#[cfg(unix)] use crate::pool::{Controls, Lease}; -use layerfs_api::internal::{self, Control}; -use layerfs_bridge::control::{SdkRefusal, SdkValue}; +#[cfg(unix)] +use layerfs_api::internal; +#[cfg(unix)] +use layerfs_api::internal::Control; +#[cfg(unix)] +use layerfs_bridge::control::SdkRefusal; +use layerfs_bridge::control::SdkValue; use std::cell::{Cell, RefCell}; +#[cfg(unix)] use std::ops::{Deref, DerefMut}; use std::time::Duration; @@ -71,16 +78,19 @@ pub(crate) fn collect(value: SdkValue) { } /// A Sandbox's control connection, traced while this thread is. +#[cfg(unix)] pub(crate) struct Traced<'a>(Lease<'a>); /// Lends a control connection for one call's exchanges; the wait for it is /// the call's queue time. +#[cfg(unix)] pub(crate) fn control(controls: &Controls) -> Result, SdkRefusal> { let (mut lease, waited) = controls.lend()?; WAITED.set(WAITED.get() + waited); internal::trace(&mut lease, SCOPE.get()); Ok(Traced(lease)) } +#[cfg(unix)] impl Drop for Traced<'_> { fn drop(&mut self) { let telemetry = internal::take_telemetry(&mut self.0); @@ -91,12 +101,14 @@ impl Drop for Traced<'_> { }); } } +#[cfg(unix)] impl Deref for Traced<'_> { type Target = Control; fn deref(&self) -> &Control { &self.0 } } +#[cfg(unix)] impl DerefMut for Traced<'_> { fn deref_mut(&mut self) -> &mut Control { &mut self.0 diff --git a/engine/crates/layerfs-overlay/Cargo.toml b/engine/crates/layerfs-overlay/Cargo.toml index a0de581508..16b14a757b 100644 --- a/engine/crates/layerfs-overlay/Cargo.toml +++ b/engine/crates/layerfs-overlay/Cargo.toml @@ -13,3 +13,11 @@ rusqlite = { version = "=0.40.2", features = ["cache", "limits", "blob"] } [target.'cfg(target_os = "linux")'.dependencies] rusqlite = { version = "=0.40.2", features = ["bundled"] } + +[target.'cfg(target_os = "windows")'.dependencies] +# Windows has no system SQLite, and the audited identity module needs the +# Win32 file-information calls; CreateFileW is additionally gated on +# Win32_Security by its SECURITY_ATTRIBUTES parameter. windows-sys is +# already in the locked graph. +rusqlite = { version = "=0.40.2", features = ["bundled"] } +windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_Storage_FileSystem"] } diff --git a/engine/crates/layerfs-overlay/src/database/mod.rs b/engine/crates/layerfs-overlay/src/database/mod.rs index 17948fa35b..daef49b89b 100644 --- a/engine/crates/layerfs-overlay/src/database/mod.rs +++ b/engine/crates/layerfs-overlay/src/database/mod.rs @@ -5,5 +5,9 @@ pub(crate) mod startup; pub(crate) mod statements; pub(crate) mod physical; +// The audited module's own `#![allow(unsafe_code)]` admits it; a second +// outer attribute is a clippy duplication. +#[cfg(target_os = "windows")] +pub(crate) mod windows_identity; pub(crate) mod accounting; diff --git a/engine/crates/layerfs-overlay/src/database/physical.rs b/engine/crates/layerfs-overlay/src/database/physical.rs index 65b7e9b7ec..b5c9b23144 100644 --- a/engine/crates/layerfs-overlay/src/database/physical.rs +++ b/engine/crates/layerfs-overlay/src/database/physical.rs @@ -94,11 +94,57 @@ impl Allocation { work: self.work.get(), }) } - #[cfg(not(any(target_os = "macos", target_os = "linux")))] + #[cfg(target_os = "windows")] + pub(crate) fn new(file: File, path: &Path) -> OverlayResult { + let identity = super::windows_identity::windows_handle_identity(&file)?; + if identity.links != 1 { + return Err(OverlayError::Invalid("allocation file identity")); + } + let allocated = super::windows_identity::windows_allocation_size(&file)?; + Ok(Self { + file, + device: identity.device, + inode: identity.index, + path: path.to_owned(), + work: Cell::new(AllocationWork::default()), + high_water: Cell::new(allocated), + }) + } + #[cfg(target_os = "windows")] + pub(crate) fn state(&self) -> OverlayResult { + let mut work = self.work.get(); + work.observations = work.observations.saturating_add(1); + self.work.set(work); + let meta = self.file.metadata()?; + let by_path = super::windows_identity::windows_path_identity(&self.path)?; + let by_handle = super::windows_identity::windows_handle_identity(&self.file)?; + if by_path.device != by_handle.device + || by_path.index != by_handle.index + || by_handle.device != self.device + || by_handle.index != self.inode + || by_handle.links != 1 + { + return Err(OverlayError::Invalid("allocation file identity")); + } + // The allocated footprint comes from FileStandardInfo on the same + // retained handle (compression/sparse aware); the disposable overlay + // never preallocates, so this tracks the tail's actual clusters. + let allocated = super::windows_identity::windows_allocation_size(&self.file)?; + self.high_water.set(self.high_water.get().max(allocated)); + Ok(AllocationState { + logical_bytes: meta.len(), + allocated_bytes: allocated, + high_water_allocated_bytes: self.high_water.get(), + reserved_tail_bytes: 0, + cleanup_headroom_bytes: 0, + work: self.work.get(), + }) + } + #[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))] pub(crate) fn new(_file: File, _path: &Path) -> OverlayResult { Err(OverlayError::UnsupportedPlatform) } - #[cfg(not(any(target_os = "macos", target_os = "linux")))] + #[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))] pub(crate) fn state(&self) -> OverlayResult { Err(OverlayError::UnsupportedPlatform) } @@ -119,7 +165,21 @@ impl Allocation { // those are not ordinary writable capacity. Retain the original cause. Ok(filesystem.blocks_available() == 0) } - #[cfg(not(any(target_os = "macos", target_os = "linux")))] + #[cfg(target_os = "windows")] + pub(crate) fn device_full(&self) -> OverlayResult { + self.state()?; + let mut work = self.work.get(); + work.capacity_checks = work.capacity_checks.saturating_add(1); + self.work.set(work); + let directory = self + .path + .parent() + .unwrap_or_else(|| std::path::Path::new(".")); + let free = super::windows_identity::windows_free_bytes(directory) + .map_err(|_| OverlayError::Invalid("capacity observation"))?; + Ok(free == 0) + } + #[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))] pub(crate) fn device_full(&self) -> OverlayResult { Err(OverlayError::UnsupportedPlatform) } diff --git a/engine/crates/layerfs-overlay/src/database/startup.rs b/engine/crates/layerfs-overlay/src/database/startup.rs index 2e7a33f8ac..db39ac8e34 100644 --- a/engine/crates/layerfs-overlay/src/database/startup.rs +++ b/engine/crates/layerfs-overlay/src/database/startup.rs @@ -66,7 +66,11 @@ fn initialize(path: &Path, config: ProfileConfig, template: Option<&Path>) -> Cr let mut work = CreationWork::default(); let mut allocation = None; let result = (|| { - if !cfg!(any(target_os = "macos", target_os = "linux")) { + if !cfg!(any( + target_os = "macos", + target_os = "linux", + target_os = "windows" + )) { return Err(crate::OverlayError::UnsupportedPlatform); } let mut options = OpenOptions::new(); diff --git a/engine/crates/layerfs-overlay/src/database/windows_identity.rs b/engine/crates/layerfs-overlay/src/database/windows_identity.rs new file mode 100644 index 0000000000..2cd338a5c0 --- /dev/null +++ b/engine/crates/layerfs-overlay/src/database/windows_identity.rs @@ -0,0 +1,135 @@ +//! The overlay's one audited Windows FFI surface: a file's real identity. +//! +//! This follows the audited-module precedent of the storage codec, the +//! persistence file-control and Windows lanes, and the project host probe: +//! one owner-authorized module of raw Win32 calls, allowed by the boundary +//! guard and compiled on Windows only; no other module in this crate may +//! use unsafe Rust. The allocation's identity checks need the volume serial +//! number, the file index and the link count, which std does not expose. +#![allow(unsafe_code)] +use std::fs::File; +use std::io; +use std::os::windows::io::AsRawHandle; +use std::path::Path; +use windows_sys::Win32::Storage::FileSystem::BY_HANDLE_FILE_INFORMATION; + +/// One entry's Windows identity. +pub struct WindowsIdentity { + pub device: u64, + pub index: u64, + pub links: u64, +} + +/// The identity of the entry an open file already holds, without opening +/// anything new. +pub fn windows_handle_identity(file: &File) -> io::Result { + use windows_sys::Win32::Storage::FileSystem::GetFileInformationByHandle; + let mut information = BY_HANDLE_FILE_INFORMATION::default(); + let read = unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut information) }; + if read == 0 { + return Err(io::Error::last_os_error()); + } + Ok(identity_of(information)) +} + +/// The entry's on-disk allocation in bytes (compression/sparse aware), +/// read through FileStandardInfo on the open handle. +pub fn windows_allocation_size(file: &File) -> io::Result { + use windows_sys::Win32::Storage::FileSystem::{ + FileStandardInfo, GetFileInformationByHandleEx, FILE_STANDARD_INFO, + }; + let mut info = FILE_STANDARD_INFO::default(); + let ok = unsafe { + GetFileInformationByHandleEx( + file.as_raw_handle(), + FileStandardInfo, + &mut info as *mut FILE_STANDARD_INFO as *mut core::ffi::c_void, + std::mem::size_of::() as u32, + ) + }; + if ok == 0 { + return Err(io::Error::last_os_error()); + } + Ok(info.AllocationSize.max(0) as u64) +} + +/// The identity of `path`'s entry, read as attributes only: no data access, +/// no link traversal; directories need the backup-semantics flag, and a +/// reparse point is opened itself, never followed, matching unix's +/// `symlink_metadata` refusal of a substituted overlay file. +pub fn windows_path_identity(path: &Path) -> io::Result { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, GetFileInformationByHandle, FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, + OPEN_EXISTING, + }; + // Raw units, not a lossy string: names stay byte-exact (K3-R3 F7). + // Deliberately not the shared verbatim_wide: the product-source + // boundary keeps this crate free of first-party dependencies, and the + // inputs are TEMP-local overlay paths — a TEMP tree beyond MAX_PATH + // fails here cleanly (recorded exception, round-3 A1 F5). + let wide: Vec = path + .as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect(); + let handle = unsafe { + CreateFileW( + wide.as_ptr(), + 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + std::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, + std::ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + let mut information = BY_HANDLE_FILE_INFORMATION::default(); + let read = unsafe { GetFileInformationByHandle(handle, &mut information) }; + // Capture the failure before another syscall can overwrite it (K3-R3 F9). + let failure = (read == 0).then(io::Error::last_os_error); + unsafe { CloseHandle(handle) }; + match failure { + Some(error) => Err(error), + None => Ok(identity_of(information)), + } +} + +fn identity_of(information: BY_HANDLE_FILE_INFORMATION) -> WindowsIdentity { + WindowsIdentity { + device: u64::from(information.dwVolumeSerialNumber), + index: (u64::from(information.nFileIndexHigh) << 32) | u64::from(information.nFileIndexLow), + links: u64::from(information.nNumberOfLinks), + } +} + +/// The bytes available to the caller on `directory`'s volume. +pub(crate) fn windows_free_bytes(directory: &Path) -> io::Result { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Storage::FileSystem::GetDiskFreeSpaceExW; + // Raw wide, like the identity probe above: the boundary guard keeps + // this crate free of first-party dependencies (round-3 A1 F5). + let wide: Vec = directory + .as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect(); + let mut free: u64 = 0; + let read = unsafe { + GetDiskFreeSpaceExW( + wide.as_ptr(), + &mut free, + std::ptr::null_mut(), + std::ptr::null_mut(), + ) + }; + if read == 0 { + return Err(io::Error::last_os_error()); + } + Ok(free) +} diff --git a/engine/crates/layerfs-overlay/src/lib.rs b/engine/crates/layerfs-overlay/src/lib.rs index 1c38d3418d..f6c7dad459 100644 --- a/engine/crates/layerfs-overlay/src/lib.rs +++ b/engine/crates/layerfs-overlay/src/lib.rs @@ -3,7 +3,7 @@ //! The engine owns SQL and profiling. Filesystem validation, scheduling, kernel //! replies and process lifetimes belong to its callers. Windows bound each job; //! they do not limit total files, changes, payload or Workspace lifetime. -#![forbid(unsafe_code)] +#![deny(unsafe_code)] mod contract; mod database; @@ -46,6 +46,10 @@ pub use contract::kernel_directory::{ pub use database::accounting::{Resources, StoredCounts}; pub use database::physical::{AllocationState, AllocationWork, ClosedFile}; pub use database::startup::{Creation, CreationWork}; +#[cfg(target_os = "windows")] +pub use database::windows_identity::{ + windows_allocation_size, windows_handle_identity, windows_path_identity, +}; pub use db::Overlay; pub use diagnostics::payload::PayloadWork; pub use error::{CloseFailure, OverlayError, OverlayResult}; diff --git a/engine/crates/layerfs-overlay/tests/allocation_windows.rs b/engine/crates/layerfs-overlay/tests/allocation_windows.rs new file mode 100644 index 0000000000..1363bcb888 --- /dev/null +++ b/engine/crates/layerfs-overlay/tests/allocation_windows.rs @@ -0,0 +1,38 @@ +//! The Windows overlay's allocated-byte accounting reads FileStandardInfo on +//! the retained handle: the footprint of the written tail, not the logical +//! length the old stand-in reported. +#![cfg(target_os = "windows")] +mod payload_support; +use payload_support::*; + +#[test] +fn allocated_bytes_report_the_ondisk_footprint_of_the_written_tail() { + let temp = Temp::new(); + let db = temp.db(); + db.open_workspace([11; 32]).unwrap(); + let source = db.acquire_base_source(1).unwrap(); + // 300,001 bytes: not a cluster multiple, so the on-disk footprint is + // strictly above the logical length on a cluster-addressed volume — + // which the old logical-length stand-in could never report. + let file = File::new(&db, source, 3, pattern(300_001, 5)); + let _ = file.read(0, 16); + let state = db.allocation().unwrap(); + assert!( + state.allocated_bytes > 0, + "a written overlay holds clusters" + ); + assert!( + state.allocated_bytes >= state.logical_bytes, + "cluster rounding keeps the footprint at or above the tail" + ); + assert!( + state.allocated_bytes.saturating_sub(state.logical_bytes) < 1 << 20, + "the footprint tracks the tail within a cluster margin, got {} for {}", + state.allocated_bytes, + state.logical_bytes + ); + println!( + "allocated={} logical={}", + state.allocated_bytes, state.logical_bytes + ); +} diff --git a/engine/crates/layerfs-persistence/Cargo.toml b/engine/crates/layerfs-persistence/Cargo.toml index 747630e708..b1d20dfdc9 100644 --- a/engine/crates/layerfs-persistence/Cargo.toml +++ b/engine/crates/layerfs-persistence/Cargo.toml @@ -12,6 +12,8 @@ layerfs-history = { path = "../../../core/crates/layerfs-history", default-featu layerfs-content = { path = "../../../core/crates/layerfs-content" } layerfs-telemetry = { path = "../../../core/crates/layerfs-telemetry" } rusqlite = { version = "=0.40.2", features = ["cache", "limits", "blob"] } +# The lane name digest; the same locked sha2 layerfs-storage already uses. +sha2 = { version = "=0.11.0", default-features = false } [dev-dependencies] blake3 = { version = "=1.8.5", default-features = false } @@ -24,3 +26,9 @@ nix = { version = "=0.31.3", default-features = false, features = ["fs", "pthrea [target.'cfg(target_os = "linux")'.dependencies] rusqlite = { version = "=0.40.2", features = ["bundled"] } + +[target.'cfg(target_os = "windows")'.dependencies] +# Windows has no system SQLite; the Store bundles the same version. The +# writer lane and volume probe FFI use windows-sys, already in the locked graph. +rusqlite = { version = "=0.40.2", features = ["bundled"] } +windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_System_Threading", "Win32_Storage_FileSystem"] } diff --git a/engine/crates/layerfs-persistence/examples/issue327_counters_v2.rs b/engine/crates/layerfs-persistence/examples/issue327_counters_v2.rs index 524b2bbed8..792a49fad0 100644 --- a/engine/crates/layerfs-persistence/examples/issue327_counters_v2.rs +++ b/engine/crates/layerfs-persistence/examples/issue327_counters_v2.rs @@ -36,6 +36,7 @@ fn fixture() -> Vec> { }) .collect() } +#[cfg(unix)] fn main() { assert_eq!( std::env::var("SCREEN_SCENARIO").as_deref(), @@ -174,3 +175,8 @@ fn main() { }; writeln!(output,"{{\"sql_statements\":{},\"sql_vm_steps\":{},\"sqlite\":\"{}\",\"operation_ns\":{operation},\"complete_ns\":{},\"file_bytes\":{},\"allocated_bytes\":{allocated},\"sealed_bytes\":{},\"objects_input\":{COUNT},\"input_bytes\":{},\"cache\":\"created-in-sample Store; fixed warm in-memory fixture\",\"status\":\"RECORDED\"}}",work.statements,work.vm_steps,sql_version,total.elapsed().as_nanos(),metadata.len(),sealed.bytes,COUNT*BYTES).unwrap(); } + +#[cfg(not(unix))] +fn main() { + eprintln!("this frozen recorder requires Unix (st_blocks); nothing recorded"); +} diff --git a/engine/crates/layerfs-persistence/examples/issue327_counters_v3.rs b/engine/crates/layerfs-persistence/examples/issue327_counters_v3.rs index 9f4f5d4db6..acc2035100 100644 --- a/engine/crates/layerfs-persistence/examples/issue327_counters_v3.rs +++ b/engine/crates/layerfs-persistence/examples/issue327_counters_v3.rs @@ -36,6 +36,7 @@ fn fixture() -> Vec> { }) .collect() } +#[cfg(unix)] fn main() { assert_eq!( std::env::var("SCREEN_SCENARIO").as_deref(), @@ -179,3 +180,8 @@ fn main() { }; writeln!(output,"{{\"sql_statements\":{},\"sql_vm_steps\":{},\"sqlite\":\"{}\",\"operation_ns\":{operation},\"complete_ns\":{},\"file_bytes\":{},\"allocated_bytes\":{allocated},\"sealed_bytes\":{},\"objects_input\":{COUNT},\"input_bytes\":{},\"cache\":\"created-in-sample Store; fixed warm in-memory fixture\",\"status\":\"RECORDED\"}}",work.statements,work.vm_steps,sql_version,total.elapsed().as_nanos(),metadata.len(),sealed.bytes,COUNT*BYTES).unwrap(); } + +#[cfg(not(unix))] +fn main() { + eprintln!("this frozen recorder requires Unix (st_blocks); nothing recorded"); +} diff --git a/engine/crates/layerfs-persistence/examples/issue327_counters_v4.rs b/engine/crates/layerfs-persistence/examples/issue327_counters_v4.rs index 9a4b1371cd..4a9ed74050 100644 --- a/engine/crates/layerfs-persistence/examples/issue327_counters_v4.rs +++ b/engine/crates/layerfs-persistence/examples/issue327_counters_v4.rs @@ -36,6 +36,7 @@ fn fixture() -> Vec> { }) .collect() } +#[cfg(unix)] fn main() { assert_eq!( std::env::var("SCREEN_SCENARIO").as_deref(), @@ -179,3 +180,8 @@ fn main() { }; writeln!(output,"{{\"sql_statement_ns\":{},\"sql_statements\":{},\"sql_vm_steps\":{},\"sqlite\":\"{}\",\"operation_ns\":{operation},\"complete_ns\":{},\"file_bytes\":{},\"allocated_bytes\":{allocated},\"sealed_bytes\":{},\"objects_input\":{COUNT},\"input_bytes\":{},\"cache\":\"created-in-sample Store; fixed warm in-memory fixture\",\"status\":\"RECORDED\"}}",work.statement_ns,work.statements,work.vm_steps,sql_version,total.elapsed().as_nanos(),metadata.len(),sealed.bytes,COUNT*BYTES).unwrap(); } + +#[cfg(not(unix))] +fn main() { + eprintln!("this frozen recorder requires Unix (st_blocks); nothing recorded"); +} diff --git a/engine/crates/layerfs-persistence/examples/issue327_counters_v5.rs b/engine/crates/layerfs-persistence/examples/issue327_counters_v5.rs index 73b3479042..b770decbdf 100644 --- a/engine/crates/layerfs-persistence/examples/issue327_counters_v5.rs +++ b/engine/crates/layerfs-persistence/examples/issue327_counters_v5.rs @@ -36,6 +36,7 @@ fn fixture() -> Vec> { }) .collect() } +#[cfg(unix)] fn main() { assert_eq!( std::env::var("SCREEN_SCENARIO").as_deref(), @@ -179,3 +180,8 @@ fn main() { }; writeln!(output,"{{\"sql_phases_ns\":{:?},\"sql_statement_ns\":{},\"sql_statements\":{},\"sql_vm_steps\":{},\"sqlite\":\"{}\",\"operation_ns\":{operation},\"complete_ns\":{},\"file_bytes\":{},\"allocated_bytes\":{allocated},\"sealed_bytes\":{},\"objects_input\":{COUNT},\"input_bytes\":{},\"cache\":\"created-in-sample Store; fixed warm in-memory fixture\",\"status\":\"RECORDED\"}}",work.statement_phases.map(|p| p.wall_ns),work.statement_ns,work.statements,work.vm_steps,sql_version,total.elapsed().as_nanos(),metadata.len(),sealed.bytes,COUNT*BYTES).unwrap(); } + +#[cfg(not(unix))] +fn main() { + eprintln!("this frozen recorder requires Unix (st_blocks); nothing recorded"); +} diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/connection.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/connection.rs index f06d1e573d..1934f4c301 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/connection.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/connection.rs @@ -154,7 +154,11 @@ impl Session { creation_layout: crate::SqlitePackLayout, creation_acquisition: crate::SqliteAcquisitionSchema, ) -> Result { - if !cfg!(any(target_os = "macos", target_os = "linux")) { + if !cfg!(any( + target_os = "macos", + target_os = "linux", + target_os = "windows" + )) { return Err(BackendError::Integrity); } let flags = if writable { @@ -162,7 +166,19 @@ impl Session { } else { OpenFlags::SQLITE_OPEN_READ_ONLY } | OpenFlags::SQLITE_OPEN_NO_MUTEX; - let connection = Connection::open_with_flags(path, flags).map_err(rows::error)?; + // SQLite itself must also see the verbatim spelling: the plain one + // fails beyond MAX_PATH under a host without longPathAware, which + // would make a deep Store installable but unopenable. + #[cfg(target_os = "windows")] + let sqlite_path: std::path::PathBuf = { + use std::os::windows::ffi::OsStringExt; + let mut wide = super::windows_lane::verbatim_wide(path); + wide.pop(); // drop the NUL for the OsString form + std::path::PathBuf::from(std::ffi::OsString::from_wide(&wide)) + }; + #[cfg(not(target_os = "windows"))] + let sqlite_path: std::path::PathBuf = path.to_path_buf(); + let connection = Connection::open_with_flags(sqlite_path, flags).map_err(rows::error)?; connection.set_prepared_statement_cache_capacity(64); connection .busy_timeout(std::time::Duration::ZERO) @@ -171,6 +187,17 @@ impl Session { connection .set_db_config(rusqlite::config::DbConfig::SQLITE_DBCONFIG_DEFENSIVE, true) .map_err(rows::error)?; + // The profile pragmas are this open's first database-touching + // statements, where a WAL Store's -shm is created and recovery + // runs; two concurrent opens of one Store can meet Busy exactly + // there under the no-retry rule. The coarse writer lane replaces + // Busy: hold it for this one short setup step and release before + // the reads that follow. macOS keeps exact SQLite Busy by design. + let writer = super::writer_admission::Writer::acquire(path)?; + let connected = (|| -> Result< + (ConnectionProfile, crate::SqlitePackLayout, crate::SqliteAcquisitionSchema), + BackendError, + > { profile::apply(&connection, create, selected, creation_acquisition, &work)?; let (layout, acquisition) = if create { (creation_layout, creation_acquisition) @@ -244,6 +271,10 @@ impl Session { .map_err(rows::error)? as usize, }; profile::check(&profile)?; + Ok((profile, layout, acquisition)) + })(); + writer.release().map_err(|_| BackendError::Integrity)?; + let (profile, layout, acquisition) = connected?; super::thread_work::add(&work.borrow()); Ok(Self { admission: super::admission::Admission::default(), diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/mod.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/mod.rs index fce18a22d7..60ea28f410 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/mod.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/mod.rs @@ -7,6 +7,10 @@ pub(crate) mod connection; mod file_control; #[cfg(any(target_os = "macos", target_os = "linux"))] pub(crate) mod headroom; +#[cfg(target_os = "windows")] +pub mod windows_lane; +#[cfg(target_os = "windows")] +pub use windows_lane::windows_volume_filesystem; pub(crate) mod lane; pub(crate) mod metadata_allocation; pub(crate) mod metadata_locations; @@ -32,4 +36,4 @@ pub(crate) mod unit_io; pub(crate) mod unit_layout; mod units_publish; mod units_read; -mod writer_admission; +pub(crate) mod writer_admission; diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/seal.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/seal.rs index 3b45bffbd3..04c1d7e341 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/seal.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/seal.rs @@ -15,7 +15,7 @@ impl Session { if state.quarantined || !state.connection.is_autocommit() { return Err(PersistenceError::Uncertain); } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "windows"))] let state = { let mut state = state; state.writer_hold = Some(super::writer_admission::Writer::acquire(&self.path)?); diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/windows_lane.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/windows_lane.rs new file mode 100644 index 0000000000..6601becd87 --- /dev/null +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/windows_lane.rs @@ -0,0 +1,278 @@ +//! The Windows Store's one audited FFI surface. +//! +//! This is the Windows companion of the macOS seal file-control module: one +//! owner-authorized module of raw Win32 calls, allowed by the boundary guard +//! and compiled on Windows only; no other module in this crate may use +//! unsafe Rust. The writer lane is one named kernel mutex per Store +//! directory: the kernel releases it when a holder process exits (the next +//! waiter wakes on the abandoned mutex), which matches the Linux directory +//! flock's exit release, and it never opens a second descriptor to SQLite's +//! main/WAL/SHM files. The handle is held as an integer so `Writer` stays +//! plain data. The volume probe reports the filesystem of the directory's +//! local volume for the kernel guard's allowlist. +#![allow(unsafe_code)] +use sha2::{Digest, Sha256}; +use std::io; +use std::path::Path; +use windows_sys::Win32::Foundation::{CloseHandle, WAIT_ABANDONED, WAIT_OBJECT_0}; +use windows_sys::Win32::System::Threading::{ + CreateMutexW, ReleaseMutex, WaitForSingleObject, INFINITE, +}; + +/// The Store directory's cooperative writer lane. +pub(crate) struct LaneMutex { + handle: usize, + closed: bool, +} + +/// The lane's kernel object name: SHA-256 of the resolved directory's +/// encoded bytes, truncated to 64 bits. A versioned-stable digest, not +/// `DefaultHasher`, whose algorithm may change between Rust releases and +/// silently split one directory onto two lanes across builds (K3 review F1). +/// The `Global\` namespace keeps one lane per directory across login +/// sessions (a session-0 scheduled task and the user session share one +/// Store file; `Local\` would give each its own lane and two writers — +/// round-3 A2 F4). A mutant needs no create-global privilege. +pub(crate) fn lane_name(directory: &Path) -> String { + let resolved = std::fs::canonicalize(directory).unwrap_or_else(|_| directory.to_path_buf()); + let digest = Sha256::digest(resolved.as_os_str().as_encoded_bytes()); + let mut tag = [0u8; 8]; + tag.copy_from_slice(&digest[..8]); + format!( + "Global\\layerfs-writer-lane-{:016x}", + u64::from_be_bytes(tag) + ) +} + +impl LaneMutex { + /// One blocking kernel wait on the directory's named mutex, taken after + /// the Session's FIFO turn, so at most one thread per Session waits here. + /// Creation only opens the shared kernel object. Exits: the lane (a + /// holder process that exits releases it); a failed wait. + pub(crate) fn acquire(directory: &Path) -> Result { + let name: Vec = lane_name(directory) + .encode_utf16() + .chain(std::iter::once(0)) + .collect(); + let handle = unsafe { CreateMutexW(std::ptr::null(), 0, name.as_ptr()) }; + if handle.is_null() { + return Err(()); + } + let waited = unsafe { WaitForSingleObject(handle, INFINITE) }; + if waited == WAIT_OBJECT_0 || waited == WAIT_ABANDONED { + Ok(Self { + handle: handle as usize, + closed: false, + }) + } else { + unsafe { CloseHandle(handle) }; + Err(()) + } + } + + /// Releases the lane and destroys this handle; a failed release returns + /// the retained lane unchanged, so the quarantining caller keeps holding + /// it, and its eventual drop retries the release after SQLite teardown. + pub(crate) fn release(mut self) -> Result<(), Self> { + let handle = self.handle as windows_sys::Win32::Foundation::HANDLE; + if unsafe { ReleaseMutex(handle) } == 0 { + return Err(Self { + handle: self.handle, + closed: false, + }); + } + self.closed = true; + unsafe { CloseHandle(handle) }; + Ok(()) + } +} + +impl Drop for LaneMutex { + /// Best-effort release of a lane an error or panic path left unreleased + /// (K3-R3 F6): the release can fail on a thread that does not own the + /// mutex — the owner exits into the kernel's abandoned-mutex handoff — + /// while the descriptor closes exactly once either way. Linux gets this + /// for free from `Flock`'s own drop. + fn drop(&mut self) { + if self.closed { + return; + } + let handle = self.handle as windows_sys::Win32::Foundation::HANDLE; + self.closed = true; + unsafe { ReleaseMutex(handle) }; + unsafe { CloseHandle(handle) }; + } +} + +/// A verbatim (`\\?\`-prefixed) wide spelling of an absolute path, NUL +/// terminated, so the Win32 calls in the audited modules accept trees +/// deeper than MAX_PATH — a plain spelling can fail at 260 characters on +/// hosts without the long-path relaxation (the hosting node process +/// typically carries no `longPathAware` manifest). Already-verbatim +/// paths pass through unchanged; UNC shares become `\\?\UNC\...`; +/// relative paths pass through unchanged (every call site is absolute). +/// Forward slashes become backslashes in the spelling this function +/// builds; an already-verbatim input is passed through byte-for-byte — +/// Win32 does not normalize under the prefix, so a verbatim input +/// carrying forward slashes fails cleanly at its first use. +pub fn verbatim_wide(path: &Path) -> Vec { + use std::os::windows::ffi::OsStrExt; + let mut units: Vec = path.as_os_str().encode_wide().collect(); + const PREFIX: [u16; 4] = [92, 92, 63, 92]; // \ ? \ + if units.len() >= 4 && units[..4] == PREFIX { + // Already verbatim. + } else if matches!( + (units.first(), units.get(1)), + (Some(&92), Some(&92)) | (Some(&47), Some(&47)) + ) { + // \\server\share (or //server/share) → \\?\UNC\server\share — the + // separator after UNC is part of the spelling; without it the + // server name mis-joins. + let mut prefixed = Vec::with_capacity(units.len() + 7); + prefixed.extend_from_slice(&PREFIX); + prefixed.extend([85, 78, 67, 92]); // U N C \ + prefixed.extend( + units[2..] + .iter() + .map(|&unit| if unit == 47 { 92 } else { unit }), + ); + units = prefixed; + } else if path.is_absolute() { + let mut prefixed = Vec::with_capacity(units.len() + 4); + prefixed.extend_from_slice(&PREFIX); + prefixed.extend(units.iter().map(|&unit| if unit == 47 { 92 } else { unit })); + units = prefixed; + } + units.push(0); + units +} + +/// The filesystem name of the volume actually holding `directory`, for the +/// kernel guard's allowlist. The directory's own handle answers, and its +/// final path decides locality: the spelling is irrelevant (a verbatim +/// local path, a junction or a mapped drive all resolve), and a final path +/// that lands on a share (`\\?\UNC\...`) is refused — its locks travel the +/// SMB redirector, not this kernel's lock manager (K3-R3 F4). +pub fn windows_volume_filesystem(directory: &Path) -> io::Result { + use windows_sys::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, GetFinalPathNameByHandleW, GetVolumeInformationByHandleW, + FILE_FLAG_BACKUP_SEMANTICS, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, + OPEN_EXISTING, VOLUME_NAME_DOS, + }; + let wide = verbatim_wide(directory); + let handle = unsafe { + CreateFileW( + wide.as_ptr(), + 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + std::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + std::ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!( + "the directory {} cannot be opened to examine its volume: {}", + directory.display(), + io::Error::last_os_error() + ), + )); + } + let mut final_path: Vec = vec![0u16; 1024]; + let mut outgrew = false; + let mut written = unsafe { + GetFinalPathNameByHandleW( + handle, + final_path.as_mut_ptr(), + final_path.len() as u32, + VOLUME_NAME_DOS, + ) + }; + // An insufficient buffer returns the required length: grow for real and + // retry once. The old fixed-array fallback could never fit a longer + // requirement and fell back to a zeroed buffer, on which the `\\?\UNC\` + // refusal silently passed and a deep share was admitted as NTFS + // (round-6 D1 F1). + if written != 0 && written as usize >= final_path.len() { + final_path = vec![0u16; written as usize + 1]; + written = unsafe { + GetFinalPathNameByHandleW( + handle, + final_path.as_mut_ptr(), + final_path.len() as u32, + VOLUME_NAME_DOS, + ) + }; + // The requirement grew between the two calls (a concurrent rename + // lengthening the path): a second short read leaves the buffer + // unfilled and the UNC check would read garbage — refuse instead + // (round-8 F1 F2). + if written != 0 && written as usize >= final_path.len() { + outgrew = true; + } + } + // Capture the failure before another syscall can overwrite it (D1 F6). + let final_failure = (written == 0).then(io::Error::last_os_error).or_else(|| { + outgrew.then(|| { + io::Error::new( + io::ErrorKind::Unsupported, + "the final path outgrew its regrown buffer (a concurrent rename?)", + ) + }) + }); + let share = written != 0 + && final_path + .iter() + .take(8) + .zip(r"\\?\UNC\".encode_utf16()) + .all(|(unit, expected)| *unit == expected); + let mut filesystem = [0u16; 16]; + let examined = unsafe { + GetVolumeInformationByHandleW( + handle, + std::ptr::null_mut(), + 0, + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null_mut(), + filesystem.as_mut_ptr(), + filesystem.len() as u32, + ) + }; + // Capture the failure before another syscall can overwrite it. + let failure = (examined == 0).then(io::Error::last_os_error); + unsafe { CloseHandle(handle) }; + if let Some(error) = final_failure { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!( + "the final path of {} cannot be read: {error}", + directory.display() + ), + )); + } + if share { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!( + "the directory {} resolves onto a network share; the Store needs a local volume", + directory.display() + ), + )); + } + if let Some(error) = failure { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!( + "the volume of {} cannot be examined: {error}", + directory.display() + ), + )); + } + let end = filesystem.iter().position(|&wide| wide == 0).unwrap_or(0); + Ok(String::from_utf16_lossy(&filesystem[..end])) +} diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/writer_admission.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/writer_admission.rs index c7d42a3f82..9783af97f0 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/writer_admission.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/writer_admission.rs @@ -13,10 +13,27 @@ use { pub(crate) struct Writer { #[cfg(target_os = "linux")] lock: Flock, + #[cfg(target_os = "windows")] + lane: super::windows_lane::LaneMutex, } impl Writer { + /// Takes the cross-process writer lane. The blocking wait — another + /// process or session holding the lane — is a Store-writer wait like + /// the admission queue: the phase that waited reports it as + /// `wait.writer`, so an open or a publication that met a held lane is + /// visible in timing coverage instead of vanishing into the lock. pub(crate) fn acquire(path: &Path) -> Result { + let started = std::time::Instant::now(); + let writer = Self::acquire_lane(path)?; + layerfs_telemetry::timer::waits::add( + layerfs_telemetry::timer::waits::Wait::Writer, + started.elapsed(), + ); + Ok(writer) + } + + fn acquire_lane(path: &Path) -> Result { #[cfg(target_os = "linux")] { // Never open another FD to SQLite's main/WAL/SHM files here: @@ -54,7 +71,28 @@ impl Writer { let _ = path; Ok(Self {}) } - #[cfg(not(any(target_os = "linux", target_os = "macos")))] + #[cfg(target_os = "windows")] + { + // Windows has no flock on a directory handle. One named kernel + // mutex per Store directory is the equivalent coarse lane: it + // never opens another descriptor to SQLite's main/WAL/SHM files, + // and a holder process that exits releases it. Multiple Store + // files in one directory share this coarse lane, as on Linux. + let directory = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + if !std::fs::metadata(directory) + .map(|metadata| metadata.is_dir()) + .unwrap_or(false) + { + return Err(BackendError::Integrity); + } + let lane = super::windows_lane::LaneMutex::acquire(directory) + .map_err(|()| BackendError::Integrity)?; + Ok(Self { lane }) + } + #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] { let _ = path; Err(BackendError::Integrity) @@ -72,10 +110,14 @@ impl Writer { Err((lock, _)) => Err(Self { lock }), } } - #[cfg(not(target_os = "linux"))] + #[cfg(target_os = "macos")] { Ok(()) } + #[cfg(target_os = "windows")] + { + self.lane.release().map_err(|lane| Self { lane }) + } } } diff --git a/engine/crates/layerfs-persistence/src/lib.rs b/engine/crates/layerfs-persistence/src/lib.rs index 9a2553b4e7..92694ad248 100644 --- a/engine/crates/layerfs-persistence/src/lib.rs +++ b/engine/crates/layerfs-persistence/src/lib.rs @@ -13,11 +13,16 @@ pub use backend::sqlite::headroom::STORE_HEADROOM_BYTES; pub use backend::sqlite::reclamation::{SpaceReclamation, RECLAMATION_PAGE_LIMIT}; pub use backend::sqlite::statement_work::StatementPhaseWork; pub use backend::sqlite::thread_work::thread_work; +#[cfg(target_os = "windows")] +pub use backend::sqlite::windows_lane::verbatim_wide; +#[cfg(target_os = "windows")] +pub use backend::sqlite::windows_volume_filesystem; pub use config::{ BackendSelection, PersistenceConfig, SqliteAcquisitionSchema, SqlitePackLayout, SqlitePersistenceProfile, }; pub use handles::Handles; + pub use history::HistoryProvider; pub use storage::acquisition::AcquisitionProvider; pub use storage::merge_records::{ @@ -26,6 +31,7 @@ pub use storage::merge_records::{ pub(crate) use storage::provider as storage_provider; pub(crate) use storage::publication; pub use storage_provider::StorageProvider; +pub use store::lane::{store_lane, StoreLane}; pub use store::seal::SealedStore; mod metadata; diff --git a/engine/crates/layerfs-persistence/src/store/handles.rs b/engine/crates/layerfs-persistence/src/store/handles.rs index 5a667a288e..00f20b3d9b 100644 --- a/engine/crates/layerfs-persistence/src/store/handles.rs +++ b/engine/crates/layerfs-persistence/src/store/handles.rs @@ -35,6 +35,14 @@ impl Handles { pub fn close(&self) { self.storage.session.admission.close(); } + /// Drops this session under the writer lane, so its connection's + /// close-time checkpoint never runs beside another opener's first + /// attach. A failed lane acquisition closes bare. + pub fn shutdown(self) { + let lane = crate::store_lane(&self.storage.session.path); + drop(self); + drop(lane); + } /// Read-back settings from the actual shared connection. pub fn profile(&self) -> &ConnectionProfile { &self.storage.session.profile diff --git a/engine/crates/layerfs-persistence/src/store/lane.rs b/engine/crates/layerfs-persistence/src/store/lane.rs new file mode 100644 index 0000000000..b19b720f48 --- /dev/null +++ b/engine/crates/layerfs-persistence/src/store/lane.rs @@ -0,0 +1,21 @@ +//! The Store directory's writer lane, held for a coordinated close. +use std::path::Path; + +/// Held for a coordinated close: the connection teardown's close-time +/// checkpoint runs inside it, never beside another opener's first attach. +/// Best-effort - a failed acquisition closes bare, as an uncoordinated drop +/// always has. +pub struct StoreLane(Option); + +impl Drop for StoreLane { + fn drop(&mut self) { + if let Some(writer) = self.0.take() { + let _ = writer.release(); + } + } +} + +/// Holds the writer lane of the Store directory holding `store`. +pub fn store_lane(store: &Path) -> StoreLane { + StoreLane(crate::backend::sqlite::writer_admission::Writer::acquire(store).ok()) +} diff --git a/engine/crates/layerfs-persistence/src/store/mod.rs b/engine/crates/layerfs-persistence/src/store/mod.rs index 73d521202f..6805a0f23a 100644 --- a/engine/crates/layerfs-persistence/src/store/mod.rs +++ b/engine/crates/layerfs-persistence/src/store/mod.rs @@ -1,5 +1,6 @@ //! Responsibility-scoped implementation modules and reexports. pub(crate) mod config; pub(crate) mod handles; +pub mod lane; pub(crate) mod open; pub(crate) mod seal; diff --git a/engine/crates/layerfs-persistence/src/store/open.rs b/engine/crates/layerfs-persistence/src/store/open.rs index 2cb9e98273..753aa88f84 100644 --- a/engine/crates/layerfs-persistence/src/store/open.rs +++ b/engine/crates/layerfs-persistence/src/store/open.rs @@ -18,7 +18,11 @@ impl Handles { history: &HistoryCatalogConfig, ) -> Result { if config.backend != BackendSelection::Sqlite - || !cfg!(any(target_os = "macos", target_os = "linux")) + || !cfg!(any( + target_os = "macos", + target_os = "linux", + target_os = "windows" + )) { return Err(PersistenceError::BackendUnavailable); } @@ -70,7 +74,11 @@ impl Handles { writable: bool, ) -> Result { if config.backend != BackendSelection::Sqlite - || !cfg!(any(target_os = "macos", target_os = "linux")) + || !cfg!(any( + target_os = "macos", + target_os = "linux", + target_os = "windows" + )) { return Err(PersistenceError::BackendUnavailable); } @@ -84,13 +92,29 @@ impl Handles { config.sqlite_acquisition, )?); if writable { - session.add_binding_hints()?; - session.add_lineage(crate::history::backfill_lineage)?; - session.add_merge_scratch()?; - session.add_file_digests()?; + // The setup statements are write transactions that two + // concurrent opens of one Store can interleave under the + // no-retry rule, so the whole block runs inside one held + // publication lane: its writer lock serializes the opens + // cross-process, and the block's own statements enter as + // nested turns, which do not take the lock a second time - + // a raw second hold here would self-deadlock on Linux, where + // flock is held per open file description. + let lane = session.hold_lane()?; + let setup = (|| -> Result<(), crate::backend::records::BackendError> { + session.add_binding_hints()?; + session.add_lineage(crate::history::backfill_lineage)?; + session.add_merge_scratch()?; + session.add_file_digests() + })(); + drop(lane); + setup?; } let handles = Self::validated(session, directory, binding, cursor_key)?; if writable { + // The sweep runs as one write transaction whose admission takes + // the writer lane itself; the setup block's held lane above is + // released by then. crate::storage::merge_records::sweep(&handles.storage.session)?; } Ok(handles) diff --git a/engine/crates/layerfs-persistence/tests/serverless_store.rs b/engine/crates/layerfs-persistence/tests/serverless_store.rs index 91d10c5473..ac4a8861eb 100644 --- a/engine/crates/layerfs-persistence/tests/serverless_store.rs +++ b/engine/crates/layerfs-persistence/tests/serverless_store.rs @@ -1,4 +1,7 @@ //! Disposable serverless Store proofs through the real public provider. +//! Unix only: the child's readiness signal travels a unix-domain socket. +//! The Windows cross-process proof is tests/windows_store_native.rs. +#![cfg(unix)] mod support; use layerfs_content::{FinalizedObject, ObjectId, ObjectRole}; use layerfs_history::{HistoryCatalog, HistoryCatalogConfig, HistoryError, ReserveRequest}; diff --git a/engine/crates/layerfs-persistence/tests/support/mod.rs b/engine/crates/layerfs-persistence/tests/support/mod.rs index 86b851581a..419a11953b 100644 --- a/engine/crates/layerfs-persistence/tests/support/mod.rs +++ b/engine/crates/layerfs-persistence/tests/support/mod.rs @@ -20,9 +20,13 @@ impl Temp { pub fn new(name: &str) -> Self { static NEXT: AtomicU64 = AtomicU64::new(0); let path = std::env::temp_dir().join(format!( - "layerfs-history-{}-{}-{}", + "layerfs-history-{}-{}-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|span| span.as_nanos()) + .unwrap_or(0), name )); std::fs::create_dir_all(&path).expect("temporary directory"); diff --git a/engine/crates/layerfs-persistence/tests/verbatim_wide_windows.rs b/engine/crates/layerfs-persistence/tests/verbatim_wide_windows.rs new file mode 100644 index 0000000000..4a0f763da1 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/verbatim_wide_windows.rs @@ -0,0 +1,41 @@ +//! The verbatim wide spelling the audited Win32 calls use: prefix forms, +//! the UNC separator, slash normalization and passthrough cases. +#![cfg(target_os = "windows")] +use layerfs_persistence::verbatim_wide; +use std::path::Path; + +fn text(path: &str) -> String { + let wide = verbatim_wide(Path::new(path)); + assert_eq!(*wide.last().unwrap(), 0, "NUL terminated"); + String::from_utf16(&wide[..wide.len() - 1]).unwrap() +} + +#[test] +fn absolute_paths_gain_the_verbatim_prefix() { + assert_eq!(text(r"D:\dir\store.sqlite"), r"\\?\D:\dir\store.sqlite"); +} + +#[test] +fn forward_slashes_normalize_under_the_prefix() { + assert_eq!(text("D:/dir/store.sqlite"), r"\\?\D:\dir\store.sqlite"); +} + +#[test] +fn unc_paths_gain_the_unc_form_with_its_separator() { + // The separator after UNC is part of the spelling: without it the + // server name mis-joins (\\?\UNCserver instead of \\?\UNC\server). + assert_eq!( + text(r"\\server\share\store.sqlite"), + r"\\?\UNC\server\share\store.sqlite" + ); + assert_eq!( + text("//server/share/store.sqlite"), + r"\\?\UNC\server\share\store.sqlite" + ); +} + +#[test] +fn already_verbatim_and_relative_paths_pass_through() { + assert_eq!(text(r"\\?\D:\dir\store.sqlite"), r"\\?\D:\dir\store.sqlite"); + assert_eq!(text("relative/store.sqlite"), "relative/store.sqlite"); +} diff --git a/engine/crates/layerfs-persistence/tests/windows_lane_abandoned.rs b/engine/crates/layerfs-persistence/tests/windows_lane_abandoned.rs new file mode 100644 index 0000000000..b4d60c1924 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/windows_lane_abandoned.rs @@ -0,0 +1,80 @@ +//! The Windows lane's dead-holder path, in its own binary so it never runs +//! beside the foreign-process test's re-exec'd children. +#![cfg(windows)] +mod support; +use layerfs_history::HistoryCatalogConfig; +use layerfs_persistence::{Handles, PersistenceConfig, SqlitePersistenceProfile}; +use layerfs_storage::{port::*, StoragePolicy}; +use std::path::Path; + +fn config(path: &Path) -> PersistenceConfig { + PersistenceConfig::sqlite(path).with_sqlite_profile(SqlitePersistenceProfile::Disposable) +} + +fn reserve() -> Reserve { + Reserve { + packs: 1, + ordinals: 1, + } +} + +fn authority(name: &[u8]) -> HistoryCatalogConfig { + HistoryCatalogConfig { + binding_key: name.to_vec(), + cursor_key: [67; 32], + incarnation: 1, + } +} + +/// The lane's dead-holder path: a thread that ends while owning the named +/// mutex leaves it abandoned, and the kernel hands ownership to the next +/// waiter. The abandoned state is produced deterministically here - a +/// thread waits on the product's own mutex name and returns without +/// releasing - and the Store must still open and write through that +/// `WAIT_ABANDONED` acquisition. +#[test] +fn a_dead_lane_holder_does_not_block_the_next_writer() { + use sha2::{Digest, Sha256}; + let temp = support::Temp::new("windows-store-abandoned"); + let path = temp.join("store.sqlite"); + let catalog = authority(b"windows-store-abandoned"); + let seed = Handles::create(config(&path), StoragePolicy::frozen_default(), &catalog).unwrap(); + seed.storage.reserve(reserve()).unwrap(); + drop(seed); + + let directory = temp.path().to_path_buf(); + let holder = std::thread::spawn(move || { + // The product's own lane name: SHA-256 of the resolved directory's + // encoded bytes, truncated to 64 bits (windows_lane::lane_name). + let resolved = std::fs::canonicalize(&directory).unwrap_or_else(|_| directory.clone()); + let digest = Sha256::digest(resolved.as_os_str().as_encoded_bytes()); + let mut tag = [0u8; 8]; + tag.copy_from_slice(&digest[..8]); + let name: Vec = format!( + "Global\\layerfs-writer-lane-{:016x}", + u64::from_be_bytes(tag) + ) + .encode_utf16() + .chain(std::iter::once(0)) + .collect(); + let handle = unsafe { + windows_sys::Win32::System::Threading::CreateMutexW(std::ptr::null(), 0, name.as_ptr()) + }; + assert!(!handle.is_null()); + let waited = unsafe { + windows_sys::Win32::System::Threading::WaitForSingleObject( + handle, + windows_sys::Win32::System::Threading::INFINITE, + ) + }; + assert_eq!(waited, windows_sys::Win32::Foundation::WAIT_OBJECT_0); + // The thread owns the mutex and now ends without ReleaseMutex: the + // kernel marks it abandoned for the next waiter. + }); + holder.join().unwrap(); + + let reopened = Handles::open_writable(config(&path), &catalog.binding_key, catalog.cursor_key) + .expect("an abandoned lane is taken by the next writer"); + reopened.storage.reserve(reserve()).unwrap(); + reopened.shutdown(); +} diff --git a/engine/crates/layerfs-persistence/tests/windows_store_native.rs b/engine/crates/layerfs-persistence/tests/windows_store_native.rs new file mode 100644 index 0000000000..ae1b3df97d --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/windows_store_native.rs @@ -0,0 +1,146 @@ +//! The Windows-native Store route: creation and two concurrent sessions in +//! one process, then two foreign processes serialized through the writer +//! lane, all through public Stores and bounded waits. +#![cfg(windows)] +mod support; +use layerfs_history::HistoryCatalogConfig; +use layerfs_persistence::{Handles, PersistenceConfig, SqlitePersistenceProfile}; +use layerfs_storage::{port::*, StoragePolicy}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +const CHILD_STORE: &str = "LAYERFS_WINDOWS_STORE_CHILD_STORE"; +const CHILD_BINDING: &str = "LAYERFS_WINDOWS_STORE_CHILD_BINDING"; + +fn config(path: &Path) -> PersistenceConfig { + PersistenceConfig::sqlite(path).with_sqlite_profile(SqlitePersistenceProfile::Disposable) +} + +fn reserve() -> Reserve { + Reserve { + packs: 1, + ordinals: 1, + } +} + +fn authority(name: &[u8]) -> HistoryCatalogConfig { + HistoryCatalogConfig { + binding_key: name.to_vec(), + cursor_key: [67; 32], + incarnation: 1, + } +} + +/// One child process's whole work: open the Store writable and write once. +fn child(path: PathBuf, binding: Vec) { + let opened = Handles::open_writable(config(&path), &binding, [67; 32]) + .expect("the child opens the Store"); + opened.storage.reserve(reserve()).expect("the child writes"); + // A clean close: the child's Store is not crashed, so the sibling's + // next open does not pay WAL recovery beside the lane under test. + opened.shutdown(); +} + +/// When this binary was re-executed as a lane child, run its one job and +/// leave; otherwise this is the parent test run and work continues below. +fn dispatch_child() { + let Some(path) = std::env::var_os(CHILD_STORE) else { + return; + }; + let binding = std::env::var(CHILD_BINDING) + .expect("the child binding") + .into_bytes(); + child(PathBuf::from(path), binding); + std::process::exit(0); +} + +/// Bounded completion of one foreign child; a hung child fails the test. +/// A failed child's own report (the harness prints it to its stdout) is +/// carried into the failure instead of being discarded. +fn finish(mut child: std::process::Child) { + let until = Instant::now() + Duration::from_secs(60); + loop { + match child.try_wait().expect("child poll") { + Some(status) => { + assert!( + status.success(), + "a child writer failed: {status}; child report: {:?}", + String::from_utf8_lossy(&child.wait_with_output().expect("child drain").stdout) + ); + return; + } + None => assert!(Instant::now() < until, "a child writer is hung"), + } + std::thread::sleep(Duration::from_millis(10)); + } +} + +#[test] +fn windows_native_store_serves_sessions_and_serializes_foreign_processes() { + dispatch_child(); + let temp = support::Temp::new("windows-store-native"); + let path = temp.join("store.sqlite"); + let catalog = authority(b"windows-store-native"); + let first = Handles::create(config(&path), StoragePolicy::frozen_default(), &catalog).unwrap(); + let second = + Handles::open_writable(config(&path), &catalog.binding_key, catalog.cursor_key).unwrap(); + first.storage.reserve(reserve()).unwrap(); + second.storage.reserve(reserve()).unwrap(); + first.storage.reserve(reserve()).unwrap(); + second.storage.reserve(reserve()).unwrap(); + drop(first); + drop(second); + + let lane = temp.join("lane"); + std::fs::create_dir_all(&lane).unwrap(); + let lane_path = lane.join("store.sqlite"); + let binding = b"windows-lane-processes".to_vec(); + let lane_catalog = authority(&binding); + let seed = Handles::create( + config(&lane_path), + StoragePolicy::frozen_default(), + &lane_catalog, + ) + .unwrap(); + seed.storage.reserve(reserve()).unwrap(); + drop(seed); + let executable = std::env::current_exe().unwrap(); + let mut children = Vec::new(); + for _ in 0..2 { + children.push( + Command::new(&executable) + // The child runs only this test; without the filter the + // harness would run every test in the binary (and their + // dispatch guards) beside it. + .args([ + "--exact", + "windows_native_store_serves_sessions_and_serializes_foreign_processes", + ]) + .env(CHILD_STORE, &lane_path) + .env(CHILD_BINDING, "windows-lane-processes") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(), + ); + } + for child in children { + finish(child); + } + let reopened = + Handles::open_writable(config(&lane_path), &binding, lane_catalog.cursor_key).unwrap(); + reopened.storage.reserve(reserve()).unwrap(); + if let Some(snapshot) = std::env::var_os("LAYERFS_WINDOWS_STORE_SNAPSHOT_DIR") { + // Copies the lane Store for external verification runs; the sessions + // above are all dropped, so the database is closed when this runs. + let snapshot = PathBuf::from(snapshot); + std::fs::create_dir_all(&snapshot).unwrap(); + std::fs::copy(&lane_path, snapshot.join("store.sqlite")).unwrap(); + for side in ["store.sqlite-wal", "store.sqlite-shm"] { + if let Ok(bytes) = std::fs::read(lane.with_file_name(side)) { + std::fs::write(snapshot.join(side), bytes).unwrap(); + } + } + } +} diff --git a/engine/crates/layerfs-project/Cargo.toml b/engine/crates/layerfs-project/Cargo.toml index b1311655b6..5487fc16fd 100644 --- a/engine/crates/layerfs-project/Cargo.toml +++ b/engine/crates/layerfs-project/Cargo.toml @@ -17,3 +17,9 @@ layerfs-bridge = { path = "../layerfs-bridge" } [dev-dependencies] sha2 = "=0.11.0" + +[target.'cfg(target_os = "windows")'.dependencies] +# The audited host-identity probe; CreateFileW is additionally gated on +# Win32_Security by its SECURITY_ATTRIBUTES parameter. windows-sys is +# already in the locked graph. +windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_Storage_FileSystem"] } diff --git a/engine/crates/layerfs-project/src/import/backing.rs b/engine/crates/layerfs-project/src/import/backing.rs index 0684a071dc..31906e98bb 100644 --- a/engine/crates/layerfs-project/src/import/backing.rs +++ b/engine/crates/layerfs-project/src/import/backing.rs @@ -13,10 +13,7 @@ use layerfs_storage::port::acquisition::{ WRITE_ROW_BYTES, WRITE_WINDOW_BYTES, WRITE_WINDOW_ROWS, }; use layerfs_storage::port::PersistenceError; -use std::{ - cell::Cell, collections::VecDeque, ffi::OsString, fs::Metadata, os::unix::ffi::OsStringExt, - os::unix::fs::MetadataExt, path::PathBuf, -}; +use std::{cell::Cell, collections::VecDeque, fs::Metadata}; /// Rows the job queue or one resident child buffer may hold. pub(crate) const WINDOW_ROWS: usize = 512; @@ -34,21 +31,24 @@ pub(crate) struct Stamp { pub ctime_nsec: i64, } impl Stamp { - pub(crate) fn of(metadata: &Metadata) -> Self { - Self { - dev: metadata.dev(), - ino: metadata.ino(), - len: metadata.len(), - mode: metadata.mode(), - mtime: metadata.mtime(), - mtime_nsec: metadata.mtime_nsec(), - ctime: metadata.ctime(), - ctime_nsec: metadata.ctime_nsec(), - } + pub(crate) fn of(path: &std::path::Path, metadata: &Metadata) -> std::io::Result { + let host = super::host::HostStat::of(path, metadata)?; + Ok(Self { + dev: host.dev, + ino: host.ino, + len: host.len, + mode: host.mode, + mtime: host.mtime, + mtime_nsec: host.mtime_nsec, + ctime: host.ctime, + ctime_nsec: host.ctime_nsec, + }) } /// True while `metadata` is still the same unchanged regular file. - pub(crate) fn matches_file(&self, metadata: &Metadata) -> bool { - metadata.is_file() && !metadata.file_type().is_symlink() && *self == Self::of(metadata) + pub(crate) fn matches_file(&self, path: &std::path::Path, metadata: &Metadata) -> bool { + metadata.is_file() + && !metadata.file_type().is_symlink() + && Self::of(path, metadata).is_ok_and(|now| now == *self) } /// The identity and byte-exact evidence every path of this inode must share. pub(crate) fn identity(&self) -> HostIdentity { @@ -377,7 +377,7 @@ pub(crate) fn window_full(rows: usize, held: usize, bytes: usize) -> bool { fn changed_source(error: AcquisitionError, path: impl Fn(u64) -> Option>) -> Failure { if let AcquisitionError::Changed { position } = error { if let Some(path) = path(position) { - let path = PathBuf::from(OsString::from_vec(path)); + let path = super::host::bytes_to_path(&path); return Failure::refused(SourceRefusal::Changed, path); } } diff --git a/engine/crates/layerfs-project/src/import/error.rs b/engine/crates/layerfs-project/src/import/error.rs index d2d75e8289..5d46b175c7 100644 --- a/engine/crates/layerfs-project/src/import/error.rs +++ b/engine/crates/layerfs-project/src/import/error.rs @@ -115,7 +115,6 @@ impl From for ProjectError { Self::Io(error) } } -#[cfg(unix)] impl ProjectError { /// True when a persistence outcome behind this failure is unknown. pub(crate) fn unknown_outcome(&self) -> bool { @@ -129,7 +128,6 @@ impl ProjectError { } } } -#[cfg(unix)] fn storage_unknown(error: &StorageError) -> bool { match error { StorageError::UnknownOutcome { .. } => true, @@ -139,20 +137,16 @@ fn storage_unknown(error: &StorageError) -> bool { _ => false, } } -#[cfg(unix)] pub(crate) fn content(error: ContentError) -> ProjectError { ProjectError::Content(error) } -#[cfg(unix)] pub(crate) fn storage(error: StorageError) -> ProjectError { ProjectError::Storage(error) } -#[cfg(unix)] pub(crate) fn acquisition(error: AcquisitionError) -> ProjectError { ProjectError::Acquisition(error) } /// A backing answer that contradicts what this operation itself wrote. -#[cfg(unix)] pub(crate) fn malformed() -> ProjectError { ProjectError::Acquisition(AcquisitionError::Persistence(PersistenceError::Malformed)) } diff --git a/engine/crates/layerfs-project/src/import/files.rs b/engine/crates/layerfs-project/src/import/files.rs index 956402b049..bbd6e2ca50 100644 --- a/engine/crates/layerfs-project/src/import/files.rs +++ b/engine/crates/layerfs-project/src/import/files.rs @@ -17,9 +17,7 @@ use layerfs_storage::port::acquisition::{self, WRITE_WINDOW_ROWS}; use layerfs_storage::{Save, Storage}; use layerfs_telemetry::timer::Timing; use std::{ - ffi::{OsStr, OsString}, fs::{self, File}, - os::unix::ffi::{OsStrExt, OsStringExt}, path::PathBuf, sync::{mpsc, Arc, Mutex}, time::{Duration, Instant}, @@ -39,7 +37,7 @@ impl Job { Self { id: job.position, stamp: Stamp::from_identity(&job.host), - path: PathBuf::from(OsString::from_vec(job.host_path)), + path: super::host::bytes_to_path(&job.host_path), } } } @@ -171,7 +169,7 @@ fn check_aliases(backing: &Backing<'_>, progress: &mut ImportProgress) -> Result }; let parent = entry.key.parent.ok_or_else(malformed)?; if directory.as_ref().is_none_or(|(held, _)| *held != parent) { - let path = OsString::from_vec(backing.directory_path(parent)?); + let path = super::host::bytes_to_os(&backing.directory_path(parent)?); directory = Some((parent, PathBuf::from(path))); } if first.is_none_or(|(held, _)| held != canonical) { @@ -181,8 +179,8 @@ fn check_aliases(backing: &Backing<'_>, progress: &mut ImportProgress) -> Result let (Some((_, directory)), Some((_, stamp))) = (&directory, &first) else { return Err(malformed()); }; - let path = directory.join(OsStr::from_bytes(&entry.key.name)); - if !stamp.matches_file(&fs::symlink_metadata(&path)?) { + let path = directory.join(super::host::bytes_to_os(&entry.key.name)); + if !stamp.matches_file(&path, &fs::symlink_metadata(&path)?) { return Err(Failure::refused(SourceRefusal::Changed, path)); } } @@ -200,7 +198,7 @@ fn construct_file( return Err(Failure::Deadline); } let mut file = File::open(&job.path)?; - if !job.stamp.matches_file(&file.metadata()?) { + if !job.stamp.matches_file(&job.path, &file.metadata()?) { return Err(Failure::refused(SourceRefusal::Changed, &job.path)); } let (result, _) = Timing::disabled("history.import_file", |scope| { @@ -214,8 +212,10 @@ fn construct_file( }); let built = result.map_err(content)?; if built.logical_len != job.stamp.len - || !job.stamp.matches_file(&file.metadata()?) - || !job.stamp.matches_file(&fs::symlink_metadata(&job.path)?) + || !job.stamp.matches_file(&job.path, &file.metadata()?) + || !job + .stamp + .matches_file(&job.path, &fs::symlink_metadata(&job.path)?) { return Err(Failure::refused(SourceRefusal::Changed, &job.path)); } diff --git a/engine/crates/layerfs-project/src/import/host.rs b/engine/crates/layerfs-project/src/import/host.rs new file mode 100644 index 0000000000..1ec1776118 --- /dev/null +++ b/engine/crates/layerfs-project/src/import/host.rs @@ -0,0 +1,115 @@ +//! Host byte encoding of paths and names, and the metadata identity Init +//! reads. Unix uses the raw bytes and the stat identity; Windows names +//! travel as lossless WTF-8 (an unpaired surrogate included, byte-exact +//! round trip through the codec and the audited probes), and the +//! identity is the entry's real volume serial number and file index, its +//! length, timestamps and a mode synthesized from the read-only attribute. +//! Hard-link membership is not detected on Windows (the format supports it; +//! each link acquires as its own file). +use std::ffi::{OsStr, OsString}; +use std::fs::Metadata; +use std::io; +use std::path::{Path, PathBuf}; + +#[cfg(windows)] +#[path = "host_wtf8_inner.rs"] +mod name_bytes; + +/// One host name's bytes. +pub(super) fn os_to_bytes(value: &OsStr) -> Vec { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + value.as_bytes().to_vec() + } + #[cfg(windows)] + { + name_bytes::encode(value) + } +} + +/// The host name of `value`'s bytes. +pub(super) fn bytes_to_os(value: &[u8]) -> OsString { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStringExt; + OsString::from_vec(value.to_vec()) + } + #[cfg(windows)] + { + name_bytes::decode(value) + } +} + +/// The host path of `value`'s bytes. +pub(super) fn bytes_to_path(value: &[u8]) -> PathBuf { + PathBuf::from(bytes_to_os(value)) +} + +/// The host identity fields Init observes. +#[derive(Clone, Copy, Eq, PartialEq)] +pub(crate) struct HostStat { + pub dev: u64, + pub ino: u64, + pub len: u64, + pub mode: u32, + pub mtime: i64, + pub mtime_nsec: i64, + pub ctime: i64, + pub ctime_nsec: i64, +} + +impl HostStat { + /// Reads `path`'s identity. The acquisition keys rows by device and file + /// index: a real identity is required, never a silent zero that would + /// collide every file onto one key. + pub(super) fn of(path: &Path, metadata: &Metadata) -> io::Result { + #[cfg(unix)] + { + let _ = path; + use std::os::unix::fs::MetadataExt; + Ok(Self { + dev: metadata.dev(), + ino: metadata.ino(), + len: metadata.len(), + mode: metadata.mode(), + mtime: metadata.mtime(), + mtime_nsec: metadata.mtime_nsec(), + ctime: metadata.ctime(), + ctime_nsec: metadata.ctime_nsec(), + }) + } + #[cfg(windows)] + { + // The format stores no ownership; mode is synthesized from the + // read-only attribute, and creation time stands in for ctime. + let base = if metadata.is_dir() { 0o555 } else { 0o444 }; + let mode = if metadata.permissions().readonly() { + base + } else { + base | 0o200 + }; + let (mtime, mtime_nsec) = match metadata.modified() { + Ok(time) => match time.duration_since(std::time::UNIX_EPOCH) { + Ok(span) => (span.as_secs() as i64, i64::from(span.subsec_nanos())), + Err(before) => ( + -(before.duration().as_secs() as i64), + -(i64::from(before.duration().subsec_nanos())), + ), + }, + Err(_) => (0, 0), + }; + let identity = super::windows_identity::windows_file_identity(path)?; + Ok(Self { + dev: identity.device, + ino: identity.index, + len: metadata.len(), + mode, + mtime, + mtime_nsec, + ctime: identity.created, + ctime_nsec: identity.created_nsec, + }) + } + } +} diff --git a/engine/crates/layerfs-project/src/import/host_wtf8_inner.rs b/engine/crates/layerfs-project/src/import/host_wtf8_inner.rs new file mode 100644 index 0000000000..57183fe018 --- /dev/null +++ b/engine/crates/layerfs-project/src/import/host_wtf8_inner.rs @@ -0,0 +1,100 @@ +// Lossless name bytes on Windows: the host name is UTF-16 code units, and +// a well-formed pair encodes as its supplementary scalar (the same UTF-8 +// bytes Linux stores for that name), while an unpaired surrogate keeps its +// three-byte WTF-8 form. The bytes decode back to the very same units. +// This file is the standard's encoding table and nothing else; the product +// (import/host.rs) and the name-roundtrip test each include it as a module. +use std::ffi::OsStr; +use std::ffi::OsString; +use std::os::windows::ffi::OsStrExt; +use std::os::windows::ffi::OsStringExt; + +// One host name's WTF-8 bytes. +pub fn encode(value: &OsStr) -> Vec { + let units: Vec = value.encode_wide().collect(); + let mut out = Vec::with_capacity(units.len() * 3); + let mut index = 0; + while index < units.len() { + let unit = units[index]; + if (0xD800..=0xDBFF).contains(&unit) { + if let Some(&low) = units.get(index + 1) { + if (0xDC00..=0xDFFF).contains(&low) { + let scalar = + 0x1_0000 + ((u32::from(unit) - 0xD800) << 10) + (u32::from(low) - 0xDC00); + out.push(0xF0 | (scalar >> 18) as u8); + out.push(0x80 | ((scalar >> 12) & 0x3F) as u8); + out.push(0x80 | ((scalar >> 6) & 0x3F) as u8); + out.push(0x80 | (scalar & 0x3F) as u8); + index += 2; + continue; + } + } + } + match unit { + 0x0000..=0x007F => out.push(unit as u8), + 0x0080..=0x07FF => { + out.push(0xC0 | (unit >> 6) as u8); + out.push(0x80 | (unit & 0x3F) as u8); + } + 0x0800..=0xFFFF => { + out.push(0xE0 | (unit >> 12) as u8); + out.push(0x80 | ((unit >> 6) & 0x3F) as u8); + out.push(0x80 | (unit & 0x3F) as u8); + } + } + index += 1; + } + out +} + +// The host name of WTF-8 `bytes`. +pub fn decode(bytes: &[u8]) -> OsString { + let mut units = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + let first = bytes[index]; + let (width, scalar) = if first < 0x80 { + (1, u32::from(first)) + } else if first & 0xE0 == 0xC0 { + ( + 2, + (u32::from(first & 0x1F) << 6) + | u32::from(bytes.get(index + 1).copied().unwrap_or(0) & 0x3F), + ) + } else if first & 0xF0 == 0xE0 { + ( + 3, + (u32::from(first & 0x0F) << 12) + | (u32::from(bytes.get(index + 1).copied().unwrap_or(0) & 0x3F) << 6) + | u32::from(bytes.get(index + 2).copied().unwrap_or(0) & 0x3F), + ) + } else if first & 0xF8 == 0xF0 { + ( + 4, + (u32::from(first & 0x07) << 18) + | (u32::from(bytes.get(index + 1).copied().unwrap_or(0) & 0x3F) << 12) + | (u32::from(bytes.get(index + 2).copied().unwrap_or(0) & 0x3F) << 6) + | u32::from(bytes.get(index + 3).copied().unwrap_or(0) & 0x3F), + ) + } else { + // A lead byte encode never produces; fall back to the + // replacement unit rather than panicking on foreign bytes. + units.push(0xFFFD); + index += 1; + continue; + }; + if width == 4 { + if let scalar @ 0x1_0000..=0x10FFFF = scalar { + let relative = scalar - 0x1_0000; + units.push(0xD800 + (relative >> 10) as u16); + units.push(0xDC00 + (relative & 0x3FF) as u16); + } else { + units.push(0xFFFD); + } + } else { + units.push(scalar as u16); + } + index += width; + } + OsString::from_wide(&units) +} diff --git a/engine/crates/layerfs-project/src/import/init.rs b/engine/crates/layerfs-project/src/import/init.rs index 3a068b58df..baca9ec4d0 100644 --- a/engine/crates/layerfs-project/src/import/init.rs +++ b/engine/crates/layerfs-project/src/import/init.rs @@ -1,8 +1,6 @@ //! Scan, construct/save, consume inode identities and publish one genesis stack. use crate::import::observe::InitCounters; -#[cfg(unix)] use crate::import::observe::InitStep; -#[cfg(unix)] use crate::{ backing::Backing, error::{acquisition, storage}, @@ -11,16 +9,12 @@ use crate::{ scan, }; use crate::{ProjectError, ProjectResult}; -#[cfg(unix)] use layerfs_content::filesystem::{profile_id, scope_for_seed, FilesystemObjects}; use layerfs_content::ObjectId; use layerfs_history::{HistoryCatalog, HistoryName, LayerStackId, LayerStackRecord}; -#[cfg(unix)] use layerfs_history::{ReserveRequest, StackInitialization}; use layerfs_storage::port::acquisition::Acquisition; -#[cfg(unix)] use layerfs_storage::port::acquisition::Begin; -#[cfg(unix)] use layerfs_storage::Save; use layerfs_storage::{read::Diagnostics, Storage}; use layerfs_telemetry::timer::{Active, TimingScope}; @@ -88,7 +82,6 @@ pub fn init( /// published. A failure of that removal is [`ProjectError::Cleanup`]; an /// unknown Store, history or backing outcome is [`ProjectError::Uncertain`] /// and leaves the operation untouched for its owner. -#[cfg(unix)] pub fn init_observed( store: &Storage, catalog: &dyn HistoryCatalog, @@ -96,20 +89,20 @@ pub fn init_observed( timer: &TimingScope<'_, Active>, counters: Arc, ) -> ProjectResult { - use std::os::unix::fs::MetadataExt; counters.step(InitStep::Scan); let mut progress = ImportProgress::observed(request.deadline, counters); progress.tick()?; let source = scan::check_root(request.source)?; if let Some(placement) = request.acquisition.placement() { - super::source::outside_source(&source, &placement)?; + super::source::outside_source(request.source, &source, &placement)?; } let scope = scope_for_seed(request.scope_seed); let mut stack = [0; 16]; stack.copy_from_slice(&request.stack.to_bytes()[1..]); + let host = super::host::HostStat::of(request.source, &source)?; let begun = request.acquisition.begin(&Begin { - source_device: source.dev(), - source_inode: source.ino(), + source_device: host.dev, + source_inode: host.ino, stack, scope: scope.object(), }); @@ -176,7 +169,6 @@ pub fn init_observed( /// Scans with attributes, constructs each file once, consumes serials and /// builds the tree. The returned tree save is unfinished. -#[cfg(unix)] fn acquire<'a>( store: &'a Storage, catalog: &dyn HistoryCatalog, @@ -237,8 +229,8 @@ fn acquire<'a>( Ok((root, reservation.start, scanned.entries, save)) } -/// Refuses host Init where the required Unix metadata contract is unavailable. -#[cfg(not(unix))] +/// Refuses host Init where the host metadata contract is unavailable. +#[cfg(not(any(unix, target_os = "windows")))] pub fn init_observed( _storage: &Storage, _catalog: &dyn HistoryCatalog, diff --git a/engine/crates/layerfs-project/src/import/mod.rs b/engine/crates/layerfs-project/src/import/mod.rs index f1093b1a26..8da7aba0d3 100644 --- a/engine/crates/layerfs-project/src/import/mod.rs +++ b/engine/crates/layerfs-project/src/import/mod.rs @@ -1,18 +1,15 @@ //! Responsibility-scoped implementation modules and reexports. -#[cfg(unix)] pub(crate) mod backing; -#[cfg(unix)] pub(crate) mod batch; pub(crate) mod error; -#[cfg(unix)] pub(crate) mod files; pub(crate) mod hints; +pub(crate) mod host; pub(crate) mod init; -#[cfg(unix)] pub(crate) mod namespace; pub(crate) mod observe; -#[cfg(unix)] pub(crate) mod scan; -#[cfg(unix)] pub(crate) mod source; +#[cfg(target_os = "windows")] +mod windows_identity; pub(crate) mod work; diff --git a/engine/crates/layerfs-project/src/import/scan.rs b/engine/crates/layerfs-project/src/import/scan.rs index 3ee3621b7a..26c01f3f91 100644 --- a/engine/crates/layerfs-project/src/import/scan.rs +++ b/engine/crates/layerfs-project/src/import/scan.rs @@ -18,10 +18,8 @@ use layerfs_content::filesystem::FilesystemObjects; use layerfs_content::{inode_leaf::InodeKind, PathName}; use layerfs_storage::port::acquisition::{EntryKey, NewEntry, Placed}; use std::{ - ffi::{OsStr, OsString}, fs::{self, Metadata}, - os::unix::ffi::{OsStrExt, OsStringExt}, - path::{Path, PathBuf}, + path::Path, }; /// What one completed scan placed in backing. @@ -84,11 +82,11 @@ impl Roots<'_, '_> { ) -> Result { let path = || match parent { Some(_) => child_path(directory, &child.name), - None => directory.as_os_str().as_bytes().to_vec(), + None => super::host::os_to_bytes(directory.as_os_str()), }; - let mode = child.portable_mode().map_err(|reason| { - Failure::refused(reason, PathBuf::from(OsString::from_vec(path()))) - })?; + let mode = child + .portable_mode() + .map_err(|reason| Failure::refused(reason, super::host::bytes_to_path(&path())))?; let nanos = child.stamp.mtime_nsec as u32; let key = (child.kind.code(), mode, child.stamp.mtime, nanos); let metadata = PortableMetadata { @@ -125,10 +123,11 @@ impl Roots<'_, '_> { } fn child_path(directory: &Path, name: &[u8]) -> Vec { - directory - .join(OsStr::from_bytes(name)) - .into_os_string() - .into_vec() + super::host::os_to_bytes( + &directory + .join(super::host::bytes_to_os(name)) + .into_os_string(), + ) } /// Positions being assigned while directories are read. @@ -170,7 +169,7 @@ pub(crate) fn scan( name: Vec::new(), kind: InodeKind::Directory, target: None, - stamp: Stamp::of(root), + stamp: Stamp::of(source, root)?, }; let root = roots.entry(root, None, Some(0), source)?; placement @@ -191,12 +190,15 @@ pub(crate) fn scan( placement.frontier -= 1; progress.tick()?; let parent = directory.position; - let path = PathBuf::from(OsString::from_vec(directory.host_path)); + let path = super::host::bytes_to_path(&directory.host_path); let (mut count, mut wide) = (0usize, false); for child in fs::read_dir(&path)? { progress.tick()?; let child = child?; - buffer.push(observe(&child.path(), child.file_name().as_bytes())?); + buffer.push(observe( + &child.path(), + &super::host::os_to_bytes(&child.file_name()), + )?); count += 1; progress.counters.entry(); if buffer.len() == WINDOW_ROWS { @@ -305,6 +307,6 @@ fn observe(path: &Path, name: &[u8]) -> Result { name: name.to_vec(), kind, target, - stamp: Stamp::of(&metadata), + stamp: Stamp::of(path, &metadata)?, }) } diff --git a/engine/crates/layerfs-project/src/import/source.rs b/engine/crates/layerfs-project/src/import/source.rs index 4640861a56..4735d69e52 100644 --- a/engine/crates/layerfs-project/src/import/source.rs +++ b/engine/crates/layerfs-project/src/import/source.rs @@ -1,9 +1,9 @@ //! Host source observations; symbolic links are stored without traversal. +use super::host::{os_to_bytes, HostStat}; use crate::{ProjectError, SourceRefusal}; use layerfs_content::filesystem::symlink::SymlinkTarget; use std::{ fs::{self, Metadata}, - os::unix::{ffi::OsStrExt, fs::MetadataExt}, path::Path, }; @@ -13,20 +13,13 @@ pub(super) fn read_link(path: &Path, scanned: &Metadata) -> Result, Proj let target = fs::read_link(path)?; let after = fs::symlink_metadata(path)?; if !after.file_type().is_symlink() - || after.dev() != scanned.dev() - || after.ino() != scanned.ino() - || after.len() != scanned.len() - || after.mode() != scanned.mode() - || after.mtime() != scanned.mtime() - || after.mtime_nsec() != scanned.mtime_nsec() - || after.ctime() != scanned.ctime() - || after.ctime_nsec() != scanned.ctime_nsec() + || HostStat::of(path, &after)? != HostStat::of(path, scanned)? { return Err(ProjectError::refused(SourceRefusal::Changed, path)); } // Keep the canonical target bound/grammar: no normalization or UTF-8 cast. - let target = SymlinkTarget::new(target.as_os_str().as_bytes().to_vec()) - .map_err(ProjectError::Content)?; + let target = + SymlinkTarget::new(os_to_bytes(target.as_os_str())).map_err(ProjectError::Content)?; Ok(target.as_bytes().to_vec()) } @@ -34,18 +27,54 @@ pub(super) fn read_link(path: &Path, scanned: &Metadata) -> Result, Proj /// /// The placement is resolved through its links first, then each ancestor's /// host identity is compared with the source's, so a symlinked or otherwise -/// aliased spelling of a directory inside the source is recognised. Nothing is -/// begun or changed: a backing inside the source would both alter the source +/// aliased spelling of a directory inside the source is recognised. Nothing +/// is begun or changed: a backing inside the source would both alter the source /// while it is read and be acquired as part of it. -pub(super) fn outside_source(source: &Metadata, placement: &Path) -> Result<(), ProjectError> { +/// +/// On Windows the same walk runs on the audited volume-serial + file-index +/// identity: a subst drive or junction whose directories carry the real +/// volume's identity is recognised and refused, like unix. +pub(super) fn outside_source( + source: &Path, + metadata: &Metadata, + placement: &Path, +) -> Result<(), ProjectError> { + #[cfg(unix)] let mut ancestor = fs::canonicalize(placement)?; - loop { - let metadata = fs::symlink_metadata(&ancestor)?; - if metadata.dev() == source.dev() && metadata.ino() == source.ino() { - return Err(ProjectError::BackingInsideSource); + #[cfg(unix)] + { + let _ = source; + use std::os::unix::fs::MetadataExt; + loop { + let entry = fs::symlink_metadata(&ancestor)?; + if entry.dev() == metadata.dev() && entry.ino() == metadata.ino() { + return Err(ProjectError::BackingInsideSource); + } + if !ancestor.pop() { + return Ok(()); + } } - if !ancestor.pop() { - return Ok(()); + } + #[cfg(not(unix))] + { + // Linux's algorithm on the audited Windows identity: canonicalize + // the placement once (junctions and symlinks in the spelling + // resolve), then compare every ancestor's volume-serial + file + // index with the source's. The identity walk sees through drive + // aliases a DOS-path comparison misses — a subst drive keeps its + // letter under GetFinalPathNameByHandle's DOS volume name, but its + // directories carry the real volume's device and index. + let source_resolved = fs::canonicalize(source)?; + let source_identity = super::host::HostStat::of(&source_resolved, metadata)?; + let mut ancestor = fs::canonicalize(placement)?; + loop { + let entry = super::host::HostStat::of(&ancestor, &fs::symlink_metadata(&ancestor)?)?; + if entry.dev == source_identity.dev && entry.ino == source_identity.ino { + return Err(ProjectError::BackingInsideSource); + } + if !ancestor.pop() { + return Ok(()); + } } } } diff --git a/engine/crates/layerfs-project/src/import/windows_identity.rs b/engine/crates/layerfs-project/src/import/windows_identity.rs new file mode 100644 index 0000000000..76aea18062 --- /dev/null +++ b/engine/crates/layerfs-project/src/import/windows_identity.rs @@ -0,0 +1,72 @@ +//! The Windows host-identity probe of namespace Init. +//! +//! This is layerfs-project's one owner-authorized module of raw Win32 +//! calls, allowed by the boundary guard and compiled on Windows only; no +//! other module in this crate may use unsafe Rust. The acquisition keys +//! rows by device and file index; the volume serial number and file index +//! are the entry's real identity. +#![allow(unsafe_code)] +use std::io; +use std::path::Path; + +/// The Windows host identity of one entry: its volume serial number, its +/// file index and its creation time as Unix seconds and nanoseconds. +pub(crate) struct WindowsFileIdentity { + pub(crate) device: u64, + pub(crate) index: u64, + pub(crate) created: i64, + pub(crate) created_nsec: i64, +} + +/// Reads `path`'s volume, file index and creation time without opening the +/// entry's data: attributes only, with traversal of neither links nor +/// content — a reparse point answers with its own identity, so a dangling +/// link still stamps like unix lstat. Directories need the +/// backup-semantics flag. +pub(crate) fn windows_file_identity(path: &Path) -> io::Result { + use windows_sys::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION, + FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_DELETE, + FILE_SHARE_READ, FILE_SHARE_WRITE, OPEN_EXISTING, + }; + // A verbatim spelling: raw units (an unpaired surrogate is a legal NTFS + // name and must reach CreateFileW unchanged, K3-R3 F2) under a \\?\ + // prefix, so deep trees beyond MAX_PATH stamp too. + let wide = layerfs_persistence::verbatim_wide(path); + let handle = unsafe { + CreateFileW( + wide.as_ptr(), + 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + std::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, + std::ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + let mut information = BY_HANDLE_FILE_INFORMATION::default(); + let read = unsafe { GetFileInformationByHandle(handle, &mut information) }; + // Capture the failure before another syscall can overwrite it (K3-R3 F9). + let failure = (read == 0).then(io::Error::last_os_error); + unsafe { CloseHandle(handle) }; + if let Some(error) = failure { + return Err(error); + } + // FILETIME is 100ns ticks since 1601-01-01; Unix time starts + // 11644473600 seconds later. + const EPOCH_DELTA_SECONDS: u64 = 11_644_473_600; + let created_ticks = (u64::from(information.ftCreationTime.dwHighDateTime) << 32) + | u64::from(information.ftCreationTime.dwLowDateTime); + let created_unix = (created_ticks / 10_000_000).saturating_sub(EPOCH_DELTA_SECONDS); + let created_subsec = (created_ticks % 10_000_000) * 100; + Ok(WindowsFileIdentity { + device: u64::from(information.dwVolumeSerialNumber), + index: (u64::from(information.nFileIndexHigh) << 32) | u64::from(information.nFileIndexLow), + created: created_unix as i64, + created_nsec: created_subsec as i64, + }) +} diff --git a/engine/crates/layerfs-project/src/lib.rs b/engine/crates/layerfs-project/src/lib.rs index 0e93b61b2b..6008ac38f6 100644 --- a/engine/crates/layerfs-project/src/lib.rs +++ b/engine/crates/layerfs-project/src/lib.rs @@ -1,21 +1,18 @@ //! Namespace construction and Store Init/seal/install composition. -#![forbid(unsafe_code)] +// deny, not upstream's forbid: the audited Windows identity module (owner +// direction 2026-10-25) lives in this crate (core/AGENTS.md). +#![deny(unsafe_code)] #![deny(missing_docs)] mod import; pub mod store; pub use error::{ProjectError, ProjectResult, RetainedAcquisition, SourceRefusal}; -#[cfg(unix)] pub(crate) use import::backing; -#[cfg(unix)] pub(crate) use import::batch; pub(crate) use import::error; -#[cfg(unix)] pub(crate) use import::files; pub(crate) use import::init; -#[cfg(unix)] pub(crate) use import::namespace; pub use import::observe::{InitCounters, InitCounts, InitStep}; -#[cfg(unix)] pub(crate) use import::scan; pub use init::{init, init_observed, InitRequest, Initialized}; diff --git a/engine/crates/layerfs-project/tests/init_sqlite.rs b/engine/crates/layerfs-project/tests/init_sqlite.rs index b69c02af9e..bcc1f8f5e6 100644 --- a/engine/crates/layerfs-project/tests/init_sqlite.rs +++ b/engine/crates/layerfs-project/tests/init_sqlite.rs @@ -1,38 +1,15 @@ //! Canonical namespace oracle over both explicit embedded Store profiles. //! -//! The same Store opens on macOS/Linux. Unsupported platforms retain -//! the explicit before-file-creation refusal. +//! The same Store opens on macOS, Linux and Windows (owner direction +//! 2026-10-25). Other platforms keep the product's compile-time refusal. mod support; use layerfs_history::HistoryCatalogConfig; -#[cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] use layerfs_persistence::SqliteAcquisitionSchema; use layerfs_persistence::{Handles, PersistenceConfig, SqlitePersistenceProfile}; use layerfs_storage::StoragePolicy; use support::*; -#[cfg(not(any(target_os = "macos", target_os = "linux")))] -#[test] -fn an_unsupported_platform_refuses_the_store_before_any_file_exists() { - use layerfs_storage::port::PersistenceError; - for selected in DEVELOPMENT_PROFILES { - let fixture = Fixture::new(0); - let store = fixture.path.join("store.sqlite"); - let refused = Handles::create( - PersistenceConfig::sqlite(&store).with_sqlite_profile(selected), - StoragePolicy::frozen_default(), - &HistoryCatalogConfig { - binding_key: b"init-sqlite".to_vec(), - cursor_key: [71; 32], - incarnation: 1, - }, - ); - assert!( - matches!(refused, Err(PersistenceError::BackendUnavailable)), - "{selected:?} must be refused on this platform" - ); - assert!(!store.exists(), "a refused Store creates no file"); - } -} -#[cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] #[test] fn selected_profiles_init_100_and_1000_pass_the_full_namespace_oracle() { for selected in DEVELOPMENT_PROFILES { @@ -76,7 +53,7 @@ fn selected_profiles_init_100_and_1000_pass_the_full_namespace_oracle() { } } -#[cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] fn attempt( fixture: &Fixture, handles: &Handles, @@ -103,10 +80,14 @@ fn attempt( .0 } -#[cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] fn create(store: &std::path::Path, acquisition: SqliteAcquisitionSchema) -> Handles { + // Disposable is the sole permitted execution profile (owner 2026-10-07); + // the constructor's default is Durable, which this suite never executes. Handles::create( - PersistenceConfig::sqlite(store).with_sqlite_acquisition(acquisition), + PersistenceConfig::sqlite(store) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable) + .with_sqlite_acquisition(acquisition), StoragePolicy::frozen_default(), &HistoryCatalogConfig { binding_key: b"init-sqlite".to_vec(), @@ -119,7 +100,7 @@ fn create(store: &std::path::Path, acquisition: SqliteAcquisitionSchema) -> Hand /// A Store created without the acquisition tables has no working state for /// Init, and there is no second acquisition algorithm to fall back to. -#[cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] #[test] fn a_store_without_acquisition_tables_refuses_init() { use layerfs_history::{HistoryCatalog, LayerStackId}; @@ -146,7 +127,7 @@ fn a_store_without_acquisition_tables_refuses_init() { /// The Store's own directory is acquisition backing: a Store inside the source /// would be read while it is written and acquired as part of the root. -#[cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] #[test] fn a_store_inside_the_source_is_refused_before_an_operation_begins() { use layerfs_project::ProjectError; @@ -177,6 +158,108 @@ fn a_store_inside_the_source_is_refused_before_an_operation_begins() { assert_eq!((owner.operation, owner.epoch), (1, 1)); } +/// An aliased spelling of the source refuses the backing: a directory +/// junction names the source's own directories, and the identity walk sees +/// the same volume serial + file index whatever the spelling. +#[cfg(target_os = "windows")] +#[test] +fn a_junction_spelling_of_the_source_is_refused_as_inside_it() { + use layerfs_history::{HistoryName, LayerStackId}; + use layerfs_project::{InitRequest, ProjectError}; + use support::memory_acquisition::MemoryAcquisition; + let fixture = Fixture::new(5); + let handles = create( + &fixture.path.join("store.sqlite"), + SqliteAcquisitionSchema::Tables, + ); + let junction = fixture.path.join("alias"); + std::process::Command::new("cmd") + .args(["/c", "mklink", "/J"]) + .arg(&junction) + .arg(&fixture.source) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .unwrap(); + assert!( + junction.join("d0").is_dir(), + "the junction names the source" + ); + let storage = storage(handles.storage.clone()); + let error = layerfs_telemetry::timer::Timing::disabled("project.init", |scope| { + layerfs_project::init( + &storage, + &handles.history, + InitRequest { + source: &fixture.source, + acquisition: &MemoryAcquisition::placed_in(junction), + stack: LayerStackId::from_authority([19; 16]), + name: HistoryName::new("main").unwrap(), + scope_seed: [29; 32], + deadline: std::time::Instant::now() + std::time::Duration::from_secs(60), + }, + scope, + ) + }) + .0 + .unwrap_err(); + assert!( + matches!(error, ProjectError::BackingInsideSource), + "{error:?}" + ); + std::fs::remove_dir(fixture.path.join("alias")).unwrap(); +} + +/// A dangling link initializes with the rest of the tree: the scan stamps a +/// reparse point's own identity and never follows it to a target that is +/// gone (unix lstat parity; std classifies a junction as a symlink). +#[cfg(target_os = "windows")] +#[test] +fn a_dangling_junction_initializes_with_the_rest_of_the_tree() { + use layerfs_history::{HistoryName, LayerStackId}; + use layerfs_project::InitRequest; + use support::memory_acquisition::MemoryAcquisition; + let fixture = Fixture::new(5); + let target = fixture.path.join("gone"); + std::fs::create_dir_all(&target).unwrap(); + let link = fixture.source.join("dangling"); + std::process::Command::new("cmd") + .args(["/c", "mklink", "/J"]) + .arg(&link) + .arg(&target) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .unwrap(); + std::fs::remove_dir(&target).unwrap(); + let volume = fixture.path.join("volume"); + std::fs::create_dir_all(&volume).unwrap(); + let handles = create( + &fixture.path.join("store.sqlite"), + SqliteAcquisitionSchema::Tables, + ); + let storage = storage(handles.storage.clone()); + let initialized = layerfs_telemetry::timer::Timing::disabled("project.init", |scope| { + layerfs_project::init( + &storage, + &handles.history, + InitRequest { + source: &fixture.source, + acquisition: &MemoryAcquisition::placed_in(volume), + stack: LayerStackId::from_authority([43; 16]), + name: HistoryName::new("main").unwrap(), + scope_seed: [47; 32], + deadline: std::time::Instant::now() + std::time::Duration::from_secs(60), + }, + scope, + ) + }) + .0 + .unwrap(); + // 5 directories + 5 files + the dangling link itself. + assert!(initialized.entries >= 11, "{}", initialized.entries); +} + /// Owner direction 2026-10-07: Disposable is the sole active development /// verification profile. Durable execution of these bodies is NOT_RUN — /// deferred by owner for Disposable-only development. Durable support and its diff --git a/engine/crates/layerfs-project/tests/name_bytes_windows.rs b/engine/crates/layerfs-project/tests/name_bytes_windows.rs new file mode 100644 index 0000000000..46878d7f09 --- /dev/null +++ b/engine/crates/layerfs-project/tests/name_bytes_windows.rs @@ -0,0 +1,13 @@ +//! The product's WTF-8 codec, included from its source file for the test. +//! Gated like every Windows-only target: the fast lane builds this file on +//! Linux too, and the codec's os::windows imports exist only there. +#![cfg(target_os = "windows")] +#[path = "../src/import/host_wtf8_inner.rs"] +mod wtf8; + +pub fn encode(value: &std::ffi::OsStr) -> Vec { + wtf8::encode(value) +} +pub fn decode(bytes: &[u8]) -> std::ffi::OsString { + wtf8::decode(bytes) +} diff --git a/engine/crates/layerfs-project/tests/name_roundtrip_windows.rs b/engine/crates/layerfs-project/tests/name_roundtrip_windows.rs new file mode 100644 index 0000000000..2ce00efb59 --- /dev/null +++ b/engine/crates/layerfs-project/tests/name_roundtrip_windows.rs @@ -0,0 +1,135 @@ +//! Windows host names travel as WTF-8 bytes and come back unchanged - a +//! name outside UTF-8 (an unpaired surrogate) is no longer lossy. +#![cfg(target_os = "windows")] +use std::ffi::OsString; +use std::os::windows::ffi::OsStringExt; + +mod support; + +/// The bytes module under test, exercised through its two entry points. +#[path = "name_bytes_windows.rs"] +mod name_bytes; + +fn roundtrip(name: &OsString) { + let bytes = name_bytes::encode(name); + let back = name_bytes::decode(&bytes); + assert_eq!(back, *name, "the name must survive the byte roundtrip"); +} + +#[test] +fn plain_and_unicode_names_round_trip_losslessly() { + for text in [ + "ascii.txt", + "中文目录/文件.bin", + "emoji-🚀-🧬.md", + "Ünïcødé", + ] { + roundtrip(&OsString::from(text)); + } +} + +#[test] +fn a_well_formed_pair_encodes_as_its_utf_8_scalar() { + // The bytes Linux stores for 🚀 are its UTF-8 spelling; Windows must + // agree byte for byte or one name acquires as two different trees + // (second review, finding 1). + let rocket = OsString::from("🚀"); + let bytes = name_bytes::encode(&rocket); + assert_eq!(bytes, "🚀".as_bytes()); + let dna = OsString::from("🧬"); + assert_eq!(name_bytes::encode(&dna), "🧬".as_bytes()); + // A pair next to a lone unit keeps both spellings in one name. + let mixed = OsString::from_wide(&[0xD83D, 0xDE80, 0xDC80]); + let mixed_bytes = name_bytes::encode(&mixed); + assert_eq!( + mixed_bytes, + [b"\xf0\x9f\x9a\x80".as_slice(), b"\xed\xb2\x80".as_slice()].concat(), + "the pair combines, the unpaired low surrogate keeps its form" + ); + assert_eq!(name_bytes::decode(&mixed_bytes), mixed); +} + +#[test] +fn an_unpaired_surrogate_name_round_trips_losslessly() { + // A name no String can hold: one lone high surrogate. Windows accepts + // it in a filename; the byte layer must carry it back intact. + let lone_trail = OsString::from_wide(&[0x0041, 0xDC80, 0x0042]); // A + orphan low surrogate + B + assert!( + lone_trail.to_str().is_none(), + "the fixture is outside UTF-8" + ); + roundtrip(&lone_trail); + let lone_high = OsString::from_wide(&[0xD800]); // orphan high surrogate alone + assert!( + lone_high.to_str().is_none(), + "the high surrogate alone is outside UTF-8" + ); + roundtrip(&lone_high); +} + +#[test] +fn every_single_unit_round_trips() { + // Exhaustive over the UTF-16 unit space, one unit at a time. + for unit in 0u16..=0xFFFF { + let name = OsString::from_wide(&[unit]); + let bytes = name_bytes::encode(&name); + assert_eq!(name_bytes::decode(&bytes), name, "unit {unit:#06x}"); + } +} + +#[test] +fn init_acquires_a_surrogate_named_file_end_to_end() { + // End to end through the public init: the scan stamps every entry's + // identity through CreateFileW, which must receive the name's raw units + // (K3-R3 F2) - a lossy spelling opened a nonexistent path and refused + // the whole Init on a legal NTFS name. + use layerfs_history::{HistoryCatalogConfig, HistoryName, LayerStackId}; + use layerfs_persistence::{ + Handles, PersistenceConfig, SqliteAcquisitionSchema, SqlitePersistenceProfile, + }; + use layerfs_project::{init, InitRequest}; + use layerfs_storage::StoragePolicy; + use support::{memory_acquisition::MemoryAcquisition, Fixture}; + let fixture = Fixture::new(5); + let name = OsString::from_wide(&[0x0041, 0xD800, 0x0042]); // "AB" + let path = fixture.source.join(&name); + std::fs::write(&path, b"body").unwrap(); + assert!(path.exists(), "the fixture file exists outside UTF-8"); + let volume = fixture.path.join("volume"); + std::fs::create_dir_all(&volume).unwrap(); + let handles = Handles::create( + PersistenceConfig::sqlite(fixture.path.join("store.sqlite")) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable) + .with_sqlite_acquisition(SqliteAcquisitionSchema::Tables), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: b"wtf8-init".to_vec(), + cursor_key: [83; 32], + incarnation: 1, + }, + ) + .unwrap(); + let storage = support::storage(handles.storage.clone()); + let initialized = layerfs_telemetry::timer::Timing::disabled("project.init", |scope| { + init( + &storage, + &handles.history, + InitRequest { + source: &fixture.source, + acquisition: &MemoryAcquisition::placed_in(volume), + stack: LayerStackId::from_authority([37; 16]), + name: HistoryName::new("main").unwrap(), + scope_seed: [41; 32], + deadline: std::time::Instant::now() + std::time::Duration::from_secs(60), + }, + scope, + ) + }) + .0 + .unwrap(); + assert!( + initialized.entries >= 6, + "the surrogate-named file initializes with the rest of the tree: {}", + initialized.entries + ); +} diff --git a/engine/crates/layerfs-project/tests/support/mod.rs b/engine/crates/layerfs-project/tests/support/mod.rs index 94caa4bec0..a066175bc3 100644 --- a/engine/crates/layerfs-project/tests/support/mod.rs +++ b/engine/crates/layerfs-project/tests/support/mod.rs @@ -10,10 +10,11 @@ use layerfs_storage::{ Storage, }; use layerfs_telemetry::timer::Timing; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; use std::{ collections::BTreeMap, fs, - os::unix::fs::MetadataExt, path::PathBuf, sync::{ atomic::{AtomicU64, Ordering}, @@ -120,9 +121,37 @@ impl Fixture { ) .unwrap(); let metadata = fs::symlink_metadata(self.source.join(&path)).unwrap(); - assert_eq!(portable.mode, metadata.mode() & 0o7777); - assert_eq!(portable.mtime_seconds, metadata.mtime()); - assert_eq!(portable.mtime_nanoseconds as i64, metadata.mtime_nsec()); + // The stored portable metadata equals the host's own reading of + // the entry: the raw stat identity on unix; on Windows the + // synthesized contract of import/host.rs (mode from the + // read-only attribute, mtime from the entry's modified time). + #[cfg(unix)] + { + assert_eq!(portable.mode, metadata.mode() & 0o7777); + assert_eq!(portable.mtime_seconds, metadata.mtime()); + assert_eq!(portable.mtime_nanoseconds as i64, metadata.mtime_nsec()); + } + #[cfg(windows)] + { + use std::time::UNIX_EPOCH; + let base = if metadata.is_dir() { 0o555 } else { 0o444 }; + let mode = if metadata.permissions().readonly() { + base + } else { + base | 0o200 + }; + assert_eq!(portable.mode, mode); + let modified = metadata + .modified() + .unwrap() + .duration_since(UNIX_EPOCH) + .unwrap(); + assert_eq!(portable.mtime_seconds, modified.as_secs() as i64); + assert_eq!( + portable.mtime_nanoseconds as i64, + i64::from(modified.subsec_nanos()) + ); + } if metadata.is_dir() { let mut after = None; loop { diff --git a/engine/crates/layerfs-store-service/Cargo.toml b/engine/crates/layerfs-store-service/Cargo.toml index 0586821434..b45b255138 100644 --- a/engine/crates/layerfs-store-service/Cargo.toml +++ b/engine/crates/layerfs-store-service/Cargo.toml @@ -17,3 +17,7 @@ layerfs-telemetry = { path = "../../../core/crates/layerfs-telemetry" } # Concrete Store opening (bootstrap) and the kernel guard's open. layerfs-persistence = { path = "../layerfs-persistence" } nix = { version = "=0.31.3", default-features = false, features = ["fs"] } + +[target.'cfg(windows)'.dependencies] +# The audited owner-only DACL module; windows-sys is already in the locked graph. +windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_Security_Authorization"] } diff --git a/engine/crates/layerfs-store-service/src/install.rs b/engine/crates/layerfs-store-service/src/install.rs index 850570a334..80aefe8011 100644 --- a/engine/crates/layerfs-store-service/src/install.rs +++ b/engine/crates/layerfs-store-service/src/install.rs @@ -21,6 +21,7 @@ use std::sync::Arc; /// The daemon's user inside its Sandbox (the runtime's `User` is `0:0`). A /// daemon opens only a Store file and directory it owns, so an installer /// must run as this user too. +#[cfg(not(target_os = "windows"))] const DAEMON_UID: u32 = 0; /// The destination and its exclusive temporary name. @@ -69,6 +70,13 @@ pub fn claim(destination: &Path, temporary: &Path) -> io::Result { use std::os::unix::fs::OpenOptionsExt; options.mode(0o600); } + #[cfg(windows)] + { + let claimed = options.open(temporary)?; + crate::windows_acl::restrict_to_owner(temporary)?; + Ok(claimed) + } + #[cfg(not(windows))] options.open(temporary) } @@ -158,16 +166,29 @@ pub fn install_local( let directory = destination .parent() .ok_or("the Store destination has no directory")?; - crate::kernel_guard::check(directory).map_err(|reason| reason.0)?; - installer_user()?; let text = |error: io::Error| error.to_string(); + // The directory is created first (owner-only): the guard examines + // it as the Store file's parent, and install owns its creation. if !directory.exists() { - use std::os::unix::fs::DirBuilderExt; - fs::DirBuilder::new() - .mode(0o700) - .create(directory) - .map_err(text)?; + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + fs::DirBuilder::new() + .mode(0o700) + .create(directory) + .map_err(text)?; + } + #[cfg(not(unix))] + { + fs::DirBuilder::new().create(directory).map_err(text)?; + #[cfg(windows)] + crate::windows_acl::restrict_to_owner(directory).map_err(text)?; + } } + // check takes the Store file and strips the parent itself, like + // every caller. + crate::kernel_guard::check(destination).map_err(|reason| reason.0)?; + installer_user()?; let (target, temporary) = paths(destination).map_err(text)?; let mut file = claim(&target, &temporary).map_err(text)?; left.temporary = Some(temporary.clone()); @@ -204,16 +225,24 @@ pub fn install_local( /// Refuses an installer that is not the daemons' user. fn installer_user() -> Result<(), String> { - use std::os::unix::fs::MetadataExt; - // the owner of /proc/self is this process's effective user - let uid = fs::metadata("/proc/self") - .map_err(|error| format!("this process's user is unknown: {error}"))? - .uid(); - if uid != DAEMON_UID { - return Err(format!( - "the Store install runs as user {uid}; it must run as user {DAEMON_UID}, the \ - daemons' user, which opens only a Store it owns" - )); + #[cfg(target_os = "windows")] + { + // Windows has no daemons' uid to check here; the destination's + // local-volume discipline is the kernel guard's, which still applies. + Ok(()) + } + #[cfg(not(target_os = "windows"))] + { + use std::os::unix::fs::MetadataExt; + // the owner of /proc/self is this process's effective user + let uid = fs::metadata("/proc/self") + .map_err(|error| format!("this process's user is unknown: {error}"))? + .uid(); + if uid != DAEMON_UID { + return Err(format!( + "the Store install runs as user {uid}; it must run as user {DAEMON_UID}, the daemons' user, which opens only a Store it owns" + )); + } + Ok(()) } - Ok(()) } diff --git a/engine/crates/layerfs-store-service/src/kernel_guard.rs b/engine/crates/layerfs-store-service/src/kernel_guard.rs index 49489cac1e..511ebfc4d7 100644 --- a/engine/crates/layerfs-store-service/src/kernel_guard.rs +++ b/engine/crates/layerfs-store-service/src/kernel_guard.rs @@ -1,12 +1,14 @@ //! Whether this process may open the global Store directly (#352 F-020). //! -//! Every opener of one Store must share one Linux kernel and the local -//! filesystem that holds it: SQLite's WAL locks and the writers' admission -//! lane (a `flock` on the Store's directory) coordinate only within one -//! kernel. The guard checks what this process can verify before it opens any -//! file: the platform, and the type of the filesystem holding the Store's -//! directory, from an allowlist of local filesystems. Anything else is -//! `Unsupported` with its reason; there is no other route to fall back to. +//! Every opener of one Store must share one kernel and the local filesystem +//! that holds it: SQLite's WAL locks and the writers' admission lane +//! coordinate only within one kernel. On Linux that is the daemons' kernel +//! over ext4, xfs or btrfs; on Windows it is this machine's kernel over +//! NTFS on a local volume. The guard checks what this process can verify +//! before it opens any file: the platform, and the type of the filesystem +//! holding the Store's directory, from an allowlist of local filesystems. +//! Anything else is `Unsupported` with its reason; there is no other route +//! to fall back to. use std::path::Path; /// Filesystems that carry the Store's locks: ext4, xfs and btrfs. @@ -28,7 +30,9 @@ impl std::fmt::Display for Unsupported { } /// Checks the platform and the filesystem holding `store`'s directory, -/// before anything is opened. Returns the filesystem's name. +/// before anything is opened. `store` is the Store **file** path — the +/// directory is its parent, taken here once so every caller agrees. +/// Returns the filesystem's name. pub fn check(store: &Path) -> Result<&'static str, Unsupported> { #[cfg(target_os = "linux")] { @@ -56,7 +60,31 @@ pub fn check(store: &Path) -> Result<&'static str, Unsupported> { )) }) } - #[cfg(not(target_os = "linux"))] + #[cfg(target_os = "windows")] + { + // The allowlist is the Store's own discipline on Windows: only NTFS + // on a local volume carries the byte-range locks SQLite's WAL needs + // between processes. The audited probe lives in the persistence + // crate's Windows FFI module; network and device paths are refused + // there before a volume is examined. + let directory = store + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + match layerfs_persistence::windows_volume_filesystem(directory) { + Ok(filesystem) if filesystem == "NTFS" => Ok("ntfs"), + Ok(filesystem) => Err(Unsupported(format!( + "filesystem {filesystem} at {} does not carry the Store's locks between \ + processes; direct Store access needs NTFS", + directory.display() + ))), + Err(error) => Err(Unsupported(format!( + "the Store's directory {} cannot be examined: {error}", + directory.display() + ))), + } + } + #[cfg(not(any(target_os = "linux", target_os = "windows")))] { let _ = store; Err(Unsupported( diff --git a/engine/crates/layerfs-store-service/src/lib.rs b/engine/crates/layerfs-store-service/src/lib.rs index fbc151dc47..0505e405e1 100644 --- a/engine/crates/layerfs-store-service/src/lib.rs +++ b/engine/crates/layerfs-store-service/src/lib.rs @@ -10,7 +10,7 @@ //! Sandbox. Each Workspace owns one overlay connection behind one lock; a //! caller runs its own short typed call under it and keeps exact failure //! custody. -#![forbid(unsafe_code)] +#![deny(unsafe_code)] pub mod bootstrap; pub mod history; @@ -23,6 +23,8 @@ pub mod service; pub mod store; mod store_service; pub mod telemetry; +#[cfg(target_os = "windows")] +pub mod windows_acl; pub mod write; pub use overlay::owner; diff --git a/engine/crates/layerfs-store-service/src/overlay/backing.rs b/engine/crates/layerfs-store-service/src/overlay/backing.rs index de78d47877..ad9b82fe53 100644 --- a/engine/crates/layerfs-store-service/src/overlay/backing.rs +++ b/engine/crates/layerfs-store-service/src/overlay/backing.rs @@ -14,7 +14,11 @@ pub struct Backing { file: Option, namespace: i64, incarnation: [u8; 32], + // The deletion identity check reads these on unix and Windows; the + // remaining platforms refuse deletion before reaching them. + #[cfg_attr(not(any(unix, target_os = "windows")), allow(dead_code))] device: u64, + #[cfg_attr(not(any(unix, target_os = "windows")), allow(dead_code))] inode: u64, pool: Weak, terminal: AtomicBool, @@ -42,7 +46,19 @@ impl Backing { } (metadata.dev(), metadata.ino(), metadata.nlink()) }; - #[cfg(not(unix))] + #[cfg(target_os = "windows")] + let (device, inode, links) = { + // The audited probe in the overlay engine reads the entry's real + // volume, file index and link count. + let identity = layerfs_overlay::windows_handle_identity(&file).map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("overlay file identity: {error}"), + ) + })?; + (identity.device, identity.index, identity.links) + }; + #[cfg(not(any(unix, target_os = "windows")))] let (device, inode, links) = (0, 0, 0); if !metadata.is_file() || links != 1 { return Err(io::Error::new( @@ -69,14 +85,14 @@ impl Backing { self.closed.store(true, Ordering::Release); } fn delete(&self) -> io::Result<()> { - let metadata = std::fs::symlink_metadata(&self.path)?; - let original = self - .file - .as_ref() - .expect("retained identity file") - .metadata()?; #[cfg(unix)] { + let metadata = std::fs::symlink_metadata(&self.path)?; + let original = self + .file + .as_ref() + .expect("retained identity file") + .metadata()?; use std::os::unix::fs::MetadataExt; if !metadata.is_file() || metadata.nlink() != 1 @@ -89,14 +105,54 @@ impl Backing { "overlay deletion identity", )); } + std::fs::remove_file(&self.path) + } + #[cfg(target_os = "windows")] + { + // The same discipline through the audited Windows identity: the + // entry at the path and the retained open handle must both still + // be the acquisition's file (volume serial + file index, sole + // link) before the removal. The retained handle was opened with + // FILE_SHARE_DELETE (std's share mode), so the removal takes + // effect when Drop closes that last handle, immediately after. + let path_identity = + layerfs_overlay::windows_path_identity(&self.path).map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("overlay deletion identity: {error}"), + ) + })?; + let retained = layerfs_overlay::windows_handle_identity( + self.file.as_ref().expect("retained identity file"), + ) + .map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("overlay deletion identity: {error}"), + ) + })?; + if path_identity.device != self.device + || path_identity.index != self.inode + || retained.device != self.device + || retained.index != self.inode + || path_identity.links != 1 + || retained.links != 1 + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "overlay deletion identity", + )); + } + std::fs::remove_file(&self.path) + } + #[cfg(not(any(unix, target_os = "windows")))] + { + let _ = &self.path; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "overlay deletion platform", + )) } - #[cfg(not(unix))] - return Err(io::Error::new( - io::ErrorKind::Unsupported, - "overlay deletion platform", - )); - #[cfg(unix)] - std::fs::remove_file(&self.path) } } impl Drop for Backing { diff --git a/engine/crates/layerfs-store-service/src/store_service.rs b/engine/crates/layerfs-store-service/src/store_service.rs index 791755cb86..c2256d4878 100644 --- a/engine/crates/layerfs-store-service/src/store_service.rs +++ b/engine/crates/layerfs-store-service/src/store_service.rs @@ -55,11 +55,18 @@ pub struct StoreService { capacity: usize, bound: AtomicUsize, retained: Mutex>, + /// The Store file whose writer lane guards this service's close. + store_file: std::path::PathBuf, } impl StoreService { /// Serves Store calls over `opened`, constructing through `owner`. pub fn new(opened: OpenedStore, owner: &Owner) -> Self { + Self::at(opened, owner, std::path::PathBuf::new()) + } + /// [`new`](Self::new) with the Store file recorded, for a close that + /// coordinates its sessions' teardown under the writer lane. + pub fn at(opened: OpenedStore, owner: &Owner, store_file: std::path::PathBuf) -> Self { Self { store: opened.store.clone(), opened, @@ -68,6 +75,7 @@ impl StoreService { capacity: owner.configuration().namespaces, bound: AtomicUsize::new(0), retained: Mutex::new(Vec::new()), + store_file, } } @@ -84,12 +92,43 @@ impl StoreService { template: &Path, ) -> Result { kernel_guard::check(store).map_err(OpenRefusal::Unsupported)?; + // A missing Store answers `Missing` on every platform before any + // entry examination: the Windows symlink/ACL block used to run + // first and surfaced a missing file as `Failed` (round-6 D1 F4). if !store.is_file() { return Err(OpenRefusal::Missing(format!( "no installed Store at {}", store.display() ))); } + // The installed Store file answers to its own entry: a substituted + // link planted at its spelling is refused, never followed, and on + // Windows its DACL must still grant nothing beyond the owner — the + // discovery tree's privacy check does not cover the installed file. + #[cfg(windows)] + { + use std::fs::symlink_metadata; + let entry = symlink_metadata(store) + .map_err(|error| OpenRefusal::Failed(format!("Store entry: {error}")))?; + if entry.file_type().is_symlink() { + return Err(OpenRefusal::Unsupported(crate::kernel_guard::Unsupported( + format!( + "the installed Store at {} is a link, not the Store file", + store.display() + ), + ))); + } + // The Store file itself is the privacy boundary (its owner-only + // DACL protects the content); the directory is restricted when + // the route creates it, and a caller-pre-created directory is + // accepted as the caller shaped it. + crate::windows_acl::require_owner_only(store).map_err(|error| { + OpenRefusal::Unsupported(crate::kernel_guard::Unsupported(format!( + "the Store at {}: {error}", + store.display() + ))) + })?; + } let opened = open_store_observed( PersistenceConfig::sqlite(store) .with_sqlite_profile(SqlitePersistenceProfile::Disposable), @@ -100,6 +139,17 @@ impl StoreService { ReservationBlocks::default(), ) .map_err(|error| OpenRefusal::Failed(format!("Store open: {error}")))?; + // SQLite's `-wal`/`-shm` sidecars inherit the directory's DACL, so + // a crash that leaves them behind exposes committed content on a + // caller-pre-created directory until the next open; whenever an + // open sees them it tightens them (best effort — the next open + // retries, and the Store stays usable either way; round-3 A1 F1). + #[cfg(windows)] + for suffix in ["-wal", "-shm"] { + let mut sidecar = store.as_os_str().to_owned(); + sidecar.push(suffix); + let _ = crate::windows_acl::restrict_to_owner(std::path::Path::new(&sidecar)); + } let owner = Owner::start( template, ProfileConfig { @@ -109,7 +159,7 @@ impl StoreService { OwnerConfig { namespaces: CALLS }, ) .map_err(|error| OpenRefusal::Failed(format!("overlay template: {error}")))?; - let mut service = Self::new(opened, &owner); + let mut service = Self::at(opened, &owner, store.to_path_buf()); service.owned = Some(owner); Ok(service) } @@ -117,10 +167,20 @@ impl StoreService { /// Stops the overlay owner this service started. Private Workspaces kept /// with an uncertain Commit stay with it until then. pub fn close(mut self) -> Result<(), String> { - match self.owned.take() { + // The overlays stop first: their final Commits take the writer lane + // themselves, so no outer hold may surround this. + let stopped = match self.owned.take() { Some(owner) => owner.stop().map_err(|error| error.to_string()), None => Ok(()), - } + }; + // The sessions then close under the Store's writer lane, so their + // close-time checkpoints never run beside another opener's first + // attach. A failed lane acquisition closes bare, as a drop does. + let lane = (!self.store_file.as_os_str().is_empty()) + .then(|| layerfs_persistence::store_lane(&self.store_file)); + drop(self); + drop(lane); + stopped } /// One private Workspace on `branch` for one file-changing call. diff --git a/engine/crates/layerfs-store-service/src/windows_acl.rs b/engine/crates/layerfs-store-service/src/windows_acl.rs new file mode 100644 index 0000000000..f1f976d8de --- /dev/null +++ b/engine/crates/layerfs-store-service/src/windows_acl.rs @@ -0,0 +1,196 @@ +//! The Windows privacy boundary of homes, discovery records and Stores: +//! owner-only DACLs, the parity of unix's 0700 directories and 0600 files. +//! +//! This is the crate's one owner-authorized module of raw Win32 calls, +//! allowed by the boundary guard and compiled on Windows only; no other +//! module in this crate may use unsafe Rust. A fresh entry is restricted to +//! its owner by a protected DACL whose single allow-ACE names the OWNER +//! RIGHTS SID (S-1-3-4): whoever owns the entry keeps full control and no +//! inherited grant survives, so a home created under a world-readable +//! parent leaks nothing (the unix arm reads the same intent from mode +//! bits). An existing entry is refused when its DACL grants access to +//! Everyone, Builtin\Users or Authenticated Users, or when it carries no +//! DACL at all (a null DACL is everyone-full). +#![allow(unsafe_code)] +use std::io; +use std::path::Path; + +/// A freshly built well-known SID, `SECURITY_MAX_SID_SIZE` bytes. +fn well_known(kind: windows_sys::Win32::Security::WELL_KNOWN_SID_TYPE) -> io::Result<[u8; 68]> { + use windows_sys::Win32::Security::{CreateWellKnownSid, PSID, SECURITY_MAX_SID_SIZE}; + let mut sid = [0u8; SECURITY_MAX_SID_SIZE as usize]; + let mut size = sid.len() as u32; + if unsafe { + CreateWellKnownSid( + kind, + std::ptr::null_mut(), + sid.as_mut_ptr() as PSID, + &mut size, + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + Ok(sid) +} + +/// Restricts `path` — file or directory — to its owner: one allow-ACE for +/// OWNER RIGHTS under a protected DACL; nothing is inherited. +pub fn restrict_to_owner(path: &Path) -> io::Result<()> { + use windows_sys::Win32::Foundation::{LocalFree, HLOCAL}; + use windows_sys::Win32::Security::Authorization::{ + SetEntriesInAclW, SetNamedSecurityInfoW, EXPLICIT_ACCESS_W, GRANT_ACCESS, SE_FILE_OBJECT, + TRUSTEE_IS_SID, TRUSTEE_W, + }; + use windows_sys::Win32::Security::{ + WinCreatorOwnerRightsSid, ACL, DACL_SECURITY_INFORMATION, + PROTECTED_DACL_SECURITY_INFORMATION, + }; + let mut sid = well_known(WinCreatorOwnerRightsSid)?; + // The concrete all-file-rights mask, not GENERIC_ALL: a generic bit + // written raw into the ACE is not mapped for the file object type, and + // the owner's own write lands denied. + const FILE_ALL_ACCESS: u32 = 0x001F_01FF; + let entry = EXPLICIT_ACCESS_W { + grfAccessPermissions: FILE_ALL_ACCESS, + grfAccessMode: GRANT_ACCESS, + grfInheritance: 0, + Trustee: TRUSTEE_W { + pMultipleTrustee: std::ptr::null_mut(), + MultipleTrusteeOperation: 0, + TrusteeForm: TRUSTEE_IS_SID, + TrusteeType: 0, + // With TRUSTEE_IS_SID the name pointer is the SID itself. + ptstrName: sid.as_mut_ptr().cast(), + }, + }; + let mut dacl: *mut ACL = std::ptr::null_mut(); + let built = unsafe { SetEntriesInAclW(1, &entry, std::ptr::null(), &mut dacl) }; + if built != 0 || dacl.is_null() { + return Err(io::Error::other(format!( + "the owner-only ACL cannot be built (win32 error {built:#x})" + ))); + } + // A verbatim spelling: the named-security calls accept \\?\ paths, and a + // home beyond MAX_PATH must still be restrictable. + let mut wide = layerfs_persistence::verbatim_wide(path); + let set = unsafe { + SetNamedSecurityInfoW( + wide.as_mut_ptr(), + SE_FILE_OBJECT, + DACL_SECURITY_INFORMATION | PROTECTED_DACL_SECURITY_INFORMATION, + std::ptr::null_mut(), + std::ptr::null_mut(), + dacl, + std::ptr::null(), + ) + }; + unsafe { LocalFree(dacl as HLOCAL) }; + if set != 0 { + return Err(io::Error::other(format!( + "the owner-only ACL cannot be set on {}: win32 error {set:#x}", + path.display() + ))); + } + Ok(()) +} + +/// Refuses `path` when another local user can read it: its DACL allows +/// Everyone, Builtin\Users or Authenticated Users, or it has no DACL. +pub fn require_owner_only(path: &Path) -> io::Result<()> { + use windows_sys::Win32::Foundation::{LocalFree, HLOCAL}; + use windows_sys::Win32::Security::Authorization::{GetNamedSecurityInfoW, SE_FILE_OBJECT}; + use windows_sys::Win32::Security::{ + AclSizeInformation, EqualSid, GetAce, GetAclInformation, GetSecurityDescriptorDacl, + WinAuthenticatedUserSid, WinBuiltinUsersSid, WinWorldSid, ACL, DACL_SECURITY_INFORMATION, + PSECURITY_DESCRIPTOR, PSID, + }; + let shared = |path: &Path| { + io::Error::new( + io::ErrorKind::Unsupported, + format!( + "{} grants access beyond its owner; the Store's privacy \ + boundary needs an owner-only entry", + path.display() + ), + ) + }; + let mut wide = layerfs_persistence::verbatim_wide(path); + let mut dacl: *mut ACL = std::ptr::null_mut(); + let mut descriptor: PSECURITY_DESCRIPTOR = std::ptr::null_mut(); + let read = unsafe { + GetNamedSecurityInfoW( + wide.as_mut_ptr(), + SE_FILE_OBJECT, + DACL_SECURITY_INFORMATION, + std::ptr::null_mut(), + std::ptr::null_mut(), + &mut dacl, + std::ptr::null_mut(), + &mut descriptor, + ) + }; + if read != 0 { + return Err(io::Error::other(format!( + "the ACL of {} cannot be read: win32 error {read:#x}", + path.display() + ))); + } + // The ACL points into the descriptor; read every ACE before freeing it. + let verdict = (|| { + let mut present = 0; + let mut entry: *mut ACL = std::ptr::null_mut(); + let mut defaulted = 0; + if unsafe { + GetSecurityDescriptorDacl(descriptor, &mut present, &mut entry, &mut defaulted) + } == 0 + { + return Err(io::Error::other( + "the security descriptor has no DACL field", + )); + } + if present == 0 || entry.is_null() { + return Err(shared(path)); + } + let mut size = windows_sys::Win32::Security::ACL_SIZE_INFORMATION::default(); + if unsafe { + GetAclInformation( + entry, + &mut size as *mut windows_sys::Win32::Security::ACL_SIZE_INFORMATION + as *mut core::ffi::c_void, + std::mem::size_of::() as u32, + AclSizeInformation, + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + let broad = [ + well_known(WinWorldSid)?, + well_known(WinBuiltinUsersSid)?, + well_known(WinAuthenticatedUserSid)?, + ]; + for index in 0..size.AceCount { + let mut ace: *mut core::ffi::c_void = std::ptr::null_mut(); + if unsafe { GetAce(entry, index, &mut ace) } == 0 || ace.is_null() { + return Err(io::Error::last_os_error()); + } + // repr(C) ACCESS_ALLOWED_ACE: 4-byte header, u32 mask, then the + // SID; only an allow ACE (type 0) can widen access. + let header = unsafe { *(ace as *const windows_sys::Win32::Security::ACE_HEADER) }; + if header.AceType != 0 { + continue; + } + let sid = unsafe { ace.byte_add(8) } as PSID; + if broad + .iter() + .any(|bytes| unsafe { EqualSid(sid, bytes.as_ptr() as PSID) } != 0) + { + return Err(shared(path)); + } + } + Ok(()) + })(); + unsafe { LocalFree(descriptor as HLOCAL) }; + verdict +} diff --git a/engine/crates/layerfs-store-service/src/write/recursive.rs b/engine/crates/layerfs-store-service/src/write/recursive.rs index c9557539f9..60af7ae0c7 100644 --- a/engine/crates/layerfs-store-service/src/write/recursive.rs +++ b/engine/crates/layerfs-store-service/src/write/recursive.rs @@ -29,11 +29,15 @@ impl Frame { } } fn join(path: &str, name: &[u8]) -> Result { - Ok(format!( - "{}/{}", - path.trim_end_matches('/'), - std::str::from_utf8(name).map_err(args::failed)? - )) + let name = std::str::from_utf8(name).map_err(|_| { + args::failed(format!( + "the entry {}/{} has a name outside UTF-8; the SDK cannot spell \ + or remove it until the byte-carrying path lands", + path.trim_end_matches('/'), + String::from_utf8_lossy(name) + )) + })?; + Ok(format!("{}/{}", path.trim_end_matches('/'), name)) } impl Mutation<'_> { pub fn remove(&mut self, path: &str) -> Result<()> { diff --git a/engine/crates/layerfs-store-service/tests/kernel_guard.rs b/engine/crates/layerfs-store-service/tests/kernel_guard.rs index d9050f370d..540d8f7581 100644 --- a/engine/crates/layerfs-store-service/tests/kernel_guard.rs +++ b/engine/crates/layerfs-store-service/tests/kernel_guard.rs @@ -1,11 +1,15 @@ //! Direct Store access is refused `Unsupported` where the Store's locks do -//! not reach every opener (#352 F-020): on macOS, and on Linux for a -//! filesystem outside the allowlist (here tmpfs). The refusal comes before any -//! file is opened: the Store file's bytes are unchanged and no overlay -//! template is created. +//! not reach every opener (#352 F-020): on macOS always, on Linux for a +//! filesystem outside the allowlist (here tmpfs), and on Windows for a +//! spelling outside the NTFS-on-local-volume allowlist (here UNC). The +//! refusal comes before any file is opened: the Store file's bytes are +//! unchanged and no overlay template is created. use layerfs_store_service::{OpenRefusal, StoreService}; -use std::path::{Path, PathBuf}; +use std::path::Path; +#[cfg(any(target_os = "macos", target_os = "linux"))] +use std::path::PathBuf; +#[cfg(any(target_os = "macos", target_os = "linux"))] fn refused(directory: &Path) -> (String, Vec, Vec) { let store = directory.join("store.sqlite"); std::fs::write(&store, b"not opened").unwrap(); @@ -24,13 +28,14 @@ fn refused(directory: &Path) -> (String, Vec, Vec) { (reason, before, std::fs::read(&store).unwrap()) } +#[cfg(any(target_os = "macos", target_os = "linux"))] fn scratch(parent: &Path, label: &str) -> PathBuf { let directory = parent.join(format!("layerfs-guard-{label}-{}", std::process::id())); std::fs::create_dir_all(&directory).unwrap(); directory } -#[cfg(not(target_os = "linux"))] +#[cfg(target_os = "macos")] #[test] fn direct_access_is_unsupported_off_linux() { let directory = scratch(&std::env::temp_dir(), "host"); @@ -40,6 +45,27 @@ fn direct_access_is_unsupported_off_linux() { std::fs::remove_dir_all(directory).unwrap(); } +#[cfg(target_os = "windows")] +#[test] +fn direct_access_is_unsupported_on_a_non_local_spelling() { + // A UNC spelling never reaches the NTFS-on-local-volume allowlist; the + // refusal comes before any file is opened, so no template exists to + // leave behind. + let store = Path::new(r"\\localhost\c$\layerfs-guard-nonlocal\store.sqlite"); + let template = Path::new(r"\\localhost\c$\layerfs-guard-nonlocal\overlay.sqlite"); + match StoreService::open_direct(store, b"binding", [7; 32], template) { + Err(OpenRefusal::Unsupported(reason)) => { + assert!(reason.0.contains("cannot be examined"), "{reason}") + } + Err(other) => panic!("expected Unsupported, got {other:?}"), + Ok(_) => panic!("expected Unsupported, the Store opened"), + } + assert!( + !template.exists(), + "no overlay template before the guard passes" + ); +} + #[cfg(target_os = "linux")] #[test] fn direct_access_is_unsupported_on_a_filesystem_outside_the_allowlist() { diff --git a/engine/crates/layerfs-store-service/tests/overlay_backing_windows.rs b/engine/crates/layerfs-store-service/tests/overlay_backing_windows.rs new file mode 100644 index 0000000000..1e8318e6d3 --- /dev/null +++ b/engine/crates/layerfs-store-service/tests/overlay_backing_windows.rs @@ -0,0 +1,79 @@ +//! The Windows overlay backing's last-owner deletion: the same identity +//! discipline as Linux, through the audited Windows identity. A dangling +//! `Weak` stands in for the workspace pool — `Drop` skips its callback when +//! the pool is gone, which is exactly the unit under test. +#![cfg(target_os = "windows")] +use std::sync::{Arc, Weak}; + +fn scratch(label: &str) -> std::path::PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-backing-{}-{}-{}", + label, + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|span| span.as_nanos()) + .unwrap_or(0) + )); + std::fs::create_dir_all(&directory).unwrap(); + directory +} + +/// The file goes away when the last owner releases: identity matches, the +/// removal is authorized, and closing the retained handle completes it. +#[test] +fn the_last_owner_deletes_a_matching_overlay_file() { + let directory = scratch("delete"); + let path = directory.join("overlay.sqlite"); + std::fs::write(&path, b"overlay body").unwrap(); + let backing = Arc::new( + layerfs_store_service::overlay::backing::Backing::new( + path.clone(), + 1, + [7; 32], + Weak::::new(), + ) + .unwrap(), + ); + assert!(path.exists(), "the file stays while its owner lives"); + backing.terminal(); + backing.closed(); + drop(backing); + let mut gone = false; + for _ in 0..40 { + if !path.exists() { + gone = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + assert!(gone, "the last owner's release deletes the file"); + std::fs::remove_dir_all(directory).unwrap(); +} + +/// A file swapped in behind the backing's back no longer matches the +/// retained identity: the removal is refused and the impostor stays. +#[test] +fn a_swapped_in_file_is_refused_the_deletion() { + let directory = scratch("swap"); + let path = directory.join("overlay.sqlite"); + std::fs::write(&path, b"overlay body").unwrap(); + let backing = Arc::new( + layerfs_store_service::overlay::backing::Backing::new( + path.clone(), + 1, + [7; 32], + Weak::::new(), + ) + .unwrap(), + ); + // Replace the entry at the path with a different file (different volume + // index), so the path no longer names the acquisition's file. + std::fs::remove_file(&path).unwrap(); + std::fs::write(&path, b"impostor").unwrap(); + backing.terminal(); + backing.closed(); + drop(backing); + assert!(path.exists(), "the impostor is not deleted"); + std::fs::remove_dir_all(directory).unwrap(); +} diff --git a/engine/crates/layerfs-store-service/tests/volume_guard_long_paths_windows.rs b/engine/crates/layerfs-store-service/tests/volume_guard_long_paths_windows.rs new file mode 100644 index 0000000000..b78c0b330e --- /dev/null +++ b/engine/crates/layerfs-store-service/tests/volume_guard_long_paths_windows.rs @@ -0,0 +1,68 @@ +//! The volume guard's final path beyond the fixed buffer: a deep local +//! directory still passes through the regrowth, and a deep share spelling +//! is still refused. The old fixed-array regrowth could never fit a longer +//! requirement and left a zeroed buffer on which the `\\?\UNC\` refusal +//! silently passed (round-6 D1 F1). The share spelling is derived from the +//! temporary directory's own drive letter, so the discriminating half runs +//! on every host — a hard-coded `D:` silently skipped it wherever the +//! temporary directory lives elsewhere (round-8 F1 F1). +#![cfg(target_os = "windows")] +use layerfs_store_service::kernel_guard; +use std::path::PathBuf; + +/// A directory chain whose final path exceeds 1023 UTF-16 units. The +/// verbatim spelling is built textually — `PathBuf::from(r"\\?\").join` +/// drops the prefix for an absolute component (round-8 F1 F3) — so +/// creation works beyond MAX_PATH on every host, relaxed or not. +fn deep(parent: &std::path::Path, levels: usize) -> (PathBuf, PathBuf) { + let component = |level: usize| format!("guard-{level:03}-{}", "x".repeat(190)); + let mut plain = parent.to_path_buf(); + let mut verbatim = PathBuf::from(format!(r"\\?\{}", parent.display())); + for level in 0..levels { + plain.push(component(level)); + verbatim.push(component(level)); + } + std::fs::create_dir_all(&verbatim).unwrap(); + (plain, verbatim) +} + +#[test] +fn a_deep_local_directory_still_names_its_volume() { + let base = std::env::temp_dir().join(format!("layerfs-volguard-local-{}", std::process::id())); + std::fs::create_dir_all(&base).unwrap(); + let (plain, verbatim) = deep(&base, 6); // ~1160 units of tail + let filesystem = + kernel_guard::check(&plain.join("store.sqlite")).expect("the local deep path passes"); + assert_eq!(filesystem.to_lowercase(), "ntfs"); + let _ = std::fs::remove_dir_all(&verbatim); + let _ = std::fs::remove_dir_all(&base); +} + +#[test] +fn a_deep_share_spelling_is_still_refused() { + // The same physical tree addressed through the local machine's + // administrative share for the temporary directory's own drive: the + // guard's final path is `\\?\UNC\…` and, past the fixed buffer, must + // still be recognised as a share. + let base = std::env::temp_dir().join(format!("layerfs-volguard-share-{}", std::process::id())); + std::fs::create_dir_all(&base).unwrap(); + let (plain, verbatim) = deep(&base, 6); + let spelling = plain.to_string_lossy().to_string(); + let units: Vec = spelling.chars().collect(); + assert!( + units.len() > 2 && units[1] == ':', + "this test needs a drive-letter temporary directory, got {spelling}" + ); + let drive = units[0].to_ascii_uppercase(); + let tail: String = units[3..].iter().collect(); + let share = PathBuf::from(format!(r"\\localhost\{drive}$\{tail}")); + let refusal = + kernel_guard::check(&share.join("store.sqlite")).expect_err("a share spelling is refused"); + assert!( + refusal.0.contains("network share"), + "the refusal names the share: {}", + refusal.0 + ); + let _ = std::fs::remove_dir_all(&verbatim); + let _ = std::fs::remove_dir_all(&base); +} diff --git a/engine/crates/layerfs-store-service/tests/windows_acl_windows.rs b/engine/crates/layerfs-store-service/tests/windows_acl_windows.rs new file mode 100644 index 0000000000..fa3199305b --- /dev/null +++ b/engine/crates/layerfs-store-service/tests/windows_acl_windows.rs @@ -0,0 +1,55 @@ +//! The owner-only DACL boundary on Windows: what this process creates is +//! restricted to its owner, a shared entry is refused, and the owner keeps +//! working access. +#![cfg(target_os = "windows")] +use layerfs_store_service::windows_acl::{require_owner_only, restrict_to_owner}; + +fn scratch(label: &str) -> std::path::PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-acl-{label}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&directory).unwrap(); + directory +} + +#[test] +fn a_restricted_entry_is_owner_only_and_still_usable() { + let root = scratch("owner"); + let file = root.join("record"); + std::fs::write(&file, b"body").unwrap(); + restrict_to_owner(&file).unwrap(); + restrict_to_owner(&root).unwrap(); + require_owner_only(&file).unwrap(); + require_owner_only(&root).unwrap(); + // The owner keeps full access after the restriction. + std::fs::write(&file, b"second").unwrap(); + assert_eq!(std::fs::read(&file).unwrap(), b"second"); + std::fs::remove_dir_all(&root).unwrap(); +} + +#[test] +fn an_entry_shared_with_everyone_is_refused() { + let root = scratch("shared"); + let file = root.join("record"); + std::fs::write(&file, b"body").unwrap(); + let granted = std::process::Command::new("icacls") + .arg(&file) + .args(["/grant", "*S-1-1-0:(R)"]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .unwrap(); + assert!(granted.success(), "icacls grants Everyone read"); + let error = require_owner_only(&file).unwrap_err(); + assert_eq!( + error.kind(), + std::io::ErrorKind::Unsupported, + "a shared entry never passes the privacy boundary: {error}" + ); + std::fs::remove_dir_all(&root).unwrap(); +} diff --git a/tools/check_product_boundary.py b/tools/check_product_boundary.py index c4159c3c12..6f64d89bdd 100644 --- a/tools/check_product_boundary.py +++ b/tools/check_product_boundary.py @@ -21,16 +21,20 @@ # comment in a lint name (`unsafe_code`) is not the bare word, so attr lines # never trip the scan once comments are stripped. UNSAFE_AUDITED_MODULE = { - "layerfs-storage": "src/encoding/codec.rs", - "layerfs-persistence": "src/backend/sqlite/file_control.rs", + "layerfs-storage": ("src/encoding/codec.rs",), + "layerfs-persistence": ( + "src/backend/sqlite/file_control.rs", + "src/backend/sqlite/windows_lane.rs", + ), + "layerfs-project": ("src/import/windows_identity.rs",), + "layerfs-overlay": ("src/database/windows_identity.rs",), + "layerfs-node": ("src/discovery.rs", "src/discovery_windows.rs"), + "layerfs-store-service": ("src/windows_acl.rs",), } UNSAFE_FREE_CRATES = ( "layerfs-content", "layerfs-telemetry", - "layerfs-project", - "layerfs-overlay", "layerfs-workspace", - "layerfs-store-service", "layerfs-fuse", "layerfs-daemon", "layerfs-api", @@ -42,10 +46,11 @@ "layerfs-content": "#![forbid(unsafe_code)]", "layerfs-telemetry": "#![forbid(unsafe_code)]", "layerfs-persistence": "#![deny(unsafe_code)]", - "layerfs-project": "#![forbid(unsafe_code)]", - "layerfs-overlay": "#![forbid(unsafe_code)]", + "layerfs-project": "#![deny(unsafe_code)]", + "layerfs-overlay": "#![deny(unsafe_code)]", + "layerfs-node": "#![deny(unsafe_code)]", "layerfs-workspace": "#![forbid(unsafe_code)]", - "layerfs-store-service": "#![forbid(unsafe_code)]", + "layerfs-store-service": "#![deny(unsafe_code)]", "layerfs-fuse": "#![forbid(unsafe_code)]", "layerfs-daemon": "#![forbid(unsafe_code)]", "layerfs-api": "#![forbid(unsafe_code)]", @@ -325,8 +330,9 @@ def unsafe_violations(path, source): return found relative = Path(*path.parts[path.parts.index("src") :]) code = "\n".join(line.split("//", 1)[0] for line in source.splitlines()) - if UNSAFE.search(code) and (crate in UNSAFE_FREE_CRATES or str(relative) != UNSAFE_AUDITED_MODULE[crate]): - found.append((1, "unsafe outside the audited module boundary; see core/AGENTS.md")) + if UNSAFE.search(code): + if crate in UNSAFE_FREE_CRATES or relative.as_posix() not in UNSAFE_AUDITED_MODULE[crate]: + found.append((1, "unsafe outside the audited module boundary; see core/AGENTS.md")) if relative == Path("src/lib.rs") and UNSAFE_ROOT_ATTR[crate] not in source: found.append((1, f"crate root must declare {UNSAFE_ROOT_ATTR[crate]}")) return found diff --git a/tools/test_check_product_boundary.py b/tools/test_check_product_boundary.py index ec4196fbff..8fbb57b40b 100644 --- a/tools/test_check_product_boundary.py +++ b/tools/test_check_product_boundary.py @@ -83,6 +83,26 @@ def test_persistence_seal_ffi_is_limited_to_the_authorized_module(self): self.assertFalse(unsafe_violations(base / "lib.rs", "#![deny(unsafe_code)]")) self.assertTrue(unsafe_violations(base / "lib.rs", "pub mod store;")) + def test_node_discovery_ffi_is_limited_to_the_two_audited_modules(self): + base = Path("engine/crates/layerfs-node/src") + ffi = "fn turn() { unsafe { WaitForSingleObject(handle, 0); } }" + # Both discovery twins are audited: unix flock/geteuid, Windows mutexes. + self.assertFalse(unsafe_violations(base / "discovery.rs", ffi)) + self.assertFalse(unsafe_violations(base / "discovery_windows.rs", ffi)) + for sibling in ["dispatch.rs", "direct.rs", "project.rs", "job.rs"]: + self.assertTrue(unsafe_violations(base / sibling, ffi)) + self.assertFalse(unsafe_violations(base / "lib.rs", "#![deny(unsafe_code)]\nmod dispatch;")) + self.assertTrue(unsafe_violations(base / "lib.rs", "mod dispatch;")) + + def test_store_service_acl_ffi_is_limited_to_the_audited_module(self): + base = Path("engine/crates/layerfs-store-service/src") + ffi = "fn seal() { unsafe { SetNamedSecurityInfoW(); } }" + self.assertFalse(unsafe_violations(base / "windows_acl.rs", ffi)) + for sibling in ["install.rs", "kernel_guard.rs", "store_service.rs", "overlay/backing.rs"]: + self.assertTrue(unsafe_violations(base / sibling, ffi)) + self.assertFalse(unsafe_violations(base / "lib.rs", "#![deny(unsafe_code)]\npub mod store;")) + self.assertTrue(unsafe_violations(base / "lib.rs", "pub mod store;")) + def test_retired_transport_paths_cannot_return(self): for relative in ( "layerfs-api/src/client/call.rs", @@ -190,7 +210,7 @@ def test_store_service_engine_edges(self): ) root = Path("engine/crates/layerfs-store-service/src/lib.rs") self.assertTrue(unsafe_violations(root, "pub mod store;")) - self.assertFalse(unsafe_violations(root, "#![forbid(unsafe_code)]\npub mod store;")) + self.assertFalse(unsafe_violations(root, "#![deny(unsafe_code)]\npub mod store;")) self.assertFalse( dependency_violations( '[package]\nname="layerfs-persistence"\n[dependencies]\nlayerfs-telemetry={path="../layerfs-telemetry"}\n' @@ -384,7 +404,7 @@ def test_shipped_sql_and_external_tests_scope(self): path.parent.mkdir(parents=True, exist_ok=True) path.write_text("") self.assertEqual( - [str(p.relative_to(core / "crates" / "store")) for p in production_files(core.parent)], + [p.relative_to(core / "crates" / "store").as_posix() for p in production_files(core.parent)], ["sql/schema.sql", "src/lib.rs", "src/sql/query.sql"], ) @@ -398,7 +418,7 @@ def test_flat_api_product_scope(self): path.write_text("") (package / "Cargo.toml").write_text('[package]\nname="layerfs-api"\n') self.assertEqual( - [str(p.relative_to(package)) for p in production_files(root)], + [p.relative_to(package).as_posix() for p in production_files(root)], ["src/fs/read.rs", "src/lib.rs"], ) @@ -432,11 +452,11 @@ def test_skip_sdk_leaves_only_the_sdk_out(self): path.parent.mkdir(parents=True, exist_ok=True) path.write_text("") (root / "engine/crates/layerfs-node/Cargo.toml").write_text('[package]\nname="layerfs-node"\n') - everything = [str(p.relative_to(root)) for p in production_files(core.parent)] - self.assertIn("engine/crates/layerfs-node/src/lib.rs", everything) + everything = [p.relative_to(root).as_posix() for p in production_files(core.parent)] + self.assertIn("engine/crates/layerfs-node/src/lib.rs", [Path(p).as_posix() for p in everything]) self.assertIn("api/agent/src/index.js", everything) self.assertEqual( - [str(p.relative_to(root)) for p in production_files(core.parent, include_sdk=False)], + [p.relative_to(root).as_posix() for p in production_files(core.parent, include_sdk=False)], ["core/crates/store/src/lib.rs"], )