From cbdf389c2bd481e6bc43ed40499dbbc3380e1a15 Mon Sep 17 00:00:00 2001 From: Itodo-S Date: Fri, 26 Jun 2026 23:41:43 +0100 Subject: [PATCH] feat(tooling): contract state export, migration, and dependency analysis (#496, #497, #499) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a cohesive contract-lifecycle tooling suite under scripts/, each a dependency-free TypeScript CLI (runs on Node >= 22 via --experimental-strip-types) with a pure, unit-tested core. #499 — scripts/state-exporter/ - Export user positions, pool configs, and governance to JSON/CSV. - Full snapshot + incremental change-data-capture (diff vs previous snapshot). - Verifiable: SHA-256 checksum + row-count manifest, integrity check command. - gzip compression. Pluggable StateSource (file source by default). #497 — scripts/state-migrate/ - Declarative schema migrations: add/rename/remove/transform field. - Dry-run (never mutates), atomic checkpoint rollback on op error or failed verification, post-migration integrity checks, monotonic versioned history, heuristic per-op cost estimate, CLI. #496 — scripts/dependency-analyzer/ - Scans the Rust/Soroban workspace, builds a contract dependency graph (call / import / library edges), transitive upgrade-impact + risk level, cycle detection, Graphviz DOT + JSON output. Verified against the real workspace (28 crates). Tests: 29 passing across the three cores node --experimental-strip-types --test scripts/**/**.test.ts Each tool's README documents covered acceptance criteria and the heavier infra follow-ups intentionally left out of this core (live RPC source, Parquet/SQL, S3/GCS, scheduling; on-chain gas metering; interactive D3 dashboard + CI wiring). --- scripts/dependency-analyzer/README.md | 50 +++++ scripts/dependency-analyzer/analyzer.test.ts | 95 +++++++++ scripts/dependency-analyzer/analyzer.ts | 165 +++++++++++++++ scripts/dependency-analyzer/index.ts | 103 +++++++++ scripts/dependency-analyzer/types.ts | 31 +++ scripts/state-exporter/README.md | 53 +++++ scripts/state-exporter/exporter.test.ts | 105 +++++++++ scripts/state-exporter/exporter.ts | 194 +++++++++++++++++ scripts/state-exporter/index.ts | 70 ++++++ scripts/state-exporter/types.ts | 71 +++++++ scripts/state-migrate/README.md | 63 ++++++ scripts/state-migrate/index.ts | 79 +++++++ scripts/state-migrate/migrator.test.ts | 170 +++++++++++++++ scripts/state-migrate/migrator.ts | 212 +++++++++++++++++++ scripts/state-migrate/types.ts | 53 +++++ 15 files changed, 1514 insertions(+) create mode 100644 scripts/dependency-analyzer/README.md create mode 100644 scripts/dependency-analyzer/analyzer.test.ts create mode 100644 scripts/dependency-analyzer/analyzer.ts create mode 100644 scripts/dependency-analyzer/index.ts create mode 100644 scripts/dependency-analyzer/types.ts create mode 100644 scripts/state-exporter/README.md create mode 100644 scripts/state-exporter/exporter.test.ts create mode 100644 scripts/state-exporter/exporter.ts create mode 100644 scripts/state-exporter/index.ts create mode 100644 scripts/state-exporter/types.ts create mode 100644 scripts/state-migrate/README.md create mode 100644 scripts/state-migrate/index.ts create mode 100644 scripts/state-migrate/migrator.test.ts create mode 100644 scripts/state-migrate/migrator.ts create mode 100644 scripts/state-migrate/types.ts diff --git a/scripts/dependency-analyzer/README.md b/scripts/dependency-analyzer/README.md new file mode 100644 index 0000000..a292b8f --- /dev/null +++ b/scripts/dependency-analyzer/README.md @@ -0,0 +1,50 @@ +# Contract Dependency Analyzer (#496) + +Maps cross-contract dependencies across the Rust/Soroban workspace and reports +the blast radius of upgrading a given contract. + +## Usage + +```bash +# Build the graph (JSON + Graphviz DOT) +node --experimental-strip-types scripts/dependency-analyzer/index.ts \ + --root stellar-lend --out graph.json --dot graph.dot + +# Render the DOT (optional) +dot -Tsvg graph.dot -o graph.svg + +# Upgrade-impact report for one contract +node --experimental-strip-types scripts/dependency-analyzer/index.ts \ + --root stellar-lend --impact oracle +``` + +Each crate (`Cargo.toml` `[package].name`) is a node. Edges are detected as: +- **call** — a generated cross-contract client (`Client`) +- **import** — `use ` / `mod ` / `::` +- **library** — workspace crate listed in `[dependencies]` + +## Impact analysis + +`--impact ` returns the direct and transitive **dependents** (what an +upgrade affects) plus a `riskLevel` derived from the blast radius. The engine +also detects dependency cycles. + +## Tests + +```bash +node --experimental-strip-types --test scripts/dependency-analyzer/analyzer.test.ts +``` + +## Covered acceptance criteria + +- Parse contract sources + imports to build a dependency graph. +- Detect direct calls, imports, and library (crate) links. +- Impact analysis: which contracts are affected by upgrading a specific contract. +- Cycle detection. +- Output: Graphviz DOT + JSON; per-contract upgrade-impact report with risk level. + +## Follow-ups (out of this PR's core) + +- Interactive web visualization (`web/dashboard/dependency-graph/`, D3.js/vis.js). +- Changed-interface / storage-slot impact and test-impact mapping. +- CI integration to auto-generate the graph on PRs. diff --git a/scripts/dependency-analyzer/analyzer.test.ts b/scripts/dependency-analyzer/analyzer.test.ts new file mode 100644 index 0000000..2d2e204 --- /dev/null +++ b/scripts/dependency-analyzer/analyzer.test.ts @@ -0,0 +1,95 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + buildGraph, + dependencies, + dependents, + detectCycles, + detectDependencies, + impactReport, + toDOT, +} from "./analyzer.ts"; +import { type ModuleSource } from "./types.ts"; + +function modules(): ModuleSource[] { + return [ + // lending calls oracle (client) and imports shared + { + name: "lending", + source: `use shared::types::Asset;\nlet px = OracleClient::new(&env, &id).price();`, + }, + // oracle imports shared + { name: "oracle", source: `use shared::math;\npub fn price() {}` }, + // shared depends on nothing + { name: "shared", source: `pub struct Asset {}` }, + // governance calls lending + { name: "governance", source: `let c = LendingClient::new(&env, &id);` }, + ]; +} + +test("detectDependencies finds call and import edges", () => { + const edges = detectDependencies(modules()[0], ["lending", "oracle", "shared", "governance"]); + const oracle = edges.find((e) => e.to === "oracle"); + const shared = edges.find((e) => e.to === "shared"); + assert.equal(oracle?.kind, "call"); + assert.equal(shared?.kind, "import"); + // does not depend on itself + assert.equal(edges.some((e) => e.to === "lending"), false); +}); + +test("detectDependencies includes library deps", () => { + const edges = detectDependencies(modules()[2], ["lending", "oracle", "shared"], ["soroban-sdk"]); + assert.ok(edges.some((e) => e.to === "soroban-sdk" && e.kind === "library")); +}); + +test("buildGraph assembles nodes and edges", () => { + const g = buildGraph(modules()); + assert.ok(g.nodes.includes("lending")); + assert.ok(g.edges.some((e) => e.from === "lending" && e.to === "oracle" && e.kind === "call")); + assert.ok(g.edges.some((e) => e.from === "governance" && e.to === "lending")); +}); + +test("dependencies returns transitive out-edges", () => { + const g = buildGraph(modules()); + // lending -> oracle -> shared, lending -> shared + assert.deepEqual(dependencies(g, "lending"), ["oracle", "shared"]); +}); + +test("dependents returns transitive in-edges (upgrade impact)", () => { + const g = buildGraph(modules()); + // who is affected if shared is upgraded? lending, oracle, and governance (via lending) + assert.deepEqual(dependents(g, "shared"), ["governance", "lending", "oracle"]); +}); + +test("impactReport classifies risk by blast radius", () => { + const g = buildGraph(modules()); + const shared = impactReport(g, "shared"); + assert.equal(shared.riskLevel, "high"); // 3 transitive dependents + assert.ok(shared.directDependents.includes("lending")); + assert.ok(shared.directDependents.includes("oracle")); + + const governance = impactReport(g, "governance"); + assert.equal(governance.riskLevel, "low"); // nothing depends on governance + assert.deepEqual(governance.transitiveDependents, []); +}); + +test("detectCycles finds a dependency cycle", () => { + const g = buildGraph([ + { name: "a", source: "BClient::new();" }, + { name: "b", source: "AClient::new();" }, + ]); + const cycles = detectCycles(g); + assert.ok(cycles.length >= 1); +}); + +test("acyclic graph reports no cycles", () => { + assert.deepEqual(detectCycles(buildGraph(modules())), []); +}); + +test("toDOT emits a digraph with styled edges", () => { + const dot = toDOT(buildGraph(modules())); + assert.ok(dot.startsWith("digraph contract_dependencies")); + assert.ok(dot.includes('"lending" -> "oracle"')); + assert.ok(dot.includes("style=solid")); // call edge + assert.ok(dot.includes("style=dashed")); // import edge +}); diff --git a/scripts/dependency-analyzer/analyzer.ts b/scripts/dependency-analyzer/analyzer.ts new file mode 100644 index 0000000..52971f4 --- /dev/null +++ b/scripts/dependency-analyzer/analyzer.ts @@ -0,0 +1,165 @@ +/** + * Core dependency-graph analysis for contract upgrade-impact assessment (#496). + * + * Pure functions: detect cross-contract references in source, build a graph, + * compute transitive dependents/dependencies (upgrade impact), detect cycles, + * and emit Graphviz DOT. The interactive web visualization and CI wiring are + * documented follow-ups; this is the analysis engine. + */ + +import { + type DependencyEdge, + type DependencyGraph, + type ImpactReport, + type ModuleSource, +} from "./types.ts"; + +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Detect dependency edges from `module` to other known contracts by scanning + * its source. Heuristics tuned for Rust/Soroban: + * - `import` : `use ` / `::` / `mod ` (interface/module imports) + * - `call` : `Client` (generated cross-contract client = a direct call) + * - `library` : declared elsewhere (Cargo deps), supplied via `libraryDeps` + */ +export function detectDependencies( + module: ModuleSource, + knownContracts: string[], + libraryDeps: string[] = [], +): DependencyEdge[] { + const edges: DependencyEdge[] = []; + const seen = new Set(); + + for (const target of knownContracts) { + if (target === module.name) continue; + const t = escapeRegExp(target); + // Soroban generated clients are PascalCase (`OracleClient`) while crate/node + // names are snake_case (`oracle`); compact underscores and match + // case-insensitively so `oracle` detects `OracleClient` and `stellar_lend` + // detects `StellarLendClient`. + const compact = escapeRegExp(target.replace(/_/g, "")); + + const callRe = new RegExp(`\\b${compact}Client\\b`, "i"); + const importRe = new RegExp(`\\b(use|mod)\\s+${t}\\b|\\b${t}::`); + + let kind: DependencyEdge["kind"] | null = null; + if (callRe.test(module.source)) kind = "call"; + else if (importRe.test(module.source)) kind = "import"; + + if (kind) { + const id = `${target}:${kind}`; + if (!seen.has(id)) { + seen.add(id); + edges.push({ from: module.name, to: target, kind }); + } + } + } + + for (const lib of libraryDeps) { + if (lib === module.name) continue; + edges.push({ from: module.name, to: lib, kind: "library" }); + } + + return edges; +} + +/** Build the full dependency graph from a set of module sources. */ +export function buildGraph( + modules: ModuleSource[], + libraryDepsByModule: Record = {}, +): DependencyGraph { + const names = modules.map((m) => m.name); + const nodeSet = new Set(names); + const edges: DependencyEdge[] = []; + for (const m of modules) { + for (const e of detectDependencies(m, names, libraryDepsByModule[m.name] ?? [])) { + edges.push(e); + nodeSet.add(e.to); + } + } + return { nodes: [...nodeSet].sort(), edges }; +} + +function neighbors(graph: DependencyGraph, node: string, direction: "out" | "in"): string[] { + return graph.edges + .filter((e) => (direction === "out" ? e.from === node : e.to === node)) + .map((e) => (direction === "out" ? e.to : e.from)); +} + +function transitiveClosure(graph: DependencyGraph, start: string, direction: "out" | "in"): string[] { + const visited = new Set(); + const stack = [...neighbors(graph, start, direction)]; + while (stack.length) { + const n = stack.pop() as string; + if (visited.has(n) || n === start) continue; + visited.add(n); + stack.push(...neighbors(graph, n, direction)); + } + return [...visited].sort(); +} + +/** What `target` depends on (transitively). */ +export function dependencies(graph: DependencyGraph, target: string): string[] { + return transitiveClosure(graph, target, "out"); +} + +/** What depends on `target` (transitively) — i.e. what an upgrade impacts. */ +export function dependents(graph: DependencyGraph, target: string): string[] { + return transitiveClosure(graph, target, "in"); +} + +/** Upgrade-impact report for a contract. */ +export function impactReport(graph: DependencyGraph, target: string): ImpactReport { + const direct = [...new Set(neighbors(graph, target, "in"))].sort(); + const transitive = dependents(graph, target); + const riskLevel = transitive.length >= 3 ? "high" : transitive.length >= 1 ? "medium" : "low"; + return { + target, + directDependents: direct, + transitiveDependents: transitive, + riskLevel, + }; +} + +/** Detect dependency cycles (each returned array is a cycle path). */ +export function detectCycles(graph: DependencyGraph): string[][] { + const cycles: string[][] = []; + const state = new Map(); // 0=visiting,1=done + const path: string[] = []; + + const visit = (node: string): void => { + state.set(node, 0); + path.push(node); + for (const next of neighbors(graph, node, "out")) { + if (!state.has(next)) visit(next); + else if (state.get(next) === 0) { + const idx = path.indexOf(next); + if (idx >= 0) cycles.push([...path.slice(idx), next]); + } + } + path.pop(); + state.set(node, 1); + }; + + for (const node of graph.nodes) if (!state.has(node)) visit(node); + return cycles; +} + +/** Emit Graphviz DOT, with edge styling per dependency kind. */ +export function toDOT(graph: DependencyGraph): string { + const style: Record = { + call: "solid", + import: "dashed", + library: "dotted", + }; + const lines = ["digraph contract_dependencies {", " rankdir=LR;", " node [shape=box];"]; + for (const n of graph.nodes) lines.push(` "${n}";`); + for (const e of graph.edges) { + lines.push(` "${e.from}" -> "${e.to}" [style=${style[e.kind]} label="${e.kind}"];`); + } + lines.push("}"); + return lines.join("\n"); +} diff --git a/scripts/dependency-analyzer/index.ts b/scripts/dependency-analyzer/index.ts new file mode 100644 index 0000000..623d681 --- /dev/null +++ b/scripts/dependency-analyzer/index.ts @@ -0,0 +1,103 @@ +#!/usr/bin/env node +/** + * dependency-analyzer CLI (#496). + * + * Scans a workspace of Rust/Soroban crates, builds a contract dependency graph, + * and reports upgrade impact. Each crate (a `Cargo.toml` with a `[package]` + * name) becomes a node; its `.rs` sources are scanned for cross-contract + * references and its `[dependencies]` table supplies library edges. + * + * Run (Node >= 22, no install): + * node --experimental-strip-types scripts/dependency-analyzer/index.ts \ + * --root stellar-lend --out graph.json --dot graph.dot + * node --experimental-strip-types scripts/dependency-analyzer/index.ts \ + * --root stellar-lend --impact oracle + */ + +import * as fs from "node:fs"; +import * as path from "node:path"; +import { buildGraph, impactReport, detectCycles, toDOT } from "./analyzer.ts"; +import { type ModuleSource } from "./types.ts"; + +function arg(name: string): string | undefined { + const i = process.argv.indexOf(`--${name}`); + return i >= 0 ? process.argv[i + 1] : undefined; +} + +function walk(dir: string, acc: string[] = []): string[] { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (entry.name === "target" || entry.name === "node_modules" || entry.name.startsWith(".")) { + continue; + } + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walk(full, acc); + else acc.push(full); + } + return acc; +} + +function packageName(cargoToml: string): string | null { + const m = /\[package\][\s\S]*?\bname\s*=\s*"([^"]+)"/.exec(cargoToml); + return m ? m[1] : null; +} + +function dependencyNames(cargoToml: string): string[] { + const depsSection = /\[dependencies\]([\s\S]*?)(\n\[|$)/.exec(cargoToml); + if (!depsSection) return []; + return [...depsSection[1].matchAll(/^\s*([A-Za-z0-9_-]+)\s*=/gm)].map((x) => x[1]); +} + +function collectModules(root: string): { + modules: ModuleSource[]; + libraryDeps: Record; +} { + const cargoFiles = walk(root).filter((f) => path.basename(f) === "Cargo.toml"); + const modules: ModuleSource[] = []; + const libraryDeps: Record = {}; + for (const cargo of cargoFiles) { + const toml = fs.readFileSync(cargo, "utf8"); + const name = packageName(toml); + if (!name) continue; + const crateDir = path.dirname(cargo); + const rs = walk(crateDir).filter((f) => f.endsWith(".rs")); + const source = rs.map((f) => fs.readFileSync(f, "utf8")).join("\n"); + modules.push({ name, source }); + libraryDeps[name] = dependencyNames(toml); + } + return { modules, libraryDeps }; +} + +function main(): void { + const root = arg("root") ?? "."; + const { modules, libraryDeps } = collectModules(root); + // Only keep library edges that point at another crate in this workspace. + const localNames = new Set(modules.map((m) => m.name)); + const localLibDeps: Record = {}; + for (const [name, deps] of Object.entries(libraryDeps)) { + localLibDeps[name] = deps.filter((d) => localNames.has(d)); + } + + const graph = buildGraph(modules, localLibDeps); + + const impactTarget = arg("impact"); + if (impactTarget) { + console.log(JSON.stringify(impactReport(graph, impactTarget), null, 2)); + return; + } + + const cycles = detectCycles(graph); + if (cycles.length) console.error(`⚠ ${cycles.length} dependency cycle(s) detected`); + + const out = arg("out"); + if (out) fs.writeFileSync(out, JSON.stringify(graph, null, 2)); + const dot = arg("dot"); + if (dot) fs.writeFileSync(dot, toDOT(graph)); + + console.log( + `Analyzed ${modules.length} crates: ${graph.nodes.length} nodes, ${graph.edges.length} edges` + + (cycles.length ? `, ${cycles.length} cycle(s)` : ""), + ); + if (!out && !dot) console.log(toDOT(graph)); +} + +main(); diff --git a/scripts/dependency-analyzer/types.ts b/scripts/dependency-analyzer/types.ts new file mode 100644 index 0000000..8bd6aec --- /dev/null +++ b/scripts/dependency-analyzer/types.ts @@ -0,0 +1,31 @@ +/** + * Types for the contract dependency-graph analyzer (#496). + */ + +export type DependencyKind = "call" | "import" | "library"; + +export interface DependencyEdge { + from: string; + to: string; + kind: DependencyKind; +} + +export interface DependencyGraph { + nodes: string[]; + edges: DependencyEdge[]; +} + +/** A contract/crate node and its source text, used to detect references. */ +export interface ModuleSource { + name: string; + source: string; +} + +export interface ImpactReport { + target: string; + /** Contracts that directly depend on `target`. */ + directDependents: string[]; + /** All contracts transitively affected by upgrading `target`. */ + transitiveDependents: string[]; + riskLevel: "low" | "medium" | "high"; +} diff --git a/scripts/state-exporter/README.md b/scripts/state-exporter/README.md new file mode 100644 index 0000000..f3edd0a --- /dev/null +++ b/scripts/state-exporter/README.md @@ -0,0 +1,53 @@ +# State Exporter (#499) + +Exports structured contract state — user positions, pool configs, governance — +to verifiable, structured files. + +## Usage + +No install needed on Node ≥ 22 (uses `--experimental-strip-types`): + +```bash +# Full snapshot export (JSON) +node --experimental-strip-types scripts/state-exporter/index.ts \ + --in state.json --format json --out export.json + +# CSV, gzip-compressed +node --experimental-strip-types scripts/state-exporter/index.ts \ + --in state.json --format csv --out export.csv --gzip + +# Incremental (change-data-capture) export vs a previous snapshot +node --experimental-strip-types scripts/state-exporter/index.ts \ + --in state.json --since prev.json --out delta.json + +# Verify an export against its manifest (checksum + row count) +node --experimental-strip-types scripts/state-exporter/index.ts \ + --verify export.json +``` + +Every export writes a sidecar `*.manifest.json` with `generatedAt`, format, +`rowCount` (per section + total), and a SHA-256 `checksum` of the payload. + +## Tests + +```bash +node --experimental-strip-types --test scripts/state-exporter/exporter.test.ts +``` + +## Covered acceptance criteria + +- Export user positions (collateral, debt, health factor), pool configs, and + governance proposals/votes/executed actions. +- Formats: JSON and CSV. +- Incremental export (CDC): only rows changed/added since a previous snapshot. +- Full snapshot export on demand. +- Export verification: SHA-256 checksum + row count integrity check. +- Compression: gzip. + +## Follow-ups (intentionally out of this PR's core) + +- Live Soroban RPC `StateSource` (read via contract view functions + events). + The exporter is written against a `StateSource` interface; the default file + source keeps the tool runnable/testable offline. +- Additional formats (Parquet, SQL) and sinks (S3/GCS). +- Scheduled/automated exports (daily/weekly worker) under `services/state-archiver/`. diff --git a/scripts/state-exporter/exporter.test.ts b/scripts/state-exporter/exporter.test.ts new file mode 100644 index 0000000..8fa5817 --- /dev/null +++ b/scripts/state-exporter/exporter.test.ts @@ -0,0 +1,105 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + buildExport, + checksum, + diff, + rowCount, + toCSV, + toJSON, + verifyIntegrity, +} from "./exporter.ts"; +import { type ContractState } from "./types.ts"; + +function sampleState(): ContractState { + return { + positions: [ + { address: "GA...1", collateral: "1000", debt: "400", healthFactor: "2.5" }, + { address: "GA...2", collateral: "500", debt: "0", healthFactor: "max" }, + ], + pools: [{ asset: "USDC", supplyCap: "1000000", interestRateBps: 350, collateralFactorBps: 8000 }], + governance: [ + { proposalId: 1, status: "executed", votesFor: "900", votesAgainst: "100", executed: true }, + ], + }; +} + +test("rowCount totals each section", () => { + const rc = rowCount(sampleState()); + assert.equal(rc.positions, 2); + assert.equal(rc.pools, 1); + assert.equal(rc.governance, 1); + assert.equal(rc.total, 4); +}); + +test("toJSON round-trips to an equal state", () => { + const state = sampleState(); + assert.deepEqual(JSON.parse(toJSON(state)), state); +}); + +test("checksum is deterministic and content-sensitive", () => { + const a = toJSON(sampleState()); + assert.equal(checksum(a), checksum(a)); + assert.notEqual(checksum(a), checksum(a + " ")); +}); + +test("buildExport (json) is integrity-verifiable", () => { + const { payload, manifest } = buildExport(sampleState(), { format: "json" }); + assert.equal(manifest.rowCount.total, 4); + assert.equal(manifest.incremental, false); + const result = verifyIntegrity(payload, manifest); + assert.ok(result.ok, JSON.stringify(result)); +}); + +test("buildExport (csv) row count and integrity", () => { + const { payload, manifest } = buildExport(sampleState(), { format: "csv" }); + assert.equal(manifest.format, "csv"); + assert.equal(manifest.rowCount.total, 4); + const result = verifyIntegrity(payload, manifest); + assert.ok(result.ok, JSON.stringify(result)); +}); + +test("buildExport with gzip round-trips and verifies", () => { + const { payload, manifest } = buildExport(sampleState(), { format: "json", compressed: true }); + assert.equal(manifest.compressed, true); + assert.ok(Buffer.isBuffer(payload)); + assert.ok(verifyIntegrity(payload, manifest).ok); +}); + +test("incremental diff captures only new/changed rows (CDC)", () => { + const prev = sampleState(); + const next = sampleState(); + // Change one position, add a pool, leave governance untouched. + next.positions[0].debt = "450"; + next.pools.push({ asset: "XLM", supplyCap: "5000000", interestRateBps: 200, collateralFactorBps: 7500 }); + + const delta = diff(prev, next); + assert.equal(delta.positions.length, 1); + assert.equal(delta.positions[0].address, "GA...1"); + assert.equal(delta.pools.length, 1); + assert.equal(delta.pools[0].asset, "XLM"); + assert.equal(delta.governance.length, 0); +}); + +test("buildExport incremental sets manifest + delta row counts", () => { + const prev = sampleState(); + const next = sampleState(); + next.positions[0].debt = "450"; + + const { manifest } = buildExport(next, { format: "json", previous: prev }); + assert.equal(manifest.incremental, true); + assert.equal(manifest.rowCount.total, 1); +}); + +test("integrity check fails on tampered payload", () => { + const { payload, manifest } = buildExport(sampleState(), { format: "json" }); + const tampered = String(payload).replace("1000", "9999"); + assert.equal(verifyIntegrity(tampered, manifest).ok, false); +}); + +test("toCSV emits a section marker per section", () => { + const csv = toCSV(sampleState()); + assert.ok(csv.includes("# section: positions")); + assert.ok(csv.includes("# section: pools")); + assert.ok(csv.includes("# section: governance")); +}); diff --git a/scripts/state-exporter/exporter.ts b/scripts/state-exporter/exporter.ts new file mode 100644 index 0000000..433f393 --- /dev/null +++ b/scripts/state-exporter/exporter.ts @@ -0,0 +1,194 @@ +/** + * Core, transport-free export logic for the contract state-exporter (#499). + * + * Pure functions only — serialization, checksums, row counts, change-data-capture + * (incremental) diffing, gzip (de)compression, and integrity verification — so + * the behaviour is fully unit-testable without a network or filesystem. + */ + +import { createHash } from "node:crypto"; +import { gzipSync, gunzipSync } from "node:zlib"; +import { + type ContractState, + type ExportFormat, + type ExportManifest, + type StateSection, + SECTION_KEY, + STATE_SECTIONS, +} from "./types.ts"; + +const EMPTY_STATE: ContractState = { positions: [], pools: [], governance: [] }; + +/** Deterministic SHA-256 hex digest of a payload. */ +export function checksum(payload: string): string { + return createHash("sha256").update(payload).digest("hex"); +} + +/** Row counts per section plus a total. */ +export function rowCount(state: ContractState): ExportManifest["rowCount"] { + const counts = { + positions: state.positions.length, + pools: state.pools.length, + governance: state.governance.length, + total: 0, + }; + counts.total = counts.positions + counts.pools + counts.governance; + return counts; +} + +/** Serialize full state to canonical (stable-key-order) JSON. */ +export function toJSON(state: ContractState): string { + return JSON.stringify( + { + positions: state.positions, + pools: state.pools, + governance: state.governance, + }, + null, + 2, + ); +} + +function csvEscape(value: unknown): string { + const s = String(value ?? ""); + return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s; +} + +/** Serialize a single section to CSV (header + rows). */ +export function sectionToCSV(state: ContractState, section: StateSection): string { + const rows = state[section] as Array>; + if (rows.length === 0) return ""; + const headers = Object.keys(rows[0]); + const lines = [headers.join(",")]; + for (const row of rows) { + lines.push(headers.map((h) => csvEscape(row[h])).join(",")); + } + return lines.join("\n") + "\n"; +} + +/** Serialize all sections to CSV, concatenated with section markers. */ +export function toCSV(state: ContractState): string { + return STATE_SECTIONS.map((s) => `# section: ${s}\n${sectionToCSV(state, s)}`).join("\n"); +} + +export function serialize(state: ContractState, format: ExportFormat): string { + return format === "csv" ? toCSV(state) : toJSON(state); +} + +/** + * Change-data-capture: return only rows that are new or changed relative to + * `previous`, keyed by each section's primary key. Removed rows are intentionally + * omitted (snapshot exports capture deletions; incremental exports capture + * upserts) — the manifest's row counts make the delta size explicit. + */ +export function diff(previous: ContractState, next: ContractState): ContractState { + const result: ContractState = { positions: [], pools: [], governance: [] }; + for (const section of STATE_SECTIONS) { + const key = SECTION_KEY[section]; + const prevByKey = new Map(); + for (const row of previous[section] as Array>) { + prevByKey.set(String(row[key]), JSON.stringify(row)); + } + const changed = (next[section] as Array>).filter((row) => { + const prev = prevByKey.get(String(row[key])); + return prev === undefined || prev !== JSON.stringify(row); + }); + (result[section] as unknown[]) = changed; + } + return result; +} + +export function compress(payload: string): Buffer { + return gzipSync(Buffer.from(payload, "utf8")); +} + +export function decompress(buf: Buffer): string { + return gunzipSync(buf).toString("utf8"); +} + +export interface BuildExportOptions { + format: ExportFormat; + compressed?: boolean; + /** Previous full state; when provided the export is an incremental (CDC) delta. */ + previous?: ContractState; +} + +export interface BuiltExport { + manifest: ExportManifest; + /** UTF-8 payload (or gzip buffer when compressed). */ + payload: string | Buffer; + /** The (possibly diffed) state that was exported. */ + state: ContractState; +} + +/** Build a complete, verifiable export (payload + manifest). */ +export function buildExport(state: ContractState, options: BuildExportOptions): BuiltExport { + const incremental = options.previous !== undefined; + const effective = incremental ? diff(options.previous ?? EMPTY_STATE, state) : state; + const text = serialize(effective, options.format); + const manifest: ExportManifest = { + generatedAt: new Date().toISOString(), + format: options.format, + compressed: Boolean(options.compressed), + incremental, + rowCount: rowCount(effective), + checksum: checksum(text), + }; + return { + manifest, + payload: options.compressed ? compress(text) : text, + state: effective, + }; +} + +export interface IntegrityResult { + ok: boolean; + expectedChecksum: string; + actualChecksum: string; + expectedRows: number; + actualRows: number; +} + +/** Verify a payload against its manifest (checksum + row count). */ +export function verifyIntegrity( + payload: string | Buffer, + manifest: ExportManifest, +): IntegrityResult { + const text = manifest.compressed + ? decompress(Buffer.isBuffer(payload) ? payload : Buffer.from(payload)) + : String(payload); + const actualChecksum = checksum(text); + const actualRows = countRowsInPayload(text, manifest.format); + return { + ok: actualChecksum === manifest.checksum && actualRows === manifest.rowCount.total, + expectedChecksum: manifest.checksum, + actualChecksum, + expectedRows: manifest.rowCount.total, + actualRows, + }; +} + +function countRowsInPayload(text: string, format: ExportFormat): number { + if (format === "json") { + const parsed = JSON.parse(text) as ContractState; + return rowCount(parsed).total; + } + // CSV: each "# section:" marker is followed by a header line, then data rows. + // Count only the data rows. + let count = 0; + let expectHeader = false; + for (const raw of text.split("\n")) { + const line = raw.trim(); + if (line === "") continue; + if (line.startsWith("# section:")) { + expectHeader = true; + continue; + } + if (expectHeader) { + expectHeader = false; // this is the header line for the section + continue; + } + count += 1; + } + return count; +} diff --git a/scripts/state-exporter/index.ts b/scripts/state-exporter/index.ts new file mode 100644 index 0000000..46f07f7 --- /dev/null +++ b/scripts/state-exporter/index.ts @@ -0,0 +1,70 @@ +#!/usr/bin/env node +/** + * state-exporter CLI (#499). + * + * Exports structured contract state to JSON/CSV with checksum + row-count + * verification, optional gzip compression, and incremental (CDC) exports. + * + * Run (no install needed on Node >= 22): + * node --experimental-strip-types scripts/state-exporter/index.ts \ + * --in state.json --format json --out export.json [--gzip] [--since prev.json] + * node --experimental-strip-types scripts/state-exporter/index.ts \ + * --verify export.json --manifest export.json.manifest.json + * + * The default source is a JSON snapshot file so the tool is runnable offline. + * A live Soroban RPC source (view functions + events) and additional sinks + * (Parquet/SQL, S3/GCS, scheduling) are documented follow-ups in the README. + */ + +import * as fs from "node:fs"; +import { buildExport, verifyIntegrity } from "./exporter.ts"; +import { type ContractState, type ExportFormat, type ExportManifest } from "./types.ts"; + +function arg(name: string): string | undefined { + const i = process.argv.indexOf(`--${name}`); + return i >= 0 ? process.argv[i + 1] : undefined; +} +function flag(name: string): boolean { + return process.argv.includes(`--${name}`); +} + +function readState(path: string): ContractState { + return JSON.parse(fs.readFileSync(path, "utf8")) as ContractState; +} + +function main(): void { + if (flag("verify")) { + const payloadPath = arg("verify")!; + const manifestPath = arg("manifest") ?? `${payloadPath}.manifest.json`; + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as ExportManifest; + const payload = manifest.compressed + ? fs.readFileSync(payloadPath) + : fs.readFileSync(payloadPath, "utf8"); + const result = verifyIntegrity(payload, manifest); + console.log(JSON.stringify(result, null, 2)); + process.exit(result.ok ? 0 : 1); + } + + const inPath = arg("in"); + if (!inPath) { + console.error("Missing --in (or use --verify )"); + process.exit(2); + } + const format = (arg("format") ?? "json") as ExportFormat; + const out = arg("out") ?? `export.${format}`; + const compressed = flag("gzip"); + const since = arg("since"); + + const state = readState(inPath); + const previous = since ? readState(since) : undefined; + const built = buildExport(state, { format, compressed, previous }); + + const outPath = compressed ? `${out}.gz` : out; + fs.writeFileSync(outPath, built.payload); + fs.writeFileSync(`${outPath}.manifest.json`, JSON.stringify(built.manifest, null, 2)); + + console.log(`Exported ${built.manifest.rowCount.total} rows -> ${outPath}`); + console.log(`Manifest -> ${outPath}.manifest.json (checksum ${built.manifest.checksum.slice(0, 12)}…)`); +} + +main(); diff --git a/scripts/state-exporter/types.ts b/scripts/state-exporter/types.ts new file mode 100644 index 0000000..5ef1f88 --- /dev/null +++ b/scripts/state-exporter/types.ts @@ -0,0 +1,71 @@ +/** + * Domain types for the contract state-exporter (#499). + * + * These mirror the protocol's on-chain state at a structural level so the + * exporter can run against any source — a live RPC reader or a captured JSON + * snapshot — without coupling to a specific transport. + */ + +/** A borrower/supplier position. */ +export interface UserPosition { + address: string; + collateral: string; // stringified i128 to avoid precision loss + debt: string; + healthFactor: string; +} + +/** Per-asset pool configuration. */ +export interface PoolConfig { + asset: string; + supplyCap: string; + interestRateBps: number; + collateralFactorBps: number; +} + +/** A governance proposal and its tally. */ +export interface GovernanceRecord { + proposalId: number; + status: string; + votesFor: string; + votesAgainst: string; + executed: boolean; +} + +/** Full structured contract state. */ +export interface ContractState { + positions: UserPosition[]; + pools: PoolConfig[]; + governance: GovernanceRecord[]; +} + +export type StateSection = keyof ContractState; + +export const STATE_SECTIONS: StateSection[] = ["positions", "pools", "governance"]; + +/** Stable primary key for each section, used for diffing (CDC) and CSV ordering. */ +export const SECTION_KEY: Record = { + positions: "address", + pools: "asset", + governance: "proposalId", +}; + +/** + * Abstraction over where state comes from. The default file source reads a JSON + * snapshot; a live Soroban RPC source (reading view functions + events) is a + * documented follow-up. + */ +export interface StateSource { + read(): Promise; +} + +export type ExportFormat = "json" | "csv"; + +/** Verifiable metadata accompanying every export. */ +export interface ExportManifest { + generatedAt: string; + format: ExportFormat; + compressed: boolean; + incremental: boolean; + rowCount: Record & { total: number }; + checksum: string; // sha256 of the uncompressed payload +} diff --git a/scripts/state-migrate/README.md b/scripts/state-migrate/README.md new file mode 100644 index 0000000..d0215a8 --- /dev/null +++ b/scripts/state-migrate/README.md @@ -0,0 +1,63 @@ +# State Migration Tool (#497) + +A declarative framework for migrating contract storage between schema versions, +with dry-run, atomic checkpoint rollback, verification, and audit history. + +## Migration definition + +```json +{ + "version": 2, + "description": "add tier, rename collateral, normalize debt", + "operations": [ + { "op": "addField", "field": "tier", "default": "bronze" }, + { "op": "renameField", "from": "collateral", "to": "collateral_amount" }, + { "op": "transformField", "field": "debt", "using": "toNumber" }, + { "op": "removeField", "field": "legacy_flag" } + ] +} +``` + +Supported operations: `addField`, `removeField`, `renameField`, `transformField` +(via a named, pure transform registry — `toString`, `toNumber`, `identity` +built in, extensible). + +## Usage + +```bash +# Dry-run: preview changes + heuristic cost, no writes +node --experimental-strip-types scripts/state-migrate/index.ts \ + --state state.json --plan migration.json --dry-run + +# Apply with verification + history (auto-rolls-back on failure) +node --experimental-strip-types scripts/state-migrate/index.ts \ + --state state.json --plan migration.json --out migrated.json \ + --expect expect.json --history history.json +``` + +`expect.json` (post-migration integrity): +```json +{ "requireFields": ["tier"], "forbidFields": ["legacy_flag"], "preserveRowCount": true } +``` + +## Tests + +```bash +node --experimental-strip-types --test scripts/state-migrate/migrator.test.ts +``` + +## Covered acceptance criteria + +- Declarative migration definitions; operations add/rename/remove/transform field. +- Dry-run simulation that never mutates state. +- Atomic execution with checkpoint + automatic rollback on op error or failed verification. +- Post-migration verification (required/forbidden fields, row-count preservation). +- Versioned, audit-trailed migration history (monotonic-version enforced). +- Heuristic per-operation cost estimate. +- CLI: run, dry-run, view status, append history. + +## Follow-ups (out of this PR's core) + +- On-chain atomic execution against a live contract and real gas metering via + transaction simulation (the engine exposes a heuristic cost model today). +- Migration-hub integration (`contracts/migration-tool/`). diff --git a/scripts/state-migrate/index.ts b/scripts/state-migrate/index.ts new file mode 100644 index 0000000..f3033a3 --- /dev/null +++ b/scripts/state-migrate/index.ts @@ -0,0 +1,79 @@ +#!/usr/bin/env node +/** + * state-migrate CLI (#497) — declarative contract storage schema migrations + * with dry-run, atomic checkpoint rollback, verification, and history. + * + * Run (Node >= 22, no install): + * # Dry-run (no writes): preview changes + cost + * node --experimental-strip-types scripts/state-migrate/index.ts \ + * --state state.json --plan migration.json --dry-run + * + * # Apply with verification + history; rolls back automatically on failure + * node --experimental-strip-types scripts/state-migrate/index.ts \ + * --state state.json --plan migration.json --out migrated.json \ + * --expect expect.json --history history.json + */ + +import * as fs from "node:fs"; +import { + appendHistory, + dryRun, + migrateWithRollback, +} from "./migrator.ts"; +import { + type Migration, + type MigrationExpectation, + type MigrationRecord, + type State, +} from "./types.ts"; + +function arg(name: string): string | undefined { + const i = process.argv.indexOf(`--${name}`); + return i >= 0 ? process.argv[i + 1] : undefined; +} +function flag(name: string): boolean { + return process.argv.includes(`--${name}`); +} +function readJSON(path: string): T { + return JSON.parse(fs.readFileSync(path, "utf8")) as T; +} + +function main(): void { + const statePath = arg("state"); + const planPath = arg("plan"); + if (!statePath || !planPath) { + console.error("Usage: --state --plan [--dry-run] [--out ...] [--expect ...] [--history ...]"); + process.exit(2); + } + + const state = readJSON(statePath); + const migration = readJSON(planPath); + const expect = arg("expect") ? readJSON(arg("expect")!) : undefined; + + if (flag("dry-run")) { + const report = dryRun(state, migration); + // Don't print the full preview to keep output readable. + const { preview, ...summary } = report; + void preview; + console.log(JSON.stringify(summary, null, 2)); + return; + } + + const result = migrateWithRollback(state, migration, { expect }); + + const outPath = arg("out") ?? "migrated.json"; + fs.writeFileSync(outPath, JSON.stringify(result.state, null, 2)); + + const historyPath = arg("history"); + if (historyPath) { + const history = fs.existsSync(historyPath) + ? readJSON(historyPath) + : []; + fs.writeFileSync(historyPath, JSON.stringify(appendHistory(history, result.record), null, 2)); + } + + console.log(JSON.stringify({ status: result.status, errors: result.errors, record: result.record }, null, 2)); + process.exit(result.status === "applied" ? 0 : 1); +} + +main(); diff --git a/scripts/state-migrate/migrator.test.ts b/scripts/state-migrate/migrator.test.ts new file mode 100644 index 0000000..ba0f01a --- /dev/null +++ b/scripts/state-migrate/migrator.test.ts @@ -0,0 +1,170 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + appendHistory, + applyMigration, + dryRun, + estimateCost, + migrateWithRollback, + verifyMigration, +} from "./migrator.ts"; +import { type Migration, type State } from "./types.ts"; + +function sampleState(): State { + return [ + { address: "GA1", collateral: "100", debt: "10" }, + { address: "GA2", collateral: "200", debt: "0" }, + ]; +} + +test("addField sets a default only when absent", () => { + const m: Migration = { + version: 2, + description: "add tier", + operations: [{ op: "addField", field: "tier", default: "bronze" }], + }; + const out = applyMigration(sampleState(), m); + assert.equal(out[0].tier, "bronze"); + assert.equal(out[1].tier, "bronze"); +}); + +test("removeField, renameField, transformField apply correctly", () => { + const m: Migration = { + version: 2, + description: "reshape", + operations: [ + { op: "renameField", from: "collateral", to: "collateral_amount" }, + { op: "transformField", field: "debt", using: "toNumber" }, + { op: "removeField", field: "address" }, + ], + }; + const out = applyMigration(sampleState(), m); + assert.equal(out[0].collateral_amount, "100"); + assert.equal("collateral" in out[0], false); + assert.equal(out[0].debt, 10); // transformed to number + assert.equal("address" in out[0], false); +}); + +test("applyMigration does not mutate the input state", () => { + const state = sampleState(); + applyMigration(state, { + version: 2, + description: "x", + operations: [{ op: "removeField", field: "debt" }], + }); + assert.equal("debt" in state[0], true); // original untouched +}); + +test("dryRun previews changes and cost without committing", () => { + const state = sampleState(); + const report = dryRun(state, { + version: 2, + description: "add+rename", + operations: [ + { op: "addField", field: "tier", default: "bronze" }, + { op: "renameField", from: "debt", to: "debt_amount" }, + ], + }); + assert.deepEqual(report.fieldsAdded, ["tier"]); + assert.deepEqual(report.fieldsRenamed, [{ from: "debt", to: "debt_amount" }]); + assert.equal(report.rowsBefore, 2); + assert.ok(report.estimatedCostUnits > 0); + // input state must remain unchanged after a dry-run + assert.equal("debt" in state[0], true); +}); + +test("estimateCost scales with rows and operations", () => { + const m: Migration = { + version: 2, + description: "two ops", + operations: [ + { op: "addField", field: "a", default: 1 }, // 2 units/row + { op: "transformField", field: "a", using: "toString" }, // 3 units/row + ], + }; + assert.equal(estimateCost(sampleState(), m), (2 + 3) * 2); +}); + +test("verifyMigration enforces required/forbidden fields and row count", () => { + const before = sampleState(); + const after = applyMigration(before, { + version: 2, + description: "rm debt", + operations: [{ op: "removeField", field: "debt" }], + }); + const ok = verifyMigration(before, after, { + forbidFields: ["debt"], + requireFields: ["address"], + preserveRowCount: true, + }); + assert.ok(ok.ok, JSON.stringify(ok.errors)); + + const bad = verifyMigration(before, after, { requireFields: ["debt"] }); + assert.equal(bad.ok, false); +}); + +test("migrateWithRollback commits a valid migration", () => { + const result = migrateWithRollback( + sampleState(), + { version: 2, description: "ok", operations: [{ op: "addField", field: "tier", default: "x" }] }, + { expect: { requireFields: ["tier"], preserveRowCount: true } }, + ); + assert.equal(result.status, "applied"); + assert.equal(result.state[0].tier, "x"); + assert.equal(result.record.status, "applied"); +}); + +test("migrateWithRollback restores checkpoint on verification failure", () => { + const state = sampleState(); + const result = migrateWithRollback( + state, + { version: 2, description: "bad", operations: [{ op: "removeField", field: "address" }] }, + { expect: { requireFields: ["address"] } }, // will fail + ); + assert.equal(result.status, "rolled-back"); + assert.ok(result.errors.length > 0); + // rolled back to original + assert.equal("address" in result.state[0], true); +}); + +test("migrateWithRollback restores checkpoint when an operation throws", () => { + const result = migrateWithRollback(sampleState(), { + version: 2, + description: "bad transform", + operations: [{ op: "transformField", field: "debt", using: "nonexistent" }], + }); + assert.equal(result.status, "rolled-back"); + assert.equal("debt" in result.state[0], true); +}); + +test("appendHistory enforces monotonic applied versions", () => { + let history = appendHistory([], { + version: 2, + description: "v2", + appliedAt: "t", + rows: 2, + checksum: "c", + status: "applied", + }); + assert.equal(history.length, 1); + assert.throws(() => + appendHistory(history, { + version: 2, + description: "v2-again", + appliedAt: "t", + rows: 2, + checksum: "c", + status: "applied", + }), + ); + // a rolled-back record at the same version is allowed (it didn't advance schema) + history = appendHistory(history, { + version: 2, + description: "v2-rollback", + appliedAt: "t", + rows: 2, + checksum: "c", + status: "rolled-back", + }); + assert.equal(history.length, 2); +}); diff --git a/scripts/state-migrate/migrator.ts b/scripts/state-migrate/migrator.ts new file mode 100644 index 0000000..18f755a --- /dev/null +++ b/scripts/state-migrate/migrator.ts @@ -0,0 +1,212 @@ +/** + * Core, pure migration engine for contract storage schema upgrades (#497). + * + * Applies declarative migrations to a table of records with dry-run, checkpoint + * rollback, post-migration verification, history tracking, and a heuristic cost + * estimate. Atomic on-chain execution / real gas metering are documented + * follow-ups; this engine is the safety-critical, fully-tested core. + */ + +import { createHash } from "node:crypto"; +import { + type Migration, + type MigrationExpectation, + type MigrationOperation, + type MigrationRecord, + type Row, + type State, + type TransformRegistry, + OP_COST_UNITS, +} from "./types.ts"; + +/** Built-in field transforms. Callers may extend via a custom registry. */ +export const DEFAULT_TRANSFORMS: TransformRegistry = { + toString: (v) => String(v ?? ""), + toNumber: (v) => Number(v ?? 0), + identity: (v) => v, +}; + +function clone(state: State): State { + return state.map((row) => ({ ...row })); +} + +export function stateChecksum(state: State): string { + return createHash("sha256").update(JSON.stringify(state)).digest("hex"); +} + +function applyOperation( + state: State, + op: MigrationOperation, + transforms: TransformRegistry, +): State { + return state.map((row) => { + const next: Row = { ...row }; + switch (op.op) { + case "addField": + if (!(op.field in next)) next[op.field] = op.default; + break; + case "removeField": + delete next[op.field]; + break; + case "renameField": + if (op.from in next) { + next[op.to] = next[op.from]; + delete next[op.from]; + } + break; + case "transformField": { + const fn = transforms[op.using]; + if (!fn) throw new Error(`unknown transform "${op.using}"`); + if (op.field in next) next[op.field] = fn(next[op.field], next); + break; + } + } + return next; + }); +} + +/** Apply a migration to a copy of `state` (input is never mutated). */ +export function applyMigration( + state: State, + migration: Migration, + transforms: TransformRegistry = DEFAULT_TRANSFORMS, +): State { + let result = clone(state); + for (const op of migration.operations) { + result = applyOperation(result, op, transforms); + } + return result; +} + +export interface DryRunReport { + version: number; + rowsBefore: number; + rowsAfter: number; + fieldsAdded: string[]; + fieldsRemoved: string[]; + fieldsRenamed: Array<{ from: string; to: string }>; + fieldsTransformed: string[]; + estimatedCostUnits: number; + /** Resulting state — NOT committed by the caller in dry-run mode. */ + preview: State; +} + +/** Simulate a migration without committing: returns a change summary + preview. */ +export function dryRun( + state: State, + migration: Migration, + transforms: TransformRegistry = DEFAULT_TRANSFORMS, +): DryRunReport { + const preview = applyMigration(state, migration, transforms); + const report: DryRunReport = { + version: migration.version, + rowsBefore: state.length, + rowsAfter: preview.length, + fieldsAdded: [], + fieldsRemoved: [], + fieldsRenamed: [], + fieldsTransformed: [], + estimatedCostUnits: estimateCost(state, migration), + preview, + }; + for (const op of migration.operations) { + if (op.op === "addField") report.fieldsAdded.push(op.field); + else if (op.op === "removeField") report.fieldsRemoved.push(op.field); + else if (op.op === "renameField") report.fieldsRenamed.push({ from: op.from, to: op.to }); + else report.fieldsTransformed.push(op.field); + } + return report; +} + +/** Heuristic cost estimate: sum of per-op unit cost × row count. */ +export function estimateCost(state: State, migration: Migration): number { + return migration.operations.reduce( + (sum, op) => sum + OP_COST_UNITS[op.op] * state.length, + 0, + ); +} + +/** Post-migration integrity checks. */ +export function verifyMigration( + before: State, + after: State, + expect: MigrationExpectation, +): { ok: boolean; errors: string[] } { + const errors: string[] = []; + if (expect.preserveRowCount && before.length !== after.length) { + errors.push(`row count changed: ${before.length} -> ${after.length}`); + } + for (const field of expect.requireFields ?? []) { + if (!after.every((r) => field in r)) errors.push(`missing required field "${field}"`); + } + for (const field of expect.forbidFields ?? []) { + if (after.some((r) => field in r)) errors.push(`forbidden field "${field}" still present`); + } + return { ok: errors.length === 0, errors }; +} + +export interface MigrateResult { + status: "applied" | "rolled-back"; + state: State; + errors: string[]; + record: MigrationRecord; +} + +/** + * Apply a migration atomically: snapshot a checkpoint first, run the migration + * and verification, and restore the checkpoint (rollback) if anything fails — + * so storage can never be left in a partially-migrated state. + */ +export function migrateWithRollback( + state: State, + migration: Migration, + options: { + transforms?: TransformRegistry; + expect?: MigrationExpectation; + } = {}, +): MigrateResult { + const transforms = options.transforms ?? DEFAULT_TRANSFORMS; + const checkpoint = clone(state); + let errors: string[] = []; + let next: State; + try { + next = applyMigration(state, migration, transforms); + if (options.expect) { + const verdict = verifyMigration(checkpoint, next, options.expect); + if (!verdict.ok) errors = verdict.errors; + } + } catch (err) { + errors = [err instanceof Error ? err.message : String(err)]; + next = checkpoint; + } + + const rolledBack = errors.length > 0; + const finalState = rolledBack ? checkpoint : next; + return { + status: rolledBack ? "rolled-back" : "applied", + state: finalState, + errors, + record: { + version: migration.version, + description: migration.description, + appliedAt: new Date().toISOString(), + rows: finalState.length, + checksum: stateChecksum(finalState), + status: rolledBack ? "rolled-back" : "applied", + }, + }; +} + +/** Append a migration record to a history log, enforcing monotonic versions. */ +export function appendHistory( + history: MigrationRecord[], + record: MigrationRecord, +): MigrationRecord[] { + const lastApplied = [...history].reverse().find((r) => r.status === "applied"); + if (record.status === "applied" && lastApplied && record.version <= lastApplied.version) { + throw new Error( + `non-monotonic migration version ${record.version} (last applied ${lastApplied.version})`, + ); + } + return [...history, record]; +} diff --git a/scripts/state-migrate/types.ts b/scripts/state-migrate/types.ts new file mode 100644 index 0000000..3ccca0d --- /dev/null +++ b/scripts/state-migrate/types.ts @@ -0,0 +1,53 @@ +/** + * Types for the contract storage state-migration tool (#497). + * + * State is modeled as a table of records (each a field map), so schema + * migrations — add / rename / remove / transform field — are expressed + * declaratively and applied deterministically with dry-run and rollback. + */ + +export type Row = Record; +export type State = Row[]; + +export type MigrationOperation = + | { op: "addField"; field: string; default: unknown } + | { op: "removeField"; field: string } + | { op: "renameField"; from: string; to: string } + | { op: "transformField"; field: string; using: string }; + +export interface Migration { + /** Monotonic schema version this migration upgrades TO. */ + version: number; + description: string; + operations: MigrationOperation[]; +} + +/** Named, pure field transforms referenced by `transformField.using`. */ +export type TransformRegistry = Record unknown>; + +export interface MigrationExpectation { + /** Every row must have these fields. */ + requireFields?: string[]; + /** No row may have these fields. */ + forbidFields?: string[]; + /** Row count must be unchanged after migration. */ + preserveRowCount?: boolean; +} + +export interface MigrationRecord { + version: number; + description: string; + appliedAt: string; + rows: number; + checksum: string; + status: "applied" | "rolled-back"; +} + +/** Heuristic gas/cost units per operation type (per row). On-chain estimation + * via simulation is a documented follow-up. */ +export const OP_COST_UNITS: Record = { + addField: 2, + removeField: 1, + renameField: 2, + transformField: 3, +};