From 9e1d4e3f82a85d116b3bca563d09a918944f6015 Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 03:32:22 +0700 Subject: [PATCH 01/16] test(reconciliation): add disposable scale canary --- .../large-repository-canary.mjs | 854 ++++++++++++++++++ .../test/fixtures/large-incremental-child.mjs | 15 +- 2 files changed, 867 insertions(+), 2 deletions(-) create mode 100644 gitnexus/scripts/reconciliation/large-repository-canary.mjs diff --git a/gitnexus/scripts/reconciliation/large-repository-canary.mjs b/gitnexus/scripts/reconciliation/large-repository-canary.mjs new file mode 100644 index 0000000000..e77142170b --- /dev/null +++ b/gitnexus/scripts/reconciliation/large-repository-canary.mjs @@ -0,0 +1,854 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import http from 'node:http'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const REQUIRED_ARGS = [ + 'source', + 'source-sha', + 'public-origin', + 'worktree', + 'evidence', + 'gitnexus-cli', + 'incremental-child', + 'extension-source', + 'run-id', +]; + +const parseArgs = (argv) => { + const result = {}; + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index]?.replace(/^--/, ''); + const value = argv[index + 1]; + if (!key || !value || !argv[index].startsWith('--')) { + throw new Error(`invalid argument sequence near ${argv[index] ?? ''}`); + } + result[key] = value; + } + for (const key of REQUIRED_ARGS) { + if (!result[key]) throw new Error(`missing required --${key}`); + } + return result; +}; + +const args = parseArgs(process.argv.slice(2)); +const source = path.resolve(args.source); +const sourceSha = args['source-sha']; +const publicOrigin = args['public-origin']; +const worktree = path.resolve(args.worktree); +const evidence = path.resolve(args.evidence); +const cli = path.resolve(args['gitnexus-cli']); +const incrementalChild = path.resolve(args['incremental-child']); +const extensionSource = path.resolve(args['extension-source']); +const runId = args['run-id']; +const dimensions = Number(args['embedding-dims'] ?? '8'); +const model = args['embedding-model'] ?? 'gitnexus-canary-deterministic-v1'; +const resumeFrom = args['resume-from']; +const escalationTimeoutMs = Number(args['escalation-timeout-ms'] ?? '21600000'); +const packageRoot = path.resolve(path.dirname(cli), '../..'); +const home = path.join(evidence, 'home'); +const commandDir = path.join(evidence, 'commands'); +const snapshotDir = path.join(evidence, 'snapshots'); +const failurePath = path.join(evidence, 'failure.json'); + +if (!Number.isInteger(dimensions) || dimensions <= 0) { + throw new Error('--embedding-dims must be a positive integer'); +} +if (resumeFrom && !['wide', 'forced'].includes(resumeFrom)) { + throw new Error('--resume-from currently supports only "wide" or "forced"'); +} +if (!Number.isInteger(escalationTimeoutMs) || escalationTimeoutMs <= 0) { + throw new Error('--escalation-timeout-ms must be a positive integer'); +} +if (!resumeFrom && fs.existsSync(worktree)) { + throw new Error(`refusing existing canary worktree: ${worktree}`); +} +if (resumeFrom && !fs.existsSync(worktree)) { + throw new Error(`cannot resume missing canary worktree: ${worktree}`); +} +for (const requiredPath of [source, cli, incrementalChild, extensionSource]) { + if (!fs.existsSync(requiredPath)) + throw new Error(`required path does not exist: ${requiredPath}`); +} +fs.mkdirSync(commandDir, { recursive: true }); +fs.mkdirSync(snapshotDir, { recursive: true }); +fs.mkdirSync(home, { recursive: true }); +if (resumeFrom && fs.existsSync(failurePath)) { + let archiveIndex = 1; + let archivedFailurePath; + do { + archivedFailurePath = path.join(evidence, `failure-before-resume-${archiveIndex}.json`); + archiveIndex += 1; + } while (fs.existsSync(archivedFailurePath)); + if (!archivedFailurePath) { + throw new Error('failed to allocate archived failure path'); + } + fs.renameSync(failurePath, archivedFailurePath); +} +fs.cpSync(extensionSource, path.join(home, '.lbdb', 'extension'), { recursive: true }); + +const writeJson = (filePath, value) => { + fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, 'utf8'); +}; + +const extensionFiles = []; +const collectExtensionFiles = (dir) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) collectExtensionFiles(fullPath); + if (entry.isFile()) { + extensionFiles.push({ + path: path.relative(extensionSource, fullPath), + bytes: fs.statSync(fullPath).size, + sha256: crypto.createHash('sha256').update(fs.readFileSync(fullPath)).digest('hex'), + }); + } + } +}; +collectExtensionFiles(extensionSource); +writeJson(path.join(evidence, 'extension-manifest.json'), extensionFiles); + +const commandLedgerPath = path.join(evidence, 'command-ledger.json'); +const commandLedger = + resumeFrom && fs.existsSync(commandLedgerPath) + ? JSON.parse(fs.readFileSync(commandLedgerPath, 'utf8')) + : []; +const embeddingStats = { requests: 0, items: 0 }; + +const run = async (name, command, commandArgs, options = {}) => { + const stdoutPath = path.join(commandDir, `${name}.stdout.log`); + const stderrPath = path.join(commandDir, `${name}.stderr.log`); + const stdout = fs.openSync(stdoutPath, 'w'); + const stderr = fs.openSync(stderrPath, 'w'); + const startedAt = new Date().toISOString(); + const startedMs = Date.now(); + process.stdout.write(`[${runId}] ${name}\n`); + const child = spawn(command, commandArgs, { + cwd: options.cwd ?? worktree, + env: options.env ?? process.env, + stdio: ['ignore', stdout, stderr], + }); + const result = await new Promise((resolve, reject) => { + child.once('error', reject); + child.once('close', (code, signal) => resolve({ code, signal })); + }); + fs.closeSync(stdout); + fs.closeSync(stderr); + const entry = { + name, + command, + args: commandArgs, + cwd: options.cwd ?? worktree, + startedAt, + finishedAt: new Date().toISOString(), + durationMs: Date.now() - startedMs, + code: result.code, + signal: result.signal, + stdout: path.basename(stdoutPath), + stderr: path.basename(stderrPath), + }; + commandLedger.push(entry); + writeJson(path.join(evidence, 'command-ledger.json'), commandLedger); + if (!options.allowFailure && (result.code !== 0 || result.signal !== null)) { + throw new Error(`${name} failed with code=${result.code} signal=${result.signal}`); + } + return entry; +}; + +const runText = async (name, command, commandArgs, options = {}) => { + const chunks = []; + const child = spawn(command, commandArgs, { + cwd: options.cwd ?? worktree, + env: options.env ?? process.env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + child.stdout.on('data', (chunk) => chunks.push(chunk)); + const errors = []; + child.stderr.on('data', (chunk) => errors.push(chunk)); + const result = await new Promise((resolve, reject) => { + child.once('error', reject); + child.once('close', (code, signal) => resolve({ code, signal })); + }); + if (result.code !== 0 || result.signal !== null) { + throw new Error(`${name} failed: ${Buffer.concat(errors).toString('utf8').trim()}`); + } + return Buffer.concat(chunks).toString('utf8').trim(); +}; + +const deterministicVector = (text) => { + const digest = crypto.createHash('sha256').update(text).digest(); + const vector = Array.from({ length: dimensions }, (_, index) => digest[index] / 127.5 - 1); + const magnitude = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0)) || 1; + return vector.map((value) => value / magnitude); +}; + +const embeddingServer = http.createServer((request, response) => { + if (request.method !== 'POST' || request.url !== '/v1/embeddings') { + response.writeHead(404).end(); + return; + } + const chunks = []; + request.on('data', (chunk) => chunks.push(chunk)); + request.on('end', () => { + try { + const body = JSON.parse(Buffer.concat(chunks).toString('utf8')); + const inputs = Array.isArray(body.input) ? body.input : [body.input]; + if (!inputs.every((input) => typeof input === 'string')) { + throw new Error('input must be a string or string array'); + } + embeddingStats.requests += 1; + embeddingStats.items += inputs.length; + const payload = { + object: 'list', + model, + data: inputs.map((input, index) => ({ + object: 'embedding', + index, + embedding: deterministicVector(input), + })), + usage: { prompt_tokens: 0, total_tokens: 0 }, + }; + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify(payload)); + } catch (error) { + response.writeHead(400, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ error: { message: error.message } })); + } + }); +}); + +await new Promise((resolve, reject) => { + embeddingServer.once('error', reject); + embeddingServer.listen(0, '127.0.0.1', resolve); +}); +const address = embeddingServer.address(); +if (!address || typeof address === 'string') throw new Error('embedding server has no TCP address'); + +const canaryEnv = { + ...process.env, + HOME: home, + USERPROFILE: home, + GITNEXUS_HOME: path.join(home, '.gitnexus'), + CI: '1', + NODE_OPTIONS: '--max-old-space-size=6144', + GITNEXUS_WORKER_POOL_SIZE: '2', + GITNEXUS_PARSE_CHUNK_CONCURRENCY: '1', + GITNEXUS_LBUG_EXTENSION_INSTALL: 'load-only', + GITNEXUS_EMBEDDING_URL: `http://127.0.0.1:${address.port}/v1`, + GITNEXUS_EMBEDDING_MODEL: model, + GITNEXUS_EMBEDDING_DIMS: String(dimensions), + GITNEXUS_EMBEDDING_BATCH_SIZE: '256', + GITNEXUS_EMBEDDING_SUB_BATCH_SIZE: '256', + GITNEXUS_EMBEDDING_MAX_ATTEMPTS: '3', + GITNEXUS_EMBEDDING_RETRY_CAP_MS: '1000', + GITNEXUS_EMBEDDING_MIN_INTERVAL_MS: '0', +}; + +const commitAll = async (message) => { + await run(`git-add-${commandLedger.length}`, 'git', ['add', '-A']); + await run(`git-commit-${commandLedger.length}`, 'git', [ + '-c', + 'user.name=GitNexus Canary', + '-c', + 'user.email=canary@example.invalid', + '-c', + 'commit.gpgsign=false', + 'commit', + '-q', + '-m', + message, + ]); +}; + +const snapshot = async (name, { fingerprint = false } = {}) => { + const repoManager = await import( + pathToFileURL(path.join(packageRoot, 'dist/storage/repo-manager.js')).href + ); + const adapter = await import( + pathToFileURL(path.join(packageRoot, 'dist/core/lbug/lbug-adapter.js')).href + ); + const schema = await import( + pathToFileURL(path.join(packageRoot, 'dist/core/lbug/schema.js')).href + ); + const { storagePath, lbugPath } = repoManager.getStoragePaths(worktree); + const meta = await repoManager.loadMeta(storagePath); + const status = await runText(`status-${name}`, 'git', ['status', '--porcelain=v1']); + const head = await runText(`head-${name}`, 'git', ['rev-parse', 'HEAD']); + const disk = await runText(`disk-${name}`, 'df', ['-k', path.dirname(worktree)]); + const indexKb = Number( + await runText(`du-${name}`, 'du', ['-sk', storagePath]).then((text) => text.split(/\s+/)[0]), + ); + const files = fs.existsSync(storagePath) + ? fs + .readdirSync(storagePath) + .sort() + .map((file) => { + const stat = fs.statSync(path.join(storagePath, file)); + return { file, bytes: stat.size, directory: stat.isDirectory() }; + }) + : []; + let database = null; + if (fs.existsSync(lbugPath)) { + await adapter.initLbug(lbugPath); + try { + const nodes = await adapter.executeQuery('MATCH (n) RETURN count(n) AS count'); + const edges = await adapter.executeQuery('MATCH ()-[r]->() RETURN count(r) AS count'); + const embeddings = await adapter.executeQuery( + 'MATCH (e:CodeEmbedding) RETURN count(e) AS count', + ); + const embeddingDimensions = await adapter.executeQuery( + 'MATCH (e:CodeEmbedding) RETURN DISTINCT size(e.embedding) AS dimensions ORDER BY dimensions', + ); + let graphFingerprint = null; + if (fingerprint) { + const nodeTables = {}; + for (const table of schema.NODE_TABLES) { + const countRows = await adapter.executeQuery( + `MATCH (n:\`${table}\`) RETURN count(n) AS count`, + ); + const digest = crypto.createHash('sha256'); + const streamed = await adapter.streamQuery( + `MATCH (n:\`${table}\`) RETURN n.id AS id ORDER BY id`, + (row) => digest.update(`${JSON.stringify(String(row.id ?? ''))}\n`), + ); + const count = Number(countRows[0]?.count ?? 0); + if (streamed !== count) { + throw new Error( + `${name}: ${table} fingerprint streamed ${streamed} rows, expected ${count}`, + ); + } + nodeTables[table] = { count, idSha256: digest.digest('hex') }; + } + const relationshipTypes = await adapter.executeQuery( + `MATCH ()-[r:${schema.REL_TABLE_NAME}]->() ` + + 'RETURN r.type AS type, count(r) AS count ORDER BY type', + ); + const relationshipPairs = await adapter.executeQuery( + `MATCH (source)-[r:${schema.REL_TABLE_NAME}]->(target) ` + + 'RETURN labels(source) AS sourceLabel, r.type AS type, ' + + 'labels(target) AS targetLabel, count(r) AS count ' + + 'ORDER BY type, sourceLabel, targetLabel', + ); + const relationshipIdentityDigests = new Map(); + const relationshipIdentityCounts = new Map(); + await adapter.streamQuery( + `MATCH (source)-[r:${schema.REL_TABLE_NAME}]->(target) ` + + 'RETURN labels(source) AS sourceLabel, source.id AS sourceId, ' + + 'r.type AS type, labels(target) AS targetLabel, target.id AS targetId, ' + + 'r.confidence AS confidence, r.reason AS reason, r.step AS step ' + + 'ORDER BY type, sourceLabel, sourceId, targetLabel, targetId, ' + + 'confidence, reason, step', + (row) => { + const type = String(row.type); + let digest = relationshipIdentityDigests.get(type); + if (!digest) { + digest = crypto.createHash('sha256'); + relationshipIdentityDigests.set(type, digest); + } + digest.update( + `${JSON.stringify([ + String(row.sourceLabel), + String(row.sourceId), + String(row.targetLabel), + String(row.targetId), + Number(row.confidence), + String(row.reason ?? ''), + Number(row.step), + ])}\n`, + ); + relationshipIdentityCounts.set(type, (relationshipIdentityCounts.get(type) ?? 0) + 1); + }, + ); + const relationshipIdentities = Array.from(relationshipIdentityDigests.entries()) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([type, digest]) => ({ + type, + count: relationshipIdentityCounts.get(type) ?? 0, + sha256: digest.digest('hex'), + })); + graphFingerprint = { + nodeTables, + relationshipTypes: relationshipTypes.map((row) => ({ + type: String(row.type), + count: Number(row.count), + })), + relationshipPairs: relationshipPairs.map((row) => ({ + sourceLabel: String(row.sourceLabel), + type: String(row.type), + targetLabel: String(row.targetLabel), + count: Number(row.count), + })), + relationshipIdentities, + }; + } + database = { + nodes: Number(nodes[0]?.count ?? 0), + edges: Number(edges[0]?.count ?? 0), + embeddings: Number(embeddings[0]?.count ?? 0), + embeddingDimensions: embeddingDimensions.map((row) => Number(row.dimensions)), + graphFingerprint, + }; + } finally { + await adapter.closeLbug(); + } + } + const result = { + name, + at: new Date().toISOString(), + runId, + sourceSha, + head, + gitStatus: { + clean: status.length === 0, + entryCount: status.length === 0 ? 0 : status.split(/\r?\n/).length, + }, + embedding: { model, dimensions, server: { ...embeddingStats } }, + metadata: meta ?? null, + database, + index: { path: storagePath, sizeKb: indexKb, files }, + disk: disk.split(/\r?\n/).slice(-1)[0], + }; + writeJson(path.join(snapshotDir, `${name}.json`), result); + return result; +}; + +const waitForPauseAndTerminate = async (child, readyFile, timeoutMs) => { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (fs.existsSync(readyFile)) { + const dirty = JSON.parse(fs.readFileSync(readyFile, 'utf8')); + child.kill('SIGTERM'); + const result = await new Promise((resolve) => { + child.once('close', (code, signal) => resolve({ code, signal })); + }); + return { dirty, ...result }; + } + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error('incremental child exited before the escalation pause point'); + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + child.kill('SIGKILL'); + await new Promise((resolve) => child.once('close', resolve)); + throw new Error('timed out waiting for the escalation pause point'); +}; + +const interruptAtEscalation = async () => { + const name = 'wide-interrupted'; + const readyFile = path.join(evidence, 'pause-ready.json'); + if (fs.existsSync(readyFile)) { + throw new Error(`refusing stale escalation ready file: ${readyFile}`); + } + const stdoutPath = path.join(commandDir, `${name}.stdout.log`); + const stderrPath = path.join(commandDir, `${name}.stderr.log`); + const stdout = fs.openSync(stdoutPath, 'w'); + const stderr = fs.openSync(stderrPath, 'w'); + const startedAt = new Date().toISOString(); + const startedMs = Date.now(); + process.stdout.write(`[${runId}] ${name}\n`); + const childArgs = [ + incrementalChild, + worktree, + '--embeddings', + '--pause-on-escalation', + readyFile, + ]; + const child = spawn(process.execPath, childArgs, { + cwd: packageRoot, + env: canaryEnv, + stdio: ['ignore', stdout, stderr], + }); + let result; + try { + result = await waitForPauseAndTerminate(child, readyFile, escalationTimeoutMs); + } finally { + fs.closeSync(stdout); + fs.closeSync(stderr); + } + const entry = { + name, + command: process.execPath, + args: childArgs, + cwd: packageRoot, + startedAt, + finishedAt: new Date().toISOString(), + durationMs: Date.now() - startedMs, + timeoutMs: escalationTimeoutMs, + code: result.code, + signal: result.signal, + dirty: result.dirty, + stdout: path.basename(stdoutPath), + stderr: path.basename(stderrPath), + }; + commandLedger.push(entry); + writeJson(path.join(evidence, 'command-ledger.json'), commandLedger); + if (result.code === 0 || result.signal !== 'SIGTERM') { + throw new Error(`interrupted child did not fail by SIGTERM: ${JSON.stringify(result)}`); + } + return entry; +}; + +const assertSnapshot = (value, expectedHead) => { + if (value.head !== expectedHead) throw new Error(`${value.name}: metadata head mismatch`); + if (value.metadata?.lastCommit !== expectedHead) { + throw new Error(`${value.name}: persisted commit does not match repository head`); + } + if (!value.gitStatus.clean) throw new Error(`${value.name}: repository worktree is not clean`); + if (value.metadata?.incrementalInProgress) { + throw new Error(`${value.name}: dirty marker remained after successful analysis`); + } + if (!value.database || value.database.nodes <= 0 || value.database.edges <= 0) { + throw new Error(`${value.name}: graph is empty`); + } + if (value.database.embeddings <= 0) throw new Error(`${value.name}: embeddings are empty`); + if (JSON.stringify(value.database.embeddingDimensions) !== JSON.stringify([dimensions])) { + throw new Error(`${value.name}: unexpected embedding dimensions`); + } + if (Number(value.metadata?.stats?.embeddings ?? -1) !== value.database.embeddings) { + throw new Error(`${value.name}: metadata/database embedding counts differ`); + } + if (value.metadata?.capabilities?.fts?.status !== 'available') { + throw new Error(`${value.name}: FTS capability is not available`); + } + if (value.metadata?.capabilities?.vectorSearch?.status !== 'vector-index') { + throw new Error(`${value.name}: VECTOR capability is not vector-index`); + } +}; + +const compareSnapshots = (left, right) => { + const metadata = {}; + for (const key of ['files', 'nodes', 'edges', 'communities', 'processes', 'embeddings']) { + const leftValue = Number(left.metadata?.stats?.[key] ?? -1); + const rightValue = Number(right.metadata?.stats?.[key] ?? -1); + if (leftValue !== rightValue) metadata[key] = { left: leftValue, right: rightValue }; + } + const leftFingerprint = left.database?.graphFingerprint; + const rightFingerprint = right.database?.graphFingerprint; + if (!leftFingerprint || !rightFingerprint) { + return { equal: false, metadata, fingerprint: { missing: true } }; + } + const nodeTables = {}; + for (const table of Object.keys(leftFingerprint.nodeTables)) { + const leftTable = leftFingerprint.nodeTables[table]; + const rightTable = rightFingerprint.nodeTables[table]; + if (leftTable?.count !== rightTable?.count || leftTable?.idSha256 !== rightTable?.idSha256) { + nodeTables[table] = { left: leftTable ?? null, right: rightTable ?? null }; + } + } + const relationshipTypesEqual = + JSON.stringify(leftFingerprint.relationshipTypes) === + JSON.stringify(rightFingerprint.relationshipTypes); + const relationshipPairsEqual = + JSON.stringify(leftFingerprint.relationshipPairs) === + JSON.stringify(rightFingerprint.relationshipPairs); + const relationshipIdentitiesEqual = + JSON.stringify(leftFingerprint.relationshipIdentities) === + JSON.stringify(rightFingerprint.relationshipIdentities); + return { + equal: + Object.keys(metadata).length === 0 && + Object.keys(nodeTables).length === 0 && + relationshipTypesEqual && + relationshipPairsEqual && + relationshipIdentitiesEqual, + metadata, + fingerprint: { + nodeTables, + relationshipTypes: relationshipTypesEqual + ? null + : { + left: leftFingerprint.relationshipTypes, + right: rightFingerprint.relationshipTypes, + }, + relationshipPairs: relationshipPairsEqual + ? null + : { + left: leftFingerprint.relationshipPairs, + right: rightFingerprint.relationshipPairs, + }, + relationshipIdentities: relationshipIdentitiesEqual + ? null + : { + left: leftFingerprint.relationshipIdentities, + right: rightFingerprint.relationshipIdentities, + }, + }, + }; +}; + +try { + if (resumeFrom === 'forced') { + const forcedPath = path.join(snapshotDir, 'forced.json'); + if (!fs.existsSync(forcedPath)) { + throw new Error('forced resume requires the original forced snapshot'); + } + const originalForced = JSON.parse(fs.readFileSync(forcedPath, 'utf8')); + if (originalForced.runId !== runId || originalForced.sourceSha !== sourceSha) { + throw new Error('forced resume snapshot does not match this run or source SHA'); + } + const wideHead = await runText('forced-repeat-head', 'git', ['rev-parse', 'HEAD']); + const preflight = await snapshot('forced-repeat-preflight', { fingerprint: true }); + assertSnapshot(preflight, wideHead); + if (preflight.metadata?.lastCommit !== originalForced.metadata?.lastCommit) { + throw new Error('forced repeat preflight does not match the original forced commit'); + } + await run( + 'forced-repeat-rebuild', + process.execPath, + [incrementalChild, worktree, '--force', '--embeddings', '--fts-query', 'r19CanaryNeedleV2'], + { cwd: packageRoot, env: canaryEnv }, + ); + const repeated = await snapshot('forced-repeat', { fingerprint: true }); + assertSnapshot(repeated, wideHead); + const comparison = compareSnapshots(preflight, repeated); + writeJson(path.join(evidence, 'forced-repeat-comparison.json'), comparison); + if (!comparison.equal) { + throw new Error('consecutive forced rebuilds produced different graph fingerprints'); + } + writeJson(path.join(evidence, 'forced-repeat-summary.json'), { + runId, + result: 'passed', + sourceSha, + finalHead: wideHead, + preflight: preflight.metadata.stats, + repeated: repeated.metadata.stats, + comparison, + }); + process.stdout.write(`[${runId}] forced repeat passed\n`); + } else { + let initial; + let controlled; + let wideHead; + if (resumeFrom === 'wide') { + const initialPath = path.join(snapshotDir, 'initial.json'); + const controlledPath = path.join(snapshotDir, 'controlled-change.json'); + if (!fs.existsSync(initialPath) || !fs.existsSync(controlledPath)) { + throw new Error('wide resume requires initial and controlled-change snapshots'); + } + initial = JSON.parse(fs.readFileSync(initialPath, 'utf8')); + controlled = JSON.parse(fs.readFileSync(controlledPath, 'utf8')); + if (initial.runId !== runId || controlled.runId !== runId) { + throw new Error('wide resume snapshots do not belong to this run'); + } + if (initial.sourceSha !== sourceSha || controlled.sourceSha !== sourceSha) { + throw new Error('wide resume snapshots do not match the requested source SHA'); + } + if (!fs.existsSync(path.join(evidence, 'wide-write-set.json'))) { + throw new Error('wide resume requires the recorded write set'); + } + wideHead = await runText('wide-resume-head', 'git', ['rev-parse', 'HEAD']); + const preflight = await snapshot('wide-resume-preflight'); + if (!preflight.gitStatus.clean) throw new Error('wide resume worktree is not clean'); + if (preflight.metadata?.incrementalInProgress) { + throw new Error('wide resume refuses an existing dirty marker'); + } + if (preflight.metadata?.lastCommit !== controlled.head || wideHead === controlled.head) { + throw new Error('wide resume does not match the controlled-to-wide boundary'); + } + } else { + await run('clone', 'git', ['clone', '--shared', '--no-checkout', source, worktree], { + cwd: path.dirname(worktree), + env: process.env, + }); + await run('checkout-source-sha', 'git', ['checkout', '--detach', sourceSha]); + await run('set-public-origin', 'git', ['remote', 'set-url', 'origin', publicOrigin]); + const checkedOut = await runText('verify-source-sha', 'git', ['rev-parse', 'HEAD']); + if (checkedOut !== sourceSha) throw new Error(`source SHA mismatch: ${checkedOut}`); + const initialStatus = await runText('verify-initial-clean', 'git', [ + 'status', + '--porcelain=v1', + ]); + if (initialStatus) throw new Error('fresh canary clone is not clean'); + + await run( + 'initial-analyze', + process.execPath, + [cli, 'analyze', worktree, '--embeddings', '0', '--workers', '2', '--index-only'], + { cwd: packageRoot, env: canaryEnv }, + ); + initial = await snapshot('initial'); + assertSnapshot(initial, sourceSha); + await run( + 'initial-query', + process.execPath, + [cli, 'query', 'gateway authentication', '-r', worktree], + { + cwd: packageRoot, + env: canaryEnv, + }, + ); + + const fixtureRoot = path.join(worktree, 'src/r19-canary'); + fs.mkdirSync(fixtureRoot, { recursive: true }); + fs.writeFileSync( + path.join(fixtureRoot, 'hub.ts'), + 'export function r19CanaryHub(value: number): number {\n return value + 19;\n}\n', + ); + fs.writeFileSync( + path.join(fixtureRoot, 'spoke-a.ts'), + "import { r19CanaryHub } from './hub.js';\nexport const r19CanaryNeedle = r19CanaryHub(1);\n", + ); + fs.writeFileSync( + path.join(fixtureRoot, 'spoke-b.ts'), + "import { r19CanaryHub } from './hub.js';\nexport const r19CanarySecondary = r19CanaryHub(2);\n", + ); + await commitAll('r19 controlled add'); + await run( + 'controlled-add-analyze', + process.execPath, + [incrementalChild, worktree, '--embeddings', '--fts-query', 'r19CanaryNeedle'], + { cwd: packageRoot, env: canaryEnv }, + ); + const afterAddHead = await runText('after-add-head', 'git', ['rev-parse', 'HEAD']); + const afterAdd = await snapshot('controlled-add'); + assertSnapshot(afterAdd, afterAddHead); + + fs.appendFileSync(path.join(fixtureRoot, 'hub.ts'), '\n// r19 controlled edit\n'); + fs.renameSync( + path.join(fixtureRoot, 'spoke-a.ts'), + path.join(fixtureRoot, 'spoke-renamed.ts'), + ); + fs.rmSync(path.join(fixtureRoot, 'spoke-b.ts')); + fs.writeFileSync( + path.join(fixtureRoot, 'spoke-c.ts'), + "import { r19CanaryHub } from './hub.js';\nexport const r19CanaryNeedleV2 = r19CanaryHub(3);\n", + ); + await commitAll('r19 controlled edit rename delete add'); + await run( + 'controlled-change-analyze', + process.execPath, + [incrementalChild, worktree, '--embeddings', '--fts-query', 'r19CanaryNeedleV2'], + { cwd: packageRoot, env: canaryEnv }, + ); + const controlledHead = await runText('controlled-head', 'git', ['rev-parse', 'HEAD']); + controlled = await snapshot('controlled-change'); + assertSnapshot(controlled, controlledHead); + await run( + 'controlled-query', + process.execPath, + [cli, 'query', 'r19CanaryNeedleV2', '-r', worktree], + { + cwd: packageRoot, + env: canaryEnv, + }, + ); + await run( + 'controlled-context', + process.execPath, + [cli, 'context', 'r19CanaryHub', '-r', worktree], + { + cwd: packageRoot, + env: canaryEnv, + }, + ); + await run( + 'controlled-impact', + process.execPath, + [cli, 'impact', 'r19CanaryHub', '-r', worktree], + { + cwd: packageRoot, + env: canaryEnv, + }, + ); + + const tracked = (await runText('tracked-typescript', 'git', ['ls-files', '*.ts'])) + .split(/\r?\n/) + .filter((file) => file && !file.startsWith('src/r19-canary/')) + .map((file) => ({ file, bytes: fs.statSync(path.join(worktree, file)).size })) + .sort((left, right) => left.bytes - right.bytes || left.file.localeCompare(right.file)) + .slice(0, 51); + if (tracked.length !== 51) + throw new Error(`expected 51 TypeScript files, found ${tracked.length}`); + tracked.forEach(({ file }, index) => { + fs.appendFileSync(path.join(worktree, file), `\n// gitnexus-r19-wide-${index}\n`); + }); + writeJson(path.join(evidence, 'wide-write-set.json'), tracked); + await commitAll('r19 force incremental scale escalation'); + wideHead = await runText('wide-head', 'git', ['rev-parse', 'HEAD']); + } + + const interrupted = await interruptAtEscalation(); + if (interrupted.dirty?.dirty?.phase !== 'effective-write-set') { + throw new Error(`unexpected dirty phase: ${JSON.stringify(interrupted.dirty)}`); + } + if (Number(interrupted.dirty?.dirty?.effectiveWriteCount ?? 0) < 50) { + throw new Error('incremental escalation write set was smaller than 50 files'); + } + const dirtySnapshot = await snapshot('interrupted-dirty'); + if (!dirtySnapshot.metadata?.incrementalInProgress) { + throw new Error('interrupted analysis left no dirty marker'); + } + + await run( + 'resume-after-interruption', + process.execPath, + [incrementalChild, worktree, '--embeddings', '--fts-query', 'r19CanaryNeedleV2'], + { cwd: packageRoot, env: canaryEnv }, + ); + const recovered = await snapshot('recovered', { fingerprint: true }); + assertSnapshot(recovered, wideHead); + + await run( + 'forced-rebuild', + process.execPath, + [incrementalChild, worktree, '--force', '--embeddings', '--fts-query', 'r19CanaryNeedleV2'], + { cwd: packageRoot, env: canaryEnv }, + ); + const forced = await snapshot('forced', { fingerprint: true }); + assertSnapshot(forced, wideHead); + const recoveredForcedComparison = compareSnapshots(recovered, forced); + writeJson(path.join(evidence, 'recovered-forced-comparison.json'), recoveredForcedComparison); + if (!recoveredForcedComparison.equal) { + throw new Error('recovered and forced graphs produced different fingerprints'); + } + await run('final-status', process.execPath, [cli, 'status'], { cwd: worktree, env: canaryEnv }); + await run('final-doctor', process.execPath, [cli, 'doctor', worktree], { + cwd: packageRoot, + env: canaryEnv, + }); + await run( + 'final-query', + process.execPath, + [cli, 'query', 'r19CanaryNeedleV2', '-r', worktree], + { + cwd: packageRoot, + env: canaryEnv, + }, + ); + + const summary = { + runId, + result: 'passed', + sourceSha, + finalHead: wideHead, + publicOrigin, + embedding: { model, dimensions, ...embeddingStats }, + initial: initial.metadata.stats, + controlled: controlled.metadata.stats, + recovered: recovered.metadata.stats, + forced: forced.metadata.stats, + interrupted: interrupted.dirty, + worktree, + evidence, + }; + writeJson(path.join(evidence, 'summary.json'), summary); + process.stdout.write(`[${runId}] passed\n`); + } +} catch (error) { + writeJson(failurePath, { + runId, + at: new Date().toISOString(), + error: error instanceof Error ? (error.stack ?? error.message) : String(error), + embedding: { model, dimensions, ...embeddingStats }, + }); + process.stderr.write( + `${error instanceof Error ? (error.stack ?? error.message) : String(error)}\n`, + ); + process.exitCode = 1; +} finally { + await new Promise((resolve) => embeddingServer.close(resolve)); +} diff --git a/gitnexus/test/fixtures/large-incremental-child.mjs b/gitnexus/test/fixtures/large-incremental-child.mjs index 006d382bd1..7075b770c8 100644 --- a/gitnexus/test/fixtures/large-incremental-child.mjs +++ b/gitnexus/test/fixtures/large-incremental-child.mjs @@ -12,11 +12,12 @@ const bm25IndexUrl = pathToFileURL(path.join(packageRoot, 'dist/core/search/bm25 const [repoPath, ...args] = process.argv.slice(2); if (!repoPath) { throw new Error( - 'usage: large-incremental-child.mjs [--force] [--pause-on-escalation ] [--fts-query ]', + 'usage: large-incremental-child.mjs [--force] [--embeddings] [--pause-on-escalation ] [--fts-query ]', ); } const force = args.includes('--force'); +const embeddings = args.includes('--embeddings'); const pauseIndex = args.indexOf('--pause-on-escalation'); const pauseReadyFile = pauseIndex >= 0 ? args[pauseIndex + 1] : undefined; if (pauseIndex >= 0 && !pauseReadyFile) { @@ -53,7 +54,17 @@ const onLog = (message) => { }; try { - await runFullAnalysis(repoPath, { skipAgentsMd: true, force }, { onProgress: () => {}, onLog }); + await runFullAnalysis( + repoPath, + { + skipAgentsMd: true, + skipSkills: true, + force, + embeddings, + embeddingsNodeLimit: embeddings ? 0 : undefined, + }, + { onProgress: () => {}, onLog }, + ); const meta = await loadMeta(storagePath); const lbugStat = fs.statSync(lbugPath); From 889d3446be2bb2512b7b50c21a31359a411473b4 Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 03:35:00 +0700 Subject: [PATCH 02/16] test(reconciliation): isolate canary environment --- .../reconciliation/canary-environment.mjs | 33 +++++++++++ .../large-repository-canary.mjs | 13 +++-- .../reconciliation-canary-environment.test.ts | 58 +++++++++++++++++++ 3 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 gitnexus/scripts/reconciliation/canary-environment.mjs create mode 100644 gitnexus/test/unit/reconciliation-canary-environment.test.ts diff --git a/gitnexus/scripts/reconciliation/canary-environment.mjs b/gitnexus/scripts/reconciliation/canary-environment.mjs new file mode 100644 index 0000000000..292aa54fe0 --- /dev/null +++ b/gitnexus/scripts/reconciliation/canary-environment.mjs @@ -0,0 +1,33 @@ +const PORTABLE_KEYS = [ + 'PATH', + 'SHELL', + 'TMPDIR', + 'TMP', + 'TEMP', + 'LANG', + 'LC_ALL', + 'LC_CTYPE', + 'TZ', +]; + +const WINDOWS_KEYS = ['SystemRoot', 'WINDIR', 'COMSPEC', 'PATHEXT']; + +export const createSanitizedEnvironment = (source, { home, platform = process.platform } = {}) => { + if (!home) throw new Error('sanitized environment requires an isolated home'); + + const result = {}; + const allowedKeys = platform === 'win32' ? [...PORTABLE_KEYS, ...WINDOWS_KEYS] : PORTABLE_KEYS; + for (const key of allowedKeys) { + if (typeof source[key] === 'string') result[key] = source[key]; + } + + return { + ...result, + HOME: home, + USERPROFILE: home, + CI: '1', + GIT_TERMINAL_PROMPT: '0', + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: platform === 'win32' ? 'NUL' : '/dev/null', + }; +}; diff --git a/gitnexus/scripts/reconciliation/large-repository-canary.mjs b/gitnexus/scripts/reconciliation/large-repository-canary.mjs index e77142170b..ebb1433266 100644 --- a/gitnexus/scripts/reconciliation/large-repository-canary.mjs +++ b/gitnexus/scripts/reconciliation/large-repository-canary.mjs @@ -7,6 +7,8 @@ import http from 'node:http'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; +import { createSanitizedEnvironment } from './canary-environment.mjs'; + const REQUIRED_ARGS = [ 'source', 'source-sha', @@ -54,6 +56,7 @@ const home = path.join(evidence, 'home'); const commandDir = path.join(evidence, 'commands'); const snapshotDir = path.join(evidence, 'snapshots'); const failurePath = path.join(evidence, 'failure.json'); +const baseEnv = createSanitizedEnvironment(process.env, { home }); if (!Number.isInteger(dimensions) || dimensions <= 0) { throw new Error('--embedding-dims must be a positive integer'); @@ -129,7 +132,7 @@ const run = async (name, command, commandArgs, options = {}) => { process.stdout.write(`[${runId}] ${name}\n`); const child = spawn(command, commandArgs, { cwd: options.cwd ?? worktree, - env: options.env ?? process.env, + env: options.env ?? baseEnv, stdio: ['ignore', stdout, stderr], }); const result = await new Promise((resolve, reject) => { @@ -163,7 +166,7 @@ const runText = async (name, command, commandArgs, options = {}) => { const chunks = []; const child = spawn(command, commandArgs, { cwd: options.cwd ?? worktree, - env: options.env ?? process.env, + env: options.env ?? baseEnv, stdio: ['ignore', 'pipe', 'pipe'], }); child.stdout.on('data', (chunk) => chunks.push(chunk)); @@ -229,9 +232,7 @@ const address = embeddingServer.address(); if (!address || typeof address === 'string') throw new Error('embedding server has no TCP address'); const canaryEnv = { - ...process.env, - HOME: home, - USERPROFILE: home, + ...baseEnv, GITNEXUS_HOME: path.join(home, '.gitnexus'), CI: '1', NODE_OPTIONS: '--max-old-space-size=6144', @@ -652,7 +653,7 @@ try { } else { await run('clone', 'git', ['clone', '--shared', '--no-checkout', source, worktree], { cwd: path.dirname(worktree), - env: process.env, + env: baseEnv, }); await run('checkout-source-sha', 'git', ['checkout', '--detach', sourceSha]); await run('set-public-origin', 'git', ['remote', 'set-url', 'origin', publicOrigin]); diff --git a/gitnexus/test/unit/reconciliation-canary-environment.test.ts b/gitnexus/test/unit/reconciliation-canary-environment.test.ts new file mode 100644 index 0000000000..623665dbf4 --- /dev/null +++ b/gitnexus/test/unit/reconciliation-canary-environment.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; + +import { createSanitizedEnvironment } from '../../scripts/reconciliation/canary-environment.mjs'; + +describe('reconciliation canary environment', () => { + it('retains only process plumbing and replaces user Git configuration', () => { + const result = createSanitizedEnvironment( + { + PATH: '/usr/bin', + LANG: 'en_US.UTF-8', + TMPDIR: '/tmp/canary', + GH_TOKEN: 'github-secret', + GITHUB_TOKEN: 'actions-secret', + OPENAI_API_KEY: 'openai-secret', + AWS_SECRET_ACCESS_KEY: 'aws-secret', + npm_config_token: 'npm-secret', + }, + { home: '/tmp/isolated-home', platform: 'darwin' }, + ); + + expect(result).toEqual({ + PATH: '/usr/bin', + TMPDIR: '/tmp/canary', + LANG: 'en_US.UTF-8', + HOME: '/tmp/isolated-home', + USERPROFILE: '/tmp/isolated-home', + CI: '1', + GIT_TERMINAL_PROMPT: '0', + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + }); + expect(Object.values(result)).not.toContain('github-secret'); + expect(Object.values(result)).not.toContain('openai-secret'); + }); + + it('retains required Windows process keys without inheriting credentials', () => { + const result = createSanitizedEnvironment( + { + PATH: 'C:\\Windows\\System32', + SystemRoot: 'C:\\Windows', + COMSPEC: 'C:\\Windows\\System32\\cmd.exe', + GH_TOKEN: 'github-secret', + }, + { home: 'C:\\canary-home', platform: 'win32' }, + ); + + expect(result.SystemRoot).toBe('C:\\Windows'); + expect(result.COMSPEC).toBe('C:\\Windows\\System32\\cmd.exe'); + expect(result.GIT_CONFIG_GLOBAL).toBe('NUL'); + expect(result).not.toHaveProperty('GH_TOKEN'); + }); + + it('requires an isolated home', () => { + expect(() => createSanitizedEnvironment({ PATH: '/usr/bin' }, {})).toThrow( + 'sanitized environment requires an isolated home', + ); + }); +}); From df99e1e490bf228d2ec700b5e707779ce9dabda4 Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 03:40:26 +0700 Subject: [PATCH 03/16] fix(scope): stabilize graph lookup collisions --- .../graph-bridge/node-lookup.ts | 32 ++++++++-- .../node-lookup-determinism.test.ts | 59 +++++++++++++++++++ 2 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 gitnexus/test/unit/scope-resolution/node-lookup-determinism.test.ts diff --git a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts index 9ab1649a18..680b030dd0 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts @@ -18,7 +18,7 @@ * format that downstream consumers (queries, edges, MCP) expect. */ -import type { NodeLabel, ParameterTypeClass } from 'gitnexus-shared'; +import type { GraphNode, NodeLabel, ParameterTypeClass } from 'gitnexus-shared'; import type { KnowledgeGraph } from '../../../graph/types.js'; import { isOverloadableCallable } from '../../utils/callable-labels.js'; import { templateConstraintsIdTag } from '../../utils/template-arguments.js'; @@ -67,9 +67,34 @@ export function simpleKey(filePath: string, name: string): string { return `${filePath}::${name}`; } +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function compareSourceOrder(left: GraphNode, right: GraphNode): number { + const fileOrder = compareText(left.properties.filePath, right.properties.filePath); + if (fileOrder !== 0) return fileOrder; + + const leftLine = Number.isFinite(left.properties.startLine) + ? (left.properties.startLine ?? Number.MAX_SAFE_INTEGER) + : Number.MAX_SAFE_INTEGER; + const rightLine = Number.isFinite(right.properties.startLine) + ? (right.properties.startLine ?? Number.MAX_SAFE_INTEGER) + : Number.MAX_SAFE_INTEGER; + if (leftLine !== rightLine) return leftLine - rightLine; + + return compareText(left.id, right.id); +} + export function buildGraphNodeLookup(graph: KnowledgeGraph): GraphNodeLookup { const lookup = new Map(); - for (const node of graph.iterNodes()) { + const linkableNodes = Array.from(graph.iterNodes()).filter((node) => { + const props = node.properties as { filePath?: string; name?: string }; + return props.filePath !== undefined && props.name !== undefined && isLinkableLabel(node.label); + }); + linkableNodes.sort(compareSourceOrder); + + for (const node of linkableNodes) { const props = node.properties as { filePath?: string; name?: string; @@ -77,7 +102,6 @@ export function buildGraphNodeLookup(graph: KnowledgeGraph): GraphNodeLookup { templateArguments?: readonly string[]; }; if (props.filePath === undefined || props.name === undefined) continue; - if (!isLinkableLabel(node.label)) continue; // Primary key: fully-qualified name + label, in a separate // keyspace from simple names. Class nodes carry `qualifiedName` @@ -163,7 +187,7 @@ export function buildGraphNodeLookup(graph: KnowledgeGraph): GraphNodeLookup { } } - // Fallback key: simple name. First-wins within a file — used when + // Fallback key: simple name. Source-order first-wins within a file — used when // the caller doesn't know the qualifier (unqualified free-call // fallback, cross-file resolution where MethodRegistry already // disambiguated the owner). diff --git a/gitnexus/test/unit/scope-resolution/node-lookup-determinism.test.ts b/gitnexus/test/unit/scope-resolution/node-lookup-determinism.test.ts new file mode 100644 index 0000000000..06cc8442d0 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/node-lookup-determinism.test.ts @@ -0,0 +1,59 @@ +import type { NodeLabel } from 'gitnexus-shared'; +import { describe, expect, it } from 'vitest'; + +import { createKnowledgeGraph } from '../../../src/core/graph/graph.js'; +import { + buildGraphNodeLookup, + qualifiedKey, + simpleKey, +} from '../../../src/core/ingestion/scope-resolution/graph-bridge/node-lookup.js'; + +const FILE = 'src/service.ts'; + +interface Candidate { + id: string; + startLine: number; +} + +function buildLookup(candidates: readonly Candidate[]) { + const graph = createKnowledgeGraph(); + for (const candidate of candidates) { + graph.addNode({ + id: candidate.id, + label: 'Method' as NodeLabel, + properties: { + name: 'save', + qualifiedName: 'Service.save', + filePath: FILE, + startLine: candidate.startLine, + }, + }); + } + return buildGraphNodeLookup(graph); +} + +describe('buildGraphNodeLookup determinism', () => { + it('selects the earliest source definition regardless of graph insertion order', () => { + const early = { id: `Method:${FILE}:Service.save#1`, startLine: 10 }; + const late = { id: `Method:${FILE}:Service.save#2`, startLine: 20 }; + + const lateFirst = buildLookup([late, early]); + const earlyFirst = buildLookup([early, late]); + + for (const key of [simpleKey(FILE, 'save'), qualifiedKey(FILE, 'Method', 'Service.save')]) { + expect(lateFirst.get(key)).toBe(early.id); + expect(earlyFirst.get(key)).toBe(early.id); + } + }); + + it('uses the stable node id when source positions are identical', () => { + const first = { id: `Method:${FILE}:Service.save#1`, startLine: 10 }; + const second = { id: `Method:${FILE}:Service.save#2`, startLine: 10 }; + + const firstLookup = buildLookup([second, first]); + const secondLookup = buildLookup([first, second]); + + expect(firstLookup.get(simpleKey(FILE, 'save'))).toBe(first.id); + expect(secondLookup.get(simpleKey(FILE, 'save'))).toBe(first.id); + }); +}); From 69a1ef53076a4b5708be59081a475c725088b56e Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 03:44:53 +0700 Subject: [PATCH 04/16] fix(scan): canonicalize repository path order --- .../src/core/ingestion/filesystem-walker.ts | 5 +++ .../test/unit/filesystem-walker-order.test.ts | 39 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 gitnexus/test/unit/filesystem-walker-order.test.ts diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index 0af28a9586..407d1aad77 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -83,6 +83,11 @@ export const walkRepositoryPaths = async ( } } + // Filesystem/glob traversal order is not stable across filesystems or repeated + // scans. Canonicalize once at the scan boundary so structure, scope resolution, + // suffix indexes, and parse chunking all observe the same repository order. + entries.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0)); + if (skippedLarge > 0) { const isDefault = maxFileSizeBytes === DEFAULT_MAX_FILE_SIZE_BYTES; const isOverrideUnset = !process.env.GITNEXUS_MAX_FILE_SIZE; diff --git a/gitnexus/test/unit/filesystem-walker-order.test.ts b/gitnexus/test/unit/filesystem-walker-order.test.ts new file mode 100644 index 0000000000..7b75128a47 --- /dev/null +++ b/gitnexus/test/unit/filesystem-walker-order.test.ts @@ -0,0 +1,39 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { glob } from 'glob'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('glob', () => ({ glob: vi.fn() })); +vi.mock('../../src/config/ignore-service.js', () => ({ + createIgnoreFilter: vi.fn(async () => []), +})); + +import { walkRepositoryPaths } from '../../src/core/ingestion/filesystem-walker.js'; + +const temporaryRoots: string[] = []; + +afterEach(async () => { + vi.mocked(glob).mockReset(); + await Promise.all( + temporaryRoots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })), + ); +}); + +describe('walkRepositoryPaths ordering', () => { + it('returns accepted files in canonical path order when glob order is unstable', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-scan-order-')); + temporaryRoots.push(root); + await Promise.all( + ['zeta.ts', 'alpha.ts', 'middle.ts'].map((file) => + fs.writeFile(path.join(root, file), `export const ${file[0]} = true;\n`), + ), + ); + vi.mocked(glob).mockResolvedValue(['zeta.ts', 'alpha.ts', 'middle.ts']); + + const result = await walkRepositoryPaths(root); + + expect(result.map((entry) => entry.path)).toEqual(['alpha.ts', 'middle.ts', 'zeta.ts']); + }); +}); From e554197c9f8e28f25335327b767794dd20449754 Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 04:09:28 +0700 Subject: [PATCH 05/16] fix(imports): stabilize suffix lookup without reordering phases --- .../src/core/ingestion/filesystem-walker.ts | 5 --- .../core/ingestion/import-resolvers/utils.ts | 17 ++++++-- .../test/unit/filesystem-walker-order.test.ts | 39 ------------------- .../test/unit/suffix-index-ambiguity.test.ts | 14 +++++++ 4 files changed, 28 insertions(+), 47 deletions(-) delete mode 100644 gitnexus/test/unit/filesystem-walker-order.test.ts diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index 407d1aad77..0af28a9586 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -83,11 +83,6 @@ export const walkRepositoryPaths = async ( } } - // Filesystem/glob traversal order is not stable across filesystems or repeated - // scans. Canonicalize once at the scan boundary so structure, scope resolution, - // suffix indexes, and parse chunking all observe the same repository order. - entries.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0)); - if (skippedLarge > 0) { const isDefault = maxFileSizeBytes === DEFAULT_MAX_FILE_SIZE_BYTES; const isOverrideUnset = !process.env.GITNEXUS_MAX_FILE_SIZE; diff --git a/gitnexus/src/core/ingestion/import-resolvers/utils.ts b/gitnexus/src/core/ingestion/import-resolvers/utils.ts index 6a033c1ee7..6737335d5e 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/utils.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/utils.ts @@ -95,9 +95,20 @@ export function buildSuffixIndex(normalizedFileList: string[], allFileList: stri // Map: directory suffix -> list of file paths in that directory const dirMap = new Map(); - for (let i = 0; i < normalizedFileList.length; i++) { - const normalized = normalizedFileList[i]; - const original = allFileList[i]; + // First-wins suffixes must not inherit filesystem or parser insertion order. + // Sort a paired copy so callers keep their phase-specific file order. + const orderedFiles = normalizedFileList.map((normalized, index) => ({ + normalized, + original: allFileList[index], + })); + orderedFiles.sort((left, right) => { + if (left.normalized !== right.normalized) { + return left.normalized < right.normalized ? -1 : 1; + } + return left.original < right.original ? -1 : left.original > right.original ? 1 : 0; + }); + + for (const { normalized, original } of orderedFiles) { const parts = normalized.split('/'); // Index all suffixes: "a/b/c.java" -> ["c.java", "b/c.java", "a/b/c.java"] diff --git a/gitnexus/test/unit/filesystem-walker-order.test.ts b/gitnexus/test/unit/filesystem-walker-order.test.ts deleted file mode 100644 index 7b75128a47..0000000000 --- a/gitnexus/test/unit/filesystem-walker-order.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import fs from 'node:fs/promises'; -import os from 'node:os'; -import path from 'node:path'; - -import { glob } from 'glob'; -import { afterEach, describe, expect, it, vi } from 'vitest'; - -vi.mock('glob', () => ({ glob: vi.fn() })); -vi.mock('../../src/config/ignore-service.js', () => ({ - createIgnoreFilter: vi.fn(async () => []), -})); - -import { walkRepositoryPaths } from '../../src/core/ingestion/filesystem-walker.js'; - -const temporaryRoots: string[] = []; - -afterEach(async () => { - vi.mocked(glob).mockReset(); - await Promise.all( - temporaryRoots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })), - ); -}); - -describe('walkRepositoryPaths ordering', () => { - it('returns accepted files in canonical path order when glob order is unstable', async () => { - const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-scan-order-')); - temporaryRoots.push(root); - await Promise.all( - ['zeta.ts', 'alpha.ts', 'middle.ts'].map((file) => - fs.writeFile(path.join(root, file), `export const ${file[0]} = true;\n`), - ), - ); - vi.mocked(glob).mockResolvedValue(['zeta.ts', 'alpha.ts', 'middle.ts']); - - const result = await walkRepositoryPaths(root); - - expect(result.map((entry) => entry.path)).toEqual(['alpha.ts', 'middle.ts', 'zeta.ts']); - }); -}); diff --git a/gitnexus/test/unit/suffix-index-ambiguity.test.ts b/gitnexus/test/unit/suffix-index-ambiguity.test.ts index fd786c045a..28cfe280aa 100644 --- a/gitnexus/test/unit/suffix-index-ambiguity.test.ts +++ b/gitnexus/test/unit/suffix-index-ambiguity.test.ts @@ -172,6 +172,20 @@ describe('resolvePythonImport — namespace packages (no __init__.py)', () => { }); }); +// --------------------------------------------------------------------------- +// Suffix index determinism +// --------------------------------------------------------------------------- + +describe('buildSuffixIndex determinism', () => { + it('selects the same ambiguous suffix target regardless of input order', () => { + const first = makeCtx(['packages/zeta/user.ts', 'packages/alpha/user.ts']); + const reversed = makeCtx(['packages/alpha/user.ts', 'packages/zeta/user.ts']); + + expect(first.index.get('user.ts')).toBe('packages/alpha/user.ts'); + expect(reversed.index.get('user.ts')).toBe('packages/alpha/user.ts'); + }); +}); + // --------------------------------------------------------------------------- // Ruby: bare require does NOT use proximity // --------------------------------------------------------------------------- From 7f081ffec3844594c996c17506bee52ad3a6e339 Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 04:14:31 +0700 Subject: [PATCH 06/16] test(imports): preserve caller file order --- gitnexus/src/core/ingestion/import-resolvers/utils.ts | 2 +- gitnexus/test/unit/suffix-index-ambiguity.test.ts | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/gitnexus/src/core/ingestion/import-resolvers/utils.ts b/gitnexus/src/core/ingestion/import-resolvers/utils.ts index 6737335d5e..f556769737 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/utils.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/utils.ts @@ -114,7 +114,7 @@ export function buildSuffixIndex(normalizedFileList: string[], allFileList: stri // Index all suffixes: "a/b/c.java" -> ["c.java", "b/c.java", "a/b/c.java"] for (let j = parts.length - 1; j >= 0; j--) { const suffix = parts.slice(j).join('/'); - // Only store first match (longest path wins for ambiguous suffixes) + // Only store the canonical first match for ambiguous suffixes. if (!exactMap.has(suffix)) { exactMap.set(suffix, original); } diff --git a/gitnexus/test/unit/suffix-index-ambiguity.test.ts b/gitnexus/test/unit/suffix-index-ambiguity.test.ts index 28cfe280aa..eb54c062cb 100644 --- a/gitnexus/test/unit/suffix-index-ambiguity.test.ts +++ b/gitnexus/test/unit/suffix-index-ambiguity.test.ts @@ -178,11 +178,15 @@ describe('resolvePythonImport — namespace packages (no __init__.py)', () => { describe('buildSuffixIndex determinism', () => { it('selects the same ambiguous suffix target regardless of input order', () => { - const first = makeCtx(['packages/zeta/user.ts', 'packages/alpha/user.ts']); - const reversed = makeCtx(['packages/alpha/user.ts', 'packages/zeta/user.ts']); + const firstInput = ['packages/zeta/user.ts', 'packages/alpha/user.ts']; + const reversedInput = [...firstInput].reverse(); + const first = makeCtx(firstInput); + const reversed = makeCtx(reversedInput); expect(first.index.get('user.ts')).toBe('packages/alpha/user.ts'); expect(reversed.index.get('user.ts')).toBe('packages/alpha/user.ts'); + expect(first.files).toEqual(firstInput); + expect(reversed.files).toEqual(reversedInput); }); }); From 67370753fe190c28d1f897fb9d5acba414ead18f Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 04:40:03 +0700 Subject: [PATCH 07/16] test(reconciliation): harden canary recovery evidence --- .../scripts/reconciliation/canary-safety.mjs | 253 ++++++++++++++++++ .../large-repository-canary.mjs | 112 ++++---- gitnexus/src/core/lbug/lbug-adapter.ts | 30 ++- gitnexus/src/core/run-analyze.ts | 66 ++++- .../test/fixtures/large-incremental-child.mjs | 42 +-- .../large-incremental-subprocess-e2e.test.ts | 106 +++++--- .../unit/incremental-orchestration.test.ts | 20 +- .../unit/reconciliation-canary-safety.test.ts | 223 +++++++++++++++ 8 files changed, 735 insertions(+), 117 deletions(-) create mode 100644 gitnexus/scripts/reconciliation/canary-safety.mjs create mode 100644 gitnexus/test/unit/reconciliation-canary-safety.test.ts diff --git a/gitnexus/scripts/reconciliation/canary-safety.mjs b/gitnexus/scripts/reconciliation/canary-safety.mjs new file mode 100644 index 0000000000..a82fb9d6ed --- /dev/null +++ b/gitnexus/scripts/reconciliation/canary-safety.mjs @@ -0,0 +1,253 @@ +import { spawn, spawnSync } from 'node:child_process'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +const SIGNAL_EXIT_CODES = { SIGINT: 130, SIGTERM: 143 }; + +const codePointCompare = (left, right) => (left < right ? -1 : left > right ? 1 : 0); + +const assertDirectory = (directory) => { + const stat = fs.lstatSync(directory); + if (stat.isSymbolicLink()) { + throw new Error(`refusing symbolic link in canary evidence path: ${directory}`); + } + if (!stat.isDirectory()) { + throw new Error(`expected canary evidence directory: ${directory}`); + } +}; + +const assertTreeHasNoSymlinks = (directory) => { + assertDirectory(directory); + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const entryPath = path.join(directory, entry.name); + if (entry.isSymbolicLink()) { + throw new Error(`refusing symbolic link in canary evidence path: ${entryPath}`); + } + if (entry.isDirectory()) assertTreeHasNoSymlinks(entryPath); + } +}; + +export const prepareEvidenceLayout = ({ evidence, commandDir, snapshotDir, home }) => { + fs.mkdirSync(evidence, { recursive: true }); + assertDirectory(evidence); + for (const directory of [commandDir, snapshotDir, home]) { + fs.mkdirSync(directory, { recursive: true }); + assertDirectory(directory); + } + assertTreeHasNoSymlinks(commandDir); + assertTreeHasNoSymlinks(snapshotDir); + assertTreeHasNoSymlinks(home); +}; + +export const writeJsonAtomic = (filePath, value) => { + assertDirectory(path.dirname(filePath)); + const temporaryPath = path.join( + path.dirname(filePath), + `.${path.basename(filePath)}.${process.pid}.${crypto.randomUUID()}.tmp`, + ); + try { + fs.writeFileSync(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, { + encoding: 'utf8', + flag: 'wx', + mode: 0o600, + }); + fs.renameSync(temporaryPath, filePath); + } finally { + fs.rmSync(temporaryPath, { force: true }); + } +}; + +const artifactStem = (name, attempt) => (attempt === 1 ? name : `${name}-attempt-${attempt}`); + +export const openArtifactLogs = (directory, name) => { + assertDirectory(directory); + for (let attempt = 1; attempt < 10_000; attempt += 1) { + const stem = artifactStem(name, attempt); + const stdoutPath = path.join(directory, `${stem}.stdout.log`); + const stderrPath = path.join(directory, `${stem}.stderr.log`); + let stdout; + try { + stdout = fs.openSync(stdoutPath, 'wx', 0o600); + const stderr = fs.openSync(stderrPath, 'wx', 0o600); + return { stdoutPath, stderrPath, stdout, stderr }; + } catch (error) { + if (stdout !== undefined) { + fs.closeSync(stdout); + fs.rmSync(stdoutPath, { force: true }); + } + if (error?.code !== 'EEXIST') throw error; + } + } + throw new Error(`could not allocate unique command artifacts for ${name}`); +}; + +export const writeJsonArtifact = (directory, name, value) => { + assertDirectory(directory); + for (let attempt = 1; attempt < 10_000; attempt += 1) { + const filePath = path.join(directory, `${artifactStem(name, attempt)}.json`); + try { + fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, { + encoding: 'utf8', + flag: 'wx', + mode: 0o600, + }); + return filePath; + } catch (error) { + if (error?.code !== 'EEXIST') throw error; + } + } + throw new Error(`could not allocate unique JSON artifact for ${name}`); +}; + +const isContained = (root, candidate) => { + const relative = path.relative(root, candidate); + return ( + relative === '' || + (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)) + ); +}; + +const assertNoSymlinkComponents = (root, candidate) => { + const relative = path.relative(root, candidate); + let cursor = root; + for (const component of relative.split(path.sep).filter(Boolean)) { + cursor = path.join(cursor, component); + if (fs.lstatSync(cursor).isSymbolicLink()) { + throw new Error(`refusing symbolic link in canary worktree path: ${cursor}`); + } + } +}; + +export const createSafeContainedDirectory = (root, relativePath) => { + const resolvedRoot = path.resolve(root); + const candidate = path.resolve(resolvedRoot, relativePath); + if (!isContained(resolvedRoot, candidate) || candidate === resolvedRoot) { + throw new Error(`refusing directory outside canary worktree: ${relativePath}`); + } + if (fs.existsSync(candidate)) { + throw new Error(`refusing existing canary fixture directory: ${candidate}`); + } + let cursor = resolvedRoot; + for (const component of path.relative(resolvedRoot, candidate).split(path.sep)) { + cursor = path.join(cursor, component); + if (fs.existsSync(cursor)) { + const stat = fs.lstatSync(cursor); + if (stat.isSymbolicLink()) { + throw new Error(`refusing symbolic link in canary worktree path: ${cursor}`); + } + if (!stat.isDirectory()) { + throw new Error(`refusing non-directory in canary worktree path: ${cursor}`); + } + } else { + fs.mkdirSync(cursor, { mode: 0o700 }); + } + } + const realRoot = fs.realpathSync(resolvedRoot); + const realCandidate = fs.realpathSync(candidate); + if (!isContained(realRoot, realCandidate)) { + throw new Error(`refusing directory outside real canary worktree: ${relativePath}`); + } + return candidate; +}; + +export const selectSafeTrackedFiles = (worktree, files, count) => { + const realWorktree = fs.realpathSync(worktree); + const candidates = files.map((file) => { + if (!file || path.isAbsolute(file)) { + throw new Error(`refusing unsafe tracked path: ${JSON.stringify(file)}`); + } + const candidate = path.resolve(worktree, file); + if (!isContained(path.resolve(worktree), candidate)) { + throw new Error(`refusing tracked path outside canary worktree: ${file}`); + } + assertNoSymlinkComponents(path.resolve(worktree), candidate); + const stat = fs.lstatSync(candidate); + if (stat.isSymbolicLink()) { + throw new Error(`refusing tracked symbolic link in canary mutation set: ${file}`); + } + if (!stat.isFile()) { + throw new Error(`refusing non-file in canary mutation set: ${file}`); + } + const realCandidate = fs.realpathSync(candidate); + if (!isContained(realWorktree, realCandidate)) { + throw new Error(`refusing tracked path outside real canary worktree: ${file}`); + } + return { file, bytes: stat.size }; + }); + return candidates + .sort((left, right) => left.bytes - right.bytes || codePointCompare(left.file, right.file)) + .slice(0, count); +}; + +export class ChildSupervisor { + constructor({ platform = process.platform, killTimeoutMs = 5_000 } = {}) { + this.platform = platform; + this.killTimeoutMs = killTimeoutMs; + this.children = new Set(); + this.receivedSignal = undefined; + this.signalHandlers = new Map(); + } + + spawn(command, args, options = {}) { + if (this.receivedSignal) { + throw new Error(`refusing child spawn after ${this.receivedSignal}`); + } + const child = spawn(command, args, { + ...options, + detached: this.platform !== 'win32', + }); + this.children.add(child); + child.once('close', () => this.children.delete(child)); + return child; + } + + terminate(child, signal = 'SIGTERM') { + if (!child?.pid || child.exitCode !== null || child.signalCode !== null) return; + if (this.platform === 'win32') { + const result = spawnSync('taskkill', ['/pid', String(child.pid), '/t', '/f'], { + stdio: 'ignore', + windowsHide: true, + }); + if (result.error || result.status !== 0) child.kill(signal); + return; + } + try { + process.kill(-child.pid, signal); + } catch (error) { + if (error?.code !== 'ESRCH') child.kill(signal); + } + } + + terminateAll(signal = 'SIGTERM') { + const children = [...this.children]; + for (const child of children) this.terminate(child, signal); + if (children.length > 0) { + const timer = setTimeout(() => { + for (const child of this.children) this.terminate(child, 'SIGKILL'); + }, this.killTimeoutMs); + timer.unref(); + } + } + + installSignalHandlers(targetProcess = process, onSignal = () => {}) { + for (const signal of ['SIGINT', 'SIGTERM']) { + const handler = () => { + if (this.receivedSignal) return; + this.receivedSignal = signal; + targetProcess.exitCode = SIGNAL_EXIT_CODES[signal]; + this.terminateAll(signal); + onSignal(signal); + }; + this.signalHandlers.set(signal, handler); + targetProcess.on(signal, handler); + } + } + + disposeSignalHandlers(targetProcess = process) { + for (const [signal, handler] of this.signalHandlers) { + targetProcess.off(signal, handler); + } + this.signalHandlers.clear(); + } +} diff --git a/gitnexus/scripts/reconciliation/large-repository-canary.mjs b/gitnexus/scripts/reconciliation/large-repository-canary.mjs index ebb1433266..7995334e98 100644 --- a/gitnexus/scripts/reconciliation/large-repository-canary.mjs +++ b/gitnexus/scripts/reconciliation/large-repository-canary.mjs @@ -1,6 +1,5 @@ #!/usr/bin/env node -import { spawn } from 'node:child_process'; import crypto from 'node:crypto'; import fs from 'node:fs'; import http from 'node:http'; @@ -8,6 +7,15 @@ import path from 'node:path'; import { pathToFileURL } from 'node:url'; import { createSanitizedEnvironment } from './canary-environment.mjs'; +import { + ChildSupervisor, + createSafeContainedDirectory, + openArtifactLogs, + prepareEvidenceLayout, + selectSafeTrackedFiles, + writeJsonArtifact, + writeJsonAtomic, +} from './canary-safety.mjs'; const REQUIRED_ARGS = [ 'source', @@ -77,9 +85,7 @@ for (const requiredPath of [source, cli, incrementalChild, extensionSource]) { if (!fs.existsSync(requiredPath)) throw new Error(`required path does not exist: ${requiredPath}`); } -fs.mkdirSync(commandDir, { recursive: true }); -fs.mkdirSync(snapshotDir, { recursive: true }); -fs.mkdirSync(home, { recursive: true }); +prepareEvidenceLayout({ evidence, commandDir, snapshotDir, home }); if (resumeFrom && fs.existsSync(failurePath)) { let archiveIndex = 1; let archivedFailurePath; @@ -94,9 +100,7 @@ if (resumeFrom && fs.existsSync(failurePath)) { } fs.cpSync(extensionSource, path.join(home, '.lbdb', 'extension'), { recursive: true }); -const writeJson = (filePath, value) => { - fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, 'utf8'); -}; +const writeJson = writeJsonAtomic; const extensionFiles = []; const collectExtensionFiles = (dir) => { @@ -121,26 +125,31 @@ const commandLedger = ? JSON.parse(fs.readFileSync(commandLedgerPath, 'utf8')) : []; const embeddingStats = { requests: 0, items: 0 }; +const childSupervisor = new ChildSupervisor(); const run = async (name, command, commandArgs, options = {}) => { - const stdoutPath = path.join(commandDir, `${name}.stdout.log`); - const stderrPath = path.join(commandDir, `${name}.stderr.log`); - const stdout = fs.openSync(stdoutPath, 'w'); - const stderr = fs.openSync(stderrPath, 'w'); + const { stdoutPath, stderrPath, stdout, stderr } = openArtifactLogs(commandDir, name); const startedAt = new Date().toISOString(); const startedMs = Date.now(); process.stdout.write(`[${runId}] ${name}\n`); - const child = spawn(command, commandArgs, { + const child = childSupervisor.spawn(command, commandArgs, { cwd: options.cwd ?? worktree, env: options.env ?? baseEnv, stdio: ['ignore', stdout, stderr], }); - const result = await new Promise((resolve, reject) => { - child.once('error', reject); - child.once('close', (code, signal) => resolve({ code, signal })); - }); - fs.closeSync(stdout); - fs.closeSync(stderr); + let childError; + let result = { code: null, signal: null }; + try { + result = await new Promise((resolve, reject) => { + child.once('error', reject); + child.once('close', (code, signal) => resolve({ code, signal })); + }); + } catch (error) { + childError = error; + } finally { + fs.closeSync(stdout); + fs.closeSync(stderr); + } const entry = { name, command, @@ -151,11 +160,13 @@ const run = async (name, command, commandArgs, options = {}) => { durationMs: Date.now() - startedMs, code: result.code, signal: result.signal, + error: childError instanceof Error ? childError.message : undefined, stdout: path.basename(stdoutPath), stderr: path.basename(stderrPath), }; commandLedger.push(entry); writeJson(path.join(evidence, 'command-ledger.json'), commandLedger); + if (childError) throw childError; if (!options.allowFailure && (result.code !== 0 || result.signal !== null)) { throw new Error(`${name} failed with code=${result.code} signal=${result.signal}`); } @@ -164,7 +175,7 @@ const run = async (name, command, commandArgs, options = {}) => { const runText = async (name, command, commandArgs, options = {}) => { const chunks = []; - const child = spawn(command, commandArgs, { + const child = childSupervisor.spawn(command, commandArgs, { cwd: options.cwd ?? worktree, env: options.env ?? baseEnv, stdio: ['ignore', 'pipe', 'pipe'], @@ -230,6 +241,7 @@ await new Promise((resolve, reject) => { }); const address = embeddingServer.address(); if (!address || typeof address === 'string') throw new Error('embedding server has no TCP address'); +childSupervisor.installSignalHandlers(process, () => embeddingServer.close()); const canaryEnv = { ...baseEnv, @@ -413,16 +425,22 @@ const snapshot = async (name, { fingerprint = false } = {}) => { index: { path: storagePath, sizeKb: indexKb, files }, disk: disk.split(/\r?\n/).slice(-1)[0], }; - writeJson(path.join(snapshotDir, `${name}.json`), result); + const snapshotPath = writeJsonArtifact(snapshotDir, name, result); + result.artifact = path.basename(snapshotPath); return result; }; const waitForPauseAndTerminate = async (child, readyFile, timeoutMs) => { const deadline = Date.now() + timeoutMs; + let childError; + child.once('error', (error) => { + childError = error; + }); while (Date.now() < deadline) { + if (childError) throw childError; if (fs.existsSync(readyFile)) { const dirty = JSON.parse(fs.readFileSync(readyFile, 'utf8')); - child.kill('SIGTERM'); + childSupervisor.terminate(child, 'SIGTERM'); const result = await new Promise((resolve) => { child.once('close', (code, signal) => resolve({ code, signal })); }); @@ -433,21 +451,18 @@ const waitForPauseAndTerminate = async (child, readyFile, timeoutMs) => { } await new Promise((resolve) => setTimeout(resolve, 100)); } - child.kill('SIGKILL'); + childSupervisor.terminate(child, 'SIGKILL'); await new Promise((resolve) => child.once('close', resolve)); throw new Error('timed out waiting for the escalation pause point'); }; -const interruptAtEscalation = async () => { - const name = 'wide-interrupted'; - const readyFile = path.join(evidence, 'pause-ready.json'); +const interruptAtRecoveryBoundary = async (boundary) => { + const name = `wide-interrupted-${boundary}`; + const readyFile = path.join(evidence, `pause-ready-${boundary}.json`); if (fs.existsSync(readyFile)) { throw new Error(`refusing stale escalation ready file: ${readyFile}`); } - const stdoutPath = path.join(commandDir, `${name}.stdout.log`); - const stderrPath = path.join(commandDir, `${name}.stderr.log`); - const stdout = fs.openSync(stdoutPath, 'w'); - const stderr = fs.openSync(stderrPath, 'w'); + const { stdoutPath, stderrPath, stdout, stderr } = openArtifactLogs(commandDir, name); const startedAt = new Date().toISOString(); const startedMs = Date.now(); process.stdout.write(`[${runId}] ${name}\n`); @@ -455,10 +470,12 @@ const interruptAtEscalation = async () => { incrementalChild, worktree, '--embeddings', - '--pause-on-escalation', + '--pause-at', + boundary, + '--pause-ready', readyFile, ]; - const child = spawn(process.execPath, childArgs, { + const child = childSupervisor.spawn(process.execPath, childArgs, { cwd: packageRoot, env: canaryEnv, stdio: ['ignore', stdout, stderr], @@ -488,7 +505,9 @@ const interruptAtEscalation = async () => { commandLedger.push(entry); writeJson(path.join(evidence, 'command-ledger.json'), commandLedger); if (result.code === 0 || result.signal !== 'SIGTERM') { - throw new Error(`interrupted child did not fail by SIGTERM: ${JSON.stringify(result)}`); + throw new Error( + `child interrupted at ${boundary} did not fail by SIGTERM: ${JSON.stringify(result)}`, + ); } return entry; }; @@ -606,11 +625,11 @@ try { const repeated = await snapshot('forced-repeat', { fingerprint: true }); assertSnapshot(repeated, wideHead); const comparison = compareSnapshots(preflight, repeated); - writeJson(path.join(evidence, 'forced-repeat-comparison.json'), comparison); + writeJsonArtifact(evidence, 'forced-repeat-comparison', comparison); if (!comparison.equal) { throw new Error('consecutive forced rebuilds produced different graph fingerprints'); } - writeJson(path.join(evidence, 'forced-repeat-summary.json'), { + writeJsonArtifact(evidence, 'forced-repeat-summary', { runId, result: 'passed', sourceSha, @@ -683,8 +702,7 @@ try { }, ); - const fixtureRoot = path.join(worktree, 'src/r19-canary'); - fs.mkdirSync(fixtureRoot, { recursive: true }); + const fixtureRoot = createSafeContainedDirectory(worktree, 'src/r19-canary'); fs.writeFileSync( path.join(fixtureRoot, 'hub.ts'), 'export function r19CanaryHub(value: number): number {\n return value + 19;\n}\n', @@ -756,12 +774,10 @@ try { }, ); - const tracked = (await runText('tracked-typescript', 'git', ['ls-files', '*.ts'])) - .split(/\r?\n/) - .filter((file) => file && !file.startsWith('src/r19-canary/')) - .map((file) => ({ file, bytes: fs.statSync(path.join(worktree, file)).size })) - .sort((left, right) => left.bytes - right.bytes || left.file.localeCompare(right.file)) - .slice(0, 51); + const trackedFiles = (await runText('tracked-typescript', 'git', ['ls-files', '-z', '*.ts'])) + .split('\0') + .filter((file) => file && !file.startsWith('src/r19-canary/')); + const tracked = selectSafeTrackedFiles(worktree, trackedFiles, 51); if (tracked.length !== 51) throw new Error(`expected 51 TypeScript files, found ${tracked.length}`); tracked.forEach(({ file }, index) => { @@ -772,8 +788,11 @@ try { wideHead = await runText('wide-head', 'git', ['rev-parse', 'HEAD']); } - const interrupted = await interruptAtEscalation(); - if (interrupted.dirty?.dirty?.phase !== 'effective-write-set') { + const interrupted = await interruptAtRecoveryBoundary('during-delete'); + if (interrupted.dirty?.boundary !== 'during-delete') { + throw new Error(`unexpected recovery boundary: ${JSON.stringify(interrupted.dirty)}`); + } + if (interrupted.dirty?.dirty?.phase !== 'escalated-full-write') { throw new Error(`unexpected dirty phase: ${JSON.stringify(interrupted.dirty)}`); } if (Number(interrupted.dirty?.dirty?.effectiveWriteCount ?? 0) < 50) { @@ -802,7 +821,7 @@ try { const forced = await snapshot('forced', { fingerprint: true }); assertSnapshot(forced, wideHead); const recoveredForcedComparison = compareSnapshots(recovered, forced); - writeJson(path.join(evidence, 'recovered-forced-comparison.json'), recoveredForcedComparison); + writeJsonArtifact(evidence, 'recovered-forced-comparison', recoveredForcedComparison); if (!recoveredForcedComparison.equal) { throw new Error('recovered and forced graphs produced different fingerprints'); } @@ -849,7 +868,8 @@ try { process.stderr.write( `${error instanceof Error ? (error.stack ?? error.message) : String(error)}\n`, ); - process.exitCode = 1; + if (!process.exitCode) process.exitCode = 1; } finally { + childSupervisor.disposeSignalHandlers(); await new Promise((resolve) => embeddingServer.close(resolve)); } diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 573154812b..4256aececd 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -934,11 +934,16 @@ const copyCsvWithRetry = async ( * keeps the IGNORE_ERRORS=true retry; a hard failure throws (no node rows ⇒ the * relationship COPY would dangle on missing endpoints). */ +export interface LbugLoadHooks { + onNodeCopyCommitted?: (table: NodeTableName, index: number, total: number) => void; +} + const copyNodeCSVs = async ( targetConn: lbug.Connection, nodeFileEntries: [NodeTableName, { csvPath: string; rows: number }][], log: (message: string) => void, totalSteps: number, + hooks?: LbugLoadHooks, ): Promise => { let stepsDone = 0; for (const [table, { csvPath, rows }] of nodeFileEntries) { @@ -950,6 +955,7 @@ const copyNodeCSVs = async ( const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr); throw new Error(`COPY failed for ${table}: ${retryMsg.slice(0, 200)}`); }); + hooks?.onNodeCopyCommitted?.(table, stepsDone - 1, totalSteps); } }; @@ -979,6 +985,7 @@ export const loadGraphToLbug = async ( * emits none — the manifest is the sole source and there is no double-COPY. */ pdgEmitManifest?: PdgEmitManifest, + hooks?: LbugLoadHooks, ) => { if (!conn) { throw new Error('LadybugDB not initialized. Call initLbug first.'); @@ -1050,7 +1057,7 @@ export const loadGraphToLbug = async ( // denominator is the node-table count — not +1 reserving a rel step. // .catch captures the failure so an overlapped (mid-emit) rejection cannot // surface as an unhandled rejection; it is rethrown at the FK barrier below. - nodeCopyPromise = copyNodeCSVs(writeConn, entries, log, entries.length).catch((e) => { + nodeCopyPromise = copyNodeCSVs(writeConn, entries, log, entries.length, hooks).catch((e) => { nodeCopyError = e; }); }; @@ -2085,14 +2092,25 @@ export class LbugWipeError extends Error { * DB the run is about to recreate, so neither needs (or may share) the * loud-failure contract here. */ -export const wipeLbugDbFiles = async (lbugPath: string): Promise => { +export interface LbugWipeOptions { + onRemoved?: (removedPath: string, index: number, total: number) => void; +} + +export const wipeLbugDbFiles = async ( + lbugPath: string, + options: LbugWipeOptions = {}, +): Promise => { const lockPath = `${lbugPath}.lock`; const family = [lbugPath, `${lbugPath}.wal`, `${lbugPath}.shadow`, lockPath]; let survivors: string[] = []; for (let attempt = 1; attempt <= HANDLE_RELEASE_PROBE_ATTEMPTS; attempt++) { survivors = []; - for (const f of family) { + for (const [index, f] of family.entries()) { + const existedBefore = await fs.access(f).then( + () => true, + () => false, + ); try { await fs.rm(f, { recursive: true, force: true }); } catch { @@ -2104,7 +2122,11 @@ export const wipeLbugDbFiles = async (lbugPath: string): Promise => { () => false, // still present (err: unknown) => (err as NodeJS.ErrnoException | null)?.code === 'ENOENT', ); - if (!gone) survivors.push(f); + if (!gone) { + survivors.push(f); + } else if (existedBefore) { + options.onRemoved?.(f, index, family.length); + } } if (survivors.length === 0) return; if (attempt < HANDLE_RELEASE_PROBE_ATTEMPTS) { diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index f8622a4bc1..3ee3ae03c6 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -126,6 +126,11 @@ import { STALE_HASH_SENTINEL } from './lbug/schema.js'; export interface AnalyzeCallbacks { onProgress: (phase: string, percent: number, message: string) => void; onLog?: (message: string) => void; + /** Test/recovery observability at durable incremental-write boundaries. */ + onRecoveryBoundary?: ( + boundary: 'before-delete' | 'during-delete' | 'during-insert' | 'before-finalize', + details: Readonly>, + ) => void; } export interface AnalyzeOptions { @@ -1623,6 +1628,11 @@ export async function runFullAnalysis( effectiveWriteCount: effectiveWriteSet.size, deleteCount: filesToDelete.length, }); + callbacks.onRecoveryBoundary?.('before-delete', { + phase: 'escalated-full-write', + effectiveWriteCount: effectiveWriteSet.size, + deleteCount: filesToDelete.length, + }); // Strategy switch: stop the checkpoint driver around the close so its // in-flight CHECKPOINT can't race the reopen, drop the DB files // (sidecars included), and bulk-load the full graph into a fresh DB — @@ -1633,14 +1643,52 @@ export async function runFullAnalysis( // to replace wholesale. await walCheckpointDriver.stop(); await closeLbug(); - await wipeLbugDbFiles(lbugPath); + let deletionBoundaryReported = false; + await wipeLbugDbFiles(lbugPath, { + onRemoved: (removedPath, index, total) => { + if (deletionBoundaryReported) return; + deletionBoundaryReported = true; + callbacks.onRecoveryBoundary?.('during-delete', { + phase: 'escalated-full-write', + removedPath, + index, + total, + }); + }, + }); + await saveIncrementalDirtyState('escalated-load-graph', { + toWriteCount: 0, + importerExpansion, + shadowSeedCount: shadowSeed.length, + effectiveWriteCount: effectiveWriteSet.size, + deleteCount: filesToDelete.length, + }); await initLbug(lbugPath); walCheckpointDriver = startWalCheckpointDriver(); - await loadGraphToLbug(pipelineResult.graph, pipelineResult.repoPath, storagePath, (msg) => { - lbugMsgCount++; - const pct = Math.min(84, 65 + Math.round((lbugMsgCount / (lbugMsgCount + 10)) * 19)); - progress('lbug', pct, msg); - }); + let insertionBoundaryReported = false; + await loadGraphToLbug( + pipelineResult.graph, + pipelineResult.repoPath, + storagePath, + (msg) => { + lbugMsgCount++; + const pct = Math.min(84, 65 + Math.round((lbugMsgCount / (lbugMsgCount + 10)) * 19)); + progress('lbug', pct, msg); + }, + undefined, + { + onNodeCopyCommitted: (table, index, total) => { + if (insertionBoundaryReported) return; + insertionBoundaryReported = true; + callbacks.onRecoveryBoundary?.('during-insert', { + phase: 'escalated-load-graph', + table, + index, + total, + }); + }, + }, + ); } else { // The surgical plan is now final. Only at this point may // --incremental-only write its crash marker, immediately before the @@ -2215,6 +2263,12 @@ export async function runFullAnalysis( // off==off and incremental eligibility is restored. pdg: resolvePdgConfig(options), }; + if (isIncremental && hashDiff) { + callbacks.onRecoveryBoundary?.('before-finalize', { + phase: escalatedFullWrite ? 'escalated-load-graph' : 'load-graph', + targetCommit: currentCommit, + }); + } await saveMeta(metaDir, meta); // Persist the incremental parse cache for the next run. Wraps in diff --git a/gitnexus/test/fixtures/large-incremental-child.mjs b/gitnexus/test/fixtures/large-incremental-child.mjs index 7075b770c8..8aee3a34a7 100644 --- a/gitnexus/test/fixtures/large-incremental-child.mjs +++ b/gitnexus/test/fixtures/large-incremental-child.mjs @@ -12,16 +12,18 @@ const bm25IndexUrl = pathToFileURL(path.join(packageRoot, 'dist/core/search/bm25 const [repoPath, ...args] = process.argv.slice(2); if (!repoPath) { throw new Error( - 'usage: large-incremental-child.mjs [--force] [--embeddings] [--pause-on-escalation ] [--fts-query ]', + 'usage: large-incremental-child.mjs [--force] [--embeddings] [--pause-at --pause-ready ] [--fts-query ]', ); } const force = args.includes('--force'); const embeddings = args.includes('--embeddings'); -const pauseIndex = args.indexOf('--pause-on-escalation'); -const pauseReadyFile = pauseIndex >= 0 ? args[pauseIndex + 1] : undefined; -if (pauseIndex >= 0 && !pauseReadyFile) { - throw new Error('--pause-on-escalation requires a ready-file path'); +const pauseAtIndex = args.indexOf('--pause-at'); +const pauseAt = pauseAtIndex >= 0 ? args[pauseAtIndex + 1] : undefined; +const pauseReadyIndex = args.indexOf('--pause-ready'); +const pauseReadyFile = pauseReadyIndex >= 0 ? args[pauseReadyIndex + 1] : undefined; +if ((pauseAt && !pauseReadyFile) || (!pauseAt && pauseReadyFile)) { + throw new Error('--pause-at and --pause-ready must be provided together'); } const ftsQueryIndex = args.indexOf('--fts-query'); const ftsQuery = ftsQueryIndex >= 0 ? args[ftsQueryIndex + 1] : undefined; @@ -37,20 +39,22 @@ let paused = false; const onLog = (message) => { logs.push(message); - if (!paused && pauseReadyFile && message.includes('switching to a full DB write')) { - paused = true; - const metadataPath = path.join(storagePath, 'gitnexus.json'); - const metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf8')); - fs.writeFileSync( - pauseReadyFile, - JSON.stringify({ message, dirty: metadata.incrementalInProgress }), - 'utf8', - ); +}; - const latch = new Int32Array(new SharedArrayBuffer(4)); - Atomics.wait(latch, 0, 0, 120_000); - throw new Error('pause-on-escalation timed out before the parent terminated this process'); - } +const onRecoveryBoundary = (boundary, details) => { + if (paused || !pauseReadyFile || boundary !== pauseAt) return; + paused = true; + const metadataPath = path.join(storagePath, 'gitnexus.json'); + const metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf8')); + fs.writeFileSync( + pauseReadyFile, + JSON.stringify({ boundary, details, dirty: metadata.incrementalInProgress }), + { encoding: 'utf8', flag: 'wx', mode: 0o600 }, + ); + + const latch = new Int32Array(new SharedArrayBuffer(4)); + Atomics.wait(latch, 0, 0, 120_000); + throw new Error(`pause-at ${boundary} timed out before the parent terminated this process`); }; try { @@ -63,7 +67,7 @@ try { embeddings, embeddingsNodeLimit: embeddings ? 0 : undefined, }, - { onProgress: () => {}, onLog }, + { onProgress: () => {}, onLog, onRecoveryBoundary }, ); const meta = await loadMeta(storagePath); diff --git a/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts b/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts index 4777ac11df..00478b7fa4 100644 --- a/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts +++ b/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts @@ -206,53 +206,77 @@ describe('large incremental analysis subprocess contract', () => { ); commitAll(fixture.repoPath, 'exercise incremental write set'); - const readyFile = path.join(fixture.root, 'pause-ready.json'); - const interrupted = spawn( - process.execPath, - [childRunner, fixture.repoPath, '--pause-on-escalation', readyFile], - { - env: childEnv(fixture.gitnexusHome), - stdio: ['ignore', 'pipe', 'pipe'], - }, - ); - await waitForFile(readyFile, interrupted); - const pauseState = JSON.parse(fs.readFileSync(readyFile, 'utf8')) as { - message: string; - dirty: { phase?: string; effectiveWriteCount?: number; deleteCount?: number }; - }; - expect(pauseState.message).toContain('switching to a full DB write'); - expect(pauseState.dirty.phase).toBe('effective-write-set'); - expect(pauseState.dirty.effectiveWriteCount).toBeGreaterThanOrEqual(50); - expect(pauseState.dirty.deleteCount).toBeGreaterThanOrEqual(50); + const boundaries = [ + ['before-delete', 'escalated-full-write'], + ['during-delete', 'escalated-full-write'], + ['during-insert', 'escalated-load-graph'], + ['before-finalize', 'escalated-load-graph'], + ] as const; + let recovered: HarnessResult | undefined; + for (const [index, [boundary, expectedDirtyPhase]] of boundaries.entries()) { + if (index > 0) { + fs.appendFileSync(path.join(src, 'hub.ts'), `\n// ${boundary} interruption\n`); + commitAll(fixture.repoPath, `prepare ${boundary} interruption`); + } + const readyFile = path.join(fixture.root, `pause-ready-${boundary}.json`); + const interrupted = spawn( + process.execPath, + [childRunner, fixture.repoPath, '--pause-at', boundary, '--pause-ready', readyFile], + { + env: childEnv(fixture.gitnexusHome), + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + await waitForFile(readyFile, interrupted); + const pauseState = JSON.parse(fs.readFileSync(readyFile, 'utf8')) as { + boundary: string; + details: { phase?: string; removedPath?: string; table?: string }; + dirty: { phase?: string; effectiveWriteCount?: number; deleteCount?: number }; + }; + expect(pauseState.boundary).toBe(boundary); + expect(pauseState.details.phase).toBe(expectedDirtyPhase); + expect(pauseState.dirty.phase).toBe(expectedDirtyPhase); + expect(pauseState.dirty.effectiveWriteCount).toBeGreaterThanOrEqual(50); + expect(pauseState.dirty.deleteCount).toBeGreaterThanOrEqual(50); + if (boundary === 'during-delete') { + expect(pauseState.details.removedPath).toMatch(/lbug$/); + } + if (boundary === 'during-insert') { + expect(pauseState.details.table).toBeTruthy(); + } - const stopped = await stopChild(interrupted); - expect(stopped.code).not.toBe(0); - expect(stopped.signal).toBe('SIGTERM'); - const dirtyMeta = await loadMeta(storagePath); - expect(dirtyMeta?.incrementalInProgress).toBeDefined(); + const stopped = await stopChild(interrupted); + expect(stopped.code).not.toBe(0); + expect(stopped.signal).toBe('SIGTERM'); + const dirtyMeta = await loadMeta(storagePath); + expect(dirtyMeta?.incrementalInProgress?.phase).toBe(expectedDirtyPhase); - const recovered = runChild(fixture.repoPath, fixture.gitnexusHome, [ - '--fts-query', - 'r09FtsNeedle', - ]); - expect(recovered.logs.join('\n')).toContain( - 'Previous analyze run did not complete cleanly (incrementalInProgress flag set)', - ); + recovered = runChild(fixture.repoPath, fixture.gitnexusHome, [ + '--fts-query', + 'r09FtsNeedle', + ]); + expect(recovered.logs.join('\n')).toContain( + 'Previous analyze run did not complete cleanly (incrementalInProgress flag set)', + ); + expect((await loadMeta(storagePath))?.incrementalInProgress).toBeUndefined(); + } + if (!recovered) throw new Error('recovery matrix produced no successful run'); + const finalRecovered = recovered; const recoveredMeta = await loadMeta(storagePath); expect(recoveredMeta?.incrementalInProgress).toBeUndefined(); - expect(recovered.stats.files).toBe(61); - expect(recovered.stats.nodes).toBeGreaterThan(61); - expect(recovered.capabilities).toHaveProperty('fts'); - expect(recovered.capabilities).toHaveProperty('vectorSearch'); - expect(Array.isArray(recovered.sidecars)).toBe(true); - expect(recovered.ftsSearch?.ftsAvailable).toBe(true); - expect(recovered.ftsSearch?.results.map((result) => result.filePath)).toContain( + expect(finalRecovered.stats.files).toBe(61); + expect(finalRecovered.stats.nodes).toBeGreaterThan(61); + expect(finalRecovered.capabilities).toHaveProperty('fts'); + expect(finalRecovered.capabilities).toHaveProperty('vectorSearch'); + expect(Array.isArray(finalRecovered.sidecars)).toBe(true); + expect(finalRecovered.ftsSearch?.ftsAvailable).toBe(true); + expect(finalRecovered.ftsSearch?.results.map((result) => result.filePath)).toContain( 'src/added.ts', ); const survivingEmbeddingIds = await readEmbeddingNodeIds(fixture.repoPath); expect(survivingEmbeddingIds.length).toBeGreaterThanOrEqual(4); - expect(recovered.stats.embeddings).toBe(survivingEmbeddingIds.length); + expect(finalRecovered.stats.embeddings).toBe(survivingEmbeddingIds.length); expect(survivingEmbeddingIds).not.toContain(seeded.get('src/spoke-001.ts')?.[0]); expect(survivingEmbeddingIds).not.toContain(seeded.get('src/spoke-002.ts')?.[0]); expect(await readEmbeddingDimensions(fixture.repoPath)).toEqual([384]); @@ -262,11 +286,11 @@ describe('large incremental analysis subprocess contract', () => { '--fts-query', 'r09FtsNeedle', ]); - expect(forced.stats).toEqual(recovered.stats); - expect(forced.ftsSearch).toEqual(recovered.ftsSearch); + expect(forced.stats).toEqual(finalRecovered.stats); + expect(forced.ftsSearch).toEqual(finalRecovered.ftsSearch); expect(await readEmbeddingNodeIds(fixture.repoPath)).toEqual(survivingEmbeddingIds); } finally { fs.rmSync(fixture.root, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } - }, 600_000); + }, 1_800_000); }); diff --git a/gitnexus/test/unit/incremental-orchestration.test.ts b/gitnexus/test/unit/incremental-orchestration.test.ts index 78d1ed549e..33f3372f76 100644 --- a/gitnexus/test/unit/incremental-orchestration.test.ts +++ b/gitnexus/test/unit/incremental-orchestration.test.ts @@ -319,16 +319,34 @@ describe('runFullAnalysis — incremental orchestration', () => { await writeFile(hub, (await readFile(hub, 'utf-8')) + '// escalation touch\n', 'utf-8'); const logs: string[] = []; + const recoveryBoundaries: Array<{ + boundary: string; + details: Readonly>; + }> = []; const incremental = await runFullAnalysis( repo.dbPath, { skipAgentsMd: true }, - { onProgress: () => {}, onLog: (m) => logs.push(m) }, + { + onProgress: () => {}, + onLog: (m) => logs.push(m), + onRecoveryBoundary: (boundary, details) => recoveryBoundaries.push({ boundary, details }), + }, ); expect(incremental.alreadyUpToDate).toBeUndefined(); const joined = logs.join('\n'); // The importer expansion fired AND the valve rerouted the write plan. expect(joined).toContain('importer(s) added to writable set'); expect(joined).toContain('switching to a full DB write'); + expect(recoveryBoundaries.map(({ boundary }) => boundary)).toEqual([ + 'before-delete', + 'during-delete', + 'during-insert', + 'before-finalize', + ]); + expect(recoveryBoundaries[0]?.details.phase).toBe('escalated-full-write'); + expect(recoveryBoundaries[1]?.details.phase).toBe('escalated-full-write'); + expect(recoveryBoundaries[2]?.details.phase).toBe('escalated-load-graph'); + expect(recoveryBoundaries[3]?.details.phase).toBe('escalated-load-graph'); const { storagePath } = getStoragePaths(repo.dbPath); const escalatedMeta = await loadMeta(storagePath); diff --git a/gitnexus/test/unit/reconciliation-canary-safety.test.ts b/gitnexus/test/unit/reconciliation-canary-safety.test.ts new file mode 100644 index 0000000000..8c3ed22980 --- /dev/null +++ b/gitnexus/test/unit/reconciliation-canary-safety.test.ts @@ -0,0 +1,223 @@ +import { execFileSync, spawn, spawnSync } from 'node:child_process'; +import { once } from 'node:events'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { + createSafeContainedDirectory, + openArtifactLogs, + selectSafeTrackedFiles, + writeJsonArtifact, +} from '../../scripts/reconciliation/canary-safety.mjs'; + +const temporaryRoots: string[] = []; + +const makeTemporaryRoot = (): string => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-canary-safety-')); + temporaryRoots.push(root); + return root; +}; + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) { + fs.rmSync(root, { force: true, recursive: true }); + } +}); + +describe('reconciliation canary filesystem safety', () => { + it.skipIf(process.platform === 'win32')( + 'refuses a preexisting evidence symlink without truncating its target', + () => { + const root = makeTemporaryRoot(); + const source = path.join(root, 'source'); + const evidence = path.join(root, 'evidence'); + const commands = path.join(evidence, 'commands'); + const extensionSource = path.join(root, 'extension'); + const fakeCli = path.join(root, 'fake-cli.mjs'); + const victim = path.join(root, 'victim.txt'); + const worktree = path.join(root, 'worktree'); + + fs.mkdirSync(source); + fs.mkdirSync(commands, { recursive: true }); + fs.mkdirSync(extensionSource); + fs.writeFileSync(path.join(source, 'README.md'), 'fixture\n'); + fs.writeFileSync(path.join(extensionSource, 'extension.bin'), 'fixture\n'); + fs.writeFileSync(fakeCli, 'process.exit(1);\n'); + fs.writeFileSync(victim, 'preserve-me\n'); + fs.symlinkSync(victim, path.join(commands, 'clone.stdout.log')); + + execFileSync('git', ['init', '-q'], { cwd: source }); + execFileSync('git', ['add', 'README.md'], { cwd: source }); + execFileSync( + 'git', + [ + '-c', + 'user.name=Canary Test', + '-c', + 'user.email=canary-test@example.invalid', + 'commit', + '-q', + '-m', + 'fixture', + ], + { cwd: source }, + ); + const sourceSha = execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: source, + encoding: 'utf8', + }).trim(); + + const result = spawnSync( + process.execPath, + [ + 'scripts/reconciliation/large-repository-canary.mjs', + '--source', + source, + '--source-sha', + sourceSha, + '--public-origin', + 'https://example.invalid/repository.git', + '--worktree', + worktree, + '--evidence', + evidence, + '--gitnexus-cli', + fakeCli, + '--incremental-child', + fakeCli, + '--extension-source', + extensionSource, + '--run-id', + 'symlink-safety-test', + ], + { cwd: path.resolve(import.meta.dirname, '../..'), encoding: 'utf8' }, + ); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain('symbolic link'); + expect(fs.readFileSync(victim, 'utf8')).toBe('preserve-me\n'); + }, + ); + + it.skipIf(process.platform === 'win32')( + 'refuses a symlinked fixture-directory component outside the worktree', + () => { + const root = makeTemporaryRoot(); + const worktree = path.join(root, 'worktree'); + const victimDirectory = path.join(root, 'victim'); + fs.mkdirSync(worktree); + fs.mkdirSync(victimDirectory); + fs.symlinkSync(victimDirectory, path.join(worktree, 'src')); + + expect(() => createSafeContainedDirectory(worktree, 'src/r19-canary')).toThrow( + 'symbolic link', + ); + expect(fs.readdirSync(victimDirectory)).toEqual([]); + }, + ); + + it('preserves prior command logs and snapshots across retry attempts', () => { + const root = makeTemporaryRoot(); + const commands = path.join(root, 'commands'); + const snapshots = path.join(root, 'snapshots'); + fs.mkdirSync(commands); + fs.mkdirSync(snapshots); + fs.writeFileSync(path.join(commands, 'clone.stdout.log'), 'first stdout\n'); + fs.writeFileSync(path.join(commands, 'clone.stderr.log'), 'first stderr\n'); + fs.writeFileSync(path.join(snapshots, 'initial.json'), '{"attempt":1}\n'); + + const logs = openArtifactLogs(commands, 'clone'); + fs.writeSync(logs.stdout, 'second stdout\n'); + fs.writeSync(logs.stderr, 'second stderr\n'); + fs.closeSync(logs.stdout); + fs.closeSync(logs.stderr); + const retrySnapshot = writeJsonArtifact(snapshots, 'initial', { attempt: 2 }); + + expect(path.basename(logs.stdoutPath)).toBe('clone-attempt-2.stdout.log'); + expect(path.basename(logs.stderrPath)).toBe('clone-attempt-2.stderr.log'); + expect(path.basename(retrySnapshot)).toBe('initial-attempt-2.json'); + expect(fs.readFileSync(path.join(commands, 'clone.stdout.log'), 'utf8')).toBe('first stdout\n'); + expect(fs.readFileSync(path.join(commands, 'clone.stderr.log'), 'utf8')).toBe('first stderr\n'); + expect(fs.readFileSync(path.join(snapshots, 'initial.json'), 'utf8')).toBe('{"attempt":1}\n'); + }); + + it.skipIf(process.platform === 'win32')( + 'rejects a tracked TypeScript symlink without modifying its target', + () => { + const root = makeTemporaryRoot(); + const worktree = path.join(root, 'worktree'); + const victim = path.join(root, 'victim.ts'); + fs.mkdirSync(worktree); + fs.writeFileSync(victim, 'export const preserve = true;\n'); + fs.symlinkSync(victim, path.join(worktree, 'linked.ts')); + + expect(() => selectSafeTrackedFiles(worktree, ['linked.ts'], 1)).toThrow('symbolic link'); + expect(fs.readFileSync(victim, 'utf8')).toBe('export const preserve = true;\n'); + }, + ); + + it.skipIf(process.platform === 'win32')( + 'forwards an orchestrator signal to its owned child process tree', + async () => { + const root = makeTemporaryRoot(); + const pidFile = path.join(root, 'pids.json'); + const childScript = path.join(root, 'child.mjs'); + const wrapperScript = path.join(root, 'wrapper.mjs'); + const safetyModule = path.resolve( + import.meta.dirname, + '../../scripts/reconciliation/canary-safety.mjs', + ); + fs.writeFileSync( + childScript, + `import { spawn } from 'node:child_process';\n` + + `import fs from 'node:fs';\n` + + `const grandchild = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)']);\n` + + `fs.writeFileSync(${JSON.stringify(pidFile)}, JSON.stringify({ child: process.pid, grandchild: grandchild.pid }));\n` + + `setInterval(() => {}, 1000);\n`, + ); + fs.writeFileSync( + wrapperScript, + `import { once } from 'node:events';\n` + + `import { ChildSupervisor } from ${JSON.stringify(pathToFileURL(safetyModule).href)};\n` + + `const supervisor = new ChildSupervisor({ killTimeoutMs: 200 });\n` + + `supervisor.installSignalHandlers();\n` + + `const child = supervisor.spawn(process.execPath, [${JSON.stringify(childScript)}], { stdio: 'ignore' });\n` + + `await once(child, 'close');\n` + + `supervisor.disposeSignalHandlers();\n`, + ); + + const wrapper = spawn(process.execPath, [wrapperScript], { stdio: 'ignore' }); + const deadline = Date.now() + 5_000; + while (!fs.existsSync(pidFile) && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + expect(fs.existsSync(pidFile)).toBe(true); + const pids = JSON.parse(fs.readFileSync(pidFile, 'utf8')) as { + child: number; + grandchild: number; + }; + + wrapper.kill('SIGTERM'); + await once(wrapper, 'close'); + + const isAlive = (pid: number): boolean => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }; + const exitDeadline = Date.now() + 5_000; + while ((isAlive(pids.child) || isAlive(pids.grandchild)) && Date.now() < exitDeadline) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + expect(isAlive(pids.child)).toBe(false); + expect(isAlive(pids.grandchild)).toBe(false); + }, + ); +}); From f996ac20ea03e02a081bccb88c5d13cc53955c6e Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 04:50:53 +0700 Subject: [PATCH 08/16] test(reconciliation): close canary safety gaps --- .../scripts/reconciliation/canary-safety.mjs | 72 +++++++++++--- .../large-repository-canary.mjs | 2 +- .../unit/reconciliation-canary-safety.test.ts | 95 ++++++++++++++++++- 3 files changed, 151 insertions(+), 18 deletions(-) diff --git a/gitnexus/scripts/reconciliation/canary-safety.mjs b/gitnexus/scripts/reconciliation/canary-safety.mjs index a82fb9d6ed..6c4f758fb7 100644 --- a/gitnexus/scripts/reconciliation/canary-safety.mjs +++ b/gitnexus/scripts/reconciliation/canary-safety.mjs @@ -35,9 +35,7 @@ export const prepareEvidenceLayout = ({ evidence, commandDir, snapshotDir, home fs.mkdirSync(directory, { recursive: true }); assertDirectory(directory); } - assertTreeHasNoSymlinks(commandDir); - assertTreeHasNoSymlinks(snapshotDir); - assertTreeHasNoSymlinks(home); + assertTreeHasNoSymlinks(evidence); }; export const writeJsonAtomic = (filePath, value) => { @@ -181,9 +179,23 @@ export const selectSafeTrackedFiles = (worktree, files, count) => { }; export class ChildSupervisor { - constructor({ platform = process.platform, killTimeoutMs = 5_000 } = {}) { + constructor({ + platform = process.platform, + killTimeoutMs = 5_000, + terminationEnv, + spawnSyncImpl = spawnSync, + } = {}) { this.platform = platform; this.killTimeoutMs = killTimeoutMs; + this.terminationEnv = terminationEnv ?? { + SystemRoot: process.env.SystemRoot, + WINDIR: process.env.WINDIR, + COMSPEC: process.env.COMSPEC, + PATHEXT: process.env.PATHEXT, + TEMP: process.env.TEMP, + TMP: process.env.TMP, + }; + this.spawnSyncImpl = spawnSyncImpl; this.children = new Set(); this.receivedSignal = undefined; this.signalHandlers = new Map(); @@ -204,18 +216,49 @@ export class ChildSupervisor { terminate(child, signal = 'SIGTERM') { if (!child?.pid || child.exitCode !== null || child.signalCode !== null) return; + this.terminateProcessGroup(child.pid, signal, child); + } + + terminateProcessGroup(pid, signal, directChild) { if (this.platform === 'win32') { - const result = spawnSync('taskkill', ['/pid', String(child.pid), '/t', '/f'], { - stdio: 'ignore', - windowsHide: true, - }); - if (result.error || result.status !== 0) child.kill(signal); + const systemRoot = this.terminationEnv.SystemRoot ?? this.terminationEnv.WINDIR; + let result; + if (systemRoot) { + try { + result = this.spawnSyncImpl( + path.win32.join(systemRoot, 'System32', 'taskkill.exe'), + ['/pid', String(pid), '/t', '/f'], + { + env: this.terminationEnv, + stdio: 'ignore', + windowsHide: true, + }, + ); + } catch (error) { + result = { error }; + } + } + if ( + (!result || result.error || result.status !== 0) && + directChild && + directChild.exitCode === null && + directChild.signalCode === null + ) { + directChild.kill(signal); + } return; } try { - process.kill(-child.pid, signal); + process.kill(-pid, signal); } catch (error) { - if (error?.code !== 'ESRCH') child.kill(signal); + if ( + error?.code !== 'ESRCH' && + directChild && + directChild.exitCode === null && + directChild.signalCode === null + ) { + directChild.kill(signal); + } } } @@ -223,10 +266,11 @@ export class ChildSupervisor { const children = [...this.children]; for (const child of children) this.terminate(child, signal); if (children.length > 0) { - const timer = setTimeout(() => { - for (const child of this.children) this.terminate(child, 'SIGKILL'); + setTimeout(() => { + for (const child of children) { + if (child.pid) this.terminateProcessGroup(child.pid, 'SIGKILL'); + } }, this.killTimeoutMs); - timer.unref(); } } diff --git a/gitnexus/scripts/reconciliation/large-repository-canary.mjs b/gitnexus/scripts/reconciliation/large-repository-canary.mjs index 7995334e98..076eaa7e9c 100644 --- a/gitnexus/scripts/reconciliation/large-repository-canary.mjs +++ b/gitnexus/scripts/reconciliation/large-repository-canary.mjs @@ -125,7 +125,7 @@ const commandLedger = ? JSON.parse(fs.readFileSync(commandLedgerPath, 'utf8')) : []; const embeddingStats = { requests: 0, items: 0 }; -const childSupervisor = new ChildSupervisor(); +const childSupervisor = new ChildSupervisor({ terminationEnv: baseEnv }); const run = async (name, command, commandArgs, options = {}) => { const { stdoutPath, stderrPath, stdout, stderr } = openArtifactLogs(commandDir, name); diff --git a/gitnexus/test/unit/reconciliation-canary-safety.test.ts b/gitnexus/test/unit/reconciliation-canary-safety.test.ts index 8c3ed22980..e184529500 100644 --- a/gitnexus/test/unit/reconciliation-canary-safety.test.ts +++ b/gitnexus/test/unit/reconciliation-canary-safety.test.ts @@ -8,6 +8,7 @@ import { pathToFileURL } from 'node:url'; import { afterEach, describe, expect, it } from 'vitest'; import { + ChildSupervisor, createSafeContainedDirectory, openArtifactLogs, selectSafeTrackedFiles, @@ -29,6 +30,61 @@ afterEach(() => { }); describe('reconciliation canary filesystem safety', () => { + it.skipIf(process.platform === 'win32')( + 'refuses a symlinked command ledger before reading resume evidence', + () => { + const root = makeTemporaryRoot(); + const source = path.join(root, 'source'); + const evidence = path.join(root, 'evidence'); + const extensionSource = path.join(root, 'extension'); + const fakeCli = path.join(root, 'fake-cli.mjs'); + const victim = path.join(root, 'private-ledger.json'); + const worktree = path.join(root, 'worktree'); + + fs.mkdirSync(source); + fs.mkdirSync(evidence); + fs.mkdirSync(extensionSource); + fs.mkdirSync(worktree); + fs.writeFileSync(path.join(source, 'README.md'), 'fixture\n'); + fs.writeFileSync(path.join(extensionSource, 'extension.bin'), 'fixture\n'); + fs.writeFileSync(fakeCli, 'process.exit(1);\n'); + fs.writeFileSync(victim, '[{"private":"preserve-me"}]\n'); + fs.symlinkSync(victim, path.join(evidence, 'command-ledger.json')); + + const result = spawnSync( + process.execPath, + [ + 'scripts/reconciliation/large-repository-canary.mjs', + '--source', + source, + '--source-sha', + '0123456789abcdef0123456789abcdef01234567', + '--public-origin', + 'https://example.invalid/repository.git', + '--worktree', + worktree, + '--evidence', + evidence, + '--gitnexus-cli', + fakeCli, + '--incremental-child', + fakeCli, + '--extension-source', + extensionSource, + '--run-id', + 'ledger-symlink-test', + '--resume-from', + 'wide', + ], + { cwd: path.resolve(import.meta.dirname, '../..'), encoding: 'utf8' }, + ); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain('symbolic link'); + expect(fs.readFileSync(victim, 'utf8')).toBe('[{"private":"preserve-me"}]\n'); + }, + ); + it.skipIf(process.platform === 'win32')( 'refuses a preexisting evidence symlink without truncating its target', () => { @@ -161,7 +217,7 @@ describe('reconciliation canary filesystem safety', () => { ); it.skipIf(process.platform === 'win32')( - 'forwards an orchestrator signal to its owned child process tree', + 'force-kills a signal-resistant grandchild after its direct child exits', async () => { const root = makeTemporaryRoot(); const pidFile = path.join(root, 'pids.json'); @@ -175,8 +231,8 @@ describe('reconciliation canary filesystem safety', () => { childScript, `import { spawn } from 'node:child_process';\n` + `import fs from 'node:fs';\n` + - `const grandchild = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)']);\n` + - `fs.writeFileSync(${JSON.stringify(pidFile)}, JSON.stringify({ child: process.pid, grandchild: grandchild.pid }));\n` + + `const grandchild = spawn(process.execPath, ['-e', 'process.on("SIGTERM", () => {}); process.stdout.write("ready\\\\n"); setInterval(() => {}, 1000)'], { stdio: ['ignore', 'pipe', 'ignore'] });\n` + + `grandchild.stdout.once('data', () => fs.writeFileSync(${JSON.stringify(pidFile)}, JSON.stringify({ child: process.pid, grandchild: grandchild.pid })));\n` + `setInterval(() => {}, 1000);\n`, ); fs.writeFileSync( @@ -220,4 +276,37 @@ describe('reconciliation canary filesystem safety', () => { expect(isAlive(pids.grandchild)).toBe(false); }, ); + + it('uses an absolute taskkill path and sanitized environment on Windows', () => { + const calls: Array<{ command: string; options: { env?: NodeJS.ProcessEnv } }> = []; + const terminationEnv = { + SystemRoot: 'C:\\Windows', + WINDIR: 'C:\\Windows', + PATH: 'C:\\Windows\\System32', + HOME: 'C:\\isolated', + }; + const supervisor = new ChildSupervisor({ + platform: 'win32', + terminationEnv, + spawnSyncImpl: (command: string, _args: string[], options: { env?: NodeJS.ProcessEnv }) => { + calls.push({ command, options }); + return { status: 0 }; + }, + }); + const child = { + pid: 1234, + exitCode: null, + signalCode: null, + kill: () => { + throw new Error('taskkill fallback should not run'); + }, + }; + + supervisor.terminate(child); + + expect(calls).toHaveLength(1); + expect(calls[0]?.command).toBe('C:\\Windows\\System32\\taskkill.exe'); + expect(calls[0]?.options.env).toEqual(terminationEnv); + expect(calls[0]?.options.env).not.toHaveProperty('GH_TOKEN'); + }); }); From fed1f3c96f2146f232f134d9936a3c4c55a8e6d9 Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 04:51:01 +0700 Subject: [PATCH 09/16] test(incremental): compare every recovery boundary --- .../large-incremental-subprocess-e2e.test.ts | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts b/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts index 00478b7fa4..36ec47cd72 100644 --- a/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts +++ b/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts @@ -259,6 +259,16 @@ describe('large incremental analysis subprocess contract', () => { 'Previous analyze run did not complete cleanly (incrementalInProgress flag set)', ); expect((await loadMeta(storagePath))?.incrementalInProgress).toBeUndefined(); + + const recoveredEmbeddingIds = await readEmbeddingNodeIds(fixture.repoPath); + const forced = runChild(fixture.repoPath, fixture.gitnexusHome, [ + '--force', + '--fts-query', + 'r09FtsNeedle', + ]); + expect(forced.stats).toEqual(recovered.stats); + expect(forced.ftsSearch).toEqual(recovered.ftsSearch); + expect(await readEmbeddingNodeIds(fixture.repoPath)).toEqual(recoveredEmbeddingIds); } if (!recovered) throw new Error('recovery matrix produced no successful run'); const finalRecovered = recovered; @@ -280,17 +290,8 @@ describe('large incremental analysis subprocess contract', () => { expect(survivingEmbeddingIds).not.toContain(seeded.get('src/spoke-001.ts')?.[0]); expect(survivingEmbeddingIds).not.toContain(seeded.get('src/spoke-002.ts')?.[0]); expect(await readEmbeddingDimensions(fixture.repoPath)).toEqual([384]); - - const forced = runChild(fixture.repoPath, fixture.gitnexusHome, [ - '--force', - '--fts-query', - 'r09FtsNeedle', - ]); - expect(forced.stats).toEqual(finalRecovered.stats); - expect(forced.ftsSearch).toEqual(finalRecovered.ftsSearch); - expect(await readEmbeddingNodeIds(fixture.repoPath)).toEqual(survivingEmbeddingIds); } finally { fs.rmSync(fixture.root, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } - }, 1_800_000); + }, 2_400_000); }); From 2317a78f0a66fe2bd5c39db23d1f3bf5eea36d93 Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 05:31:41 +0700 Subject: [PATCH 10/16] test(incremental): make recovery proof hermetic --- .../deterministic-embedding-server.mjs | 66 ++++++++++ .../large-incremental-subprocess-e2e.test.ts | 123 ++++++++++++++---- 2 files changed, 166 insertions(+), 23 deletions(-) create mode 100644 gitnexus/test/fixtures/deterministic-embedding-server.mjs diff --git a/gitnexus/test/fixtures/deterministic-embedding-server.mjs b/gitnexus/test/fixtures/deterministic-embedding-server.mjs new file mode 100644 index 0000000000..9dcef79660 --- /dev/null +++ b/gitnexus/test/fixtures/deterministic-embedding-server.mjs @@ -0,0 +1,66 @@ +import crypto from 'node:crypto'; +import http from 'node:http'; + +const dimensions = 384; +const model = 'gitnexus-test-deterministic-v1'; + +const deterministicVector = (text) => { + const digest = crypto.createHash('sha256').update(text).digest(); + const vector = Array.from( + { length: dimensions }, + (_, index) => digest[index % digest.length] / 127.5 - 1, + ); + const magnitude = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0)) || 1; + return vector.map((value) => value / magnitude); +}; + +const server = http.createServer((request, response) => { + if (request.method !== 'POST' || request.url !== '/v1/embeddings') { + response.writeHead(404).end(); + return; + } + + const chunks = []; + request.on('data', (chunk) => chunks.push(chunk)); + request.on('end', () => { + try { + const body = JSON.parse(Buffer.concat(chunks).toString('utf8')); + const inputs = Array.isArray(body.input) ? body.input : [body.input]; + if (!inputs.every((input) => typeof input === 'string')) { + throw new Error('input must be a string or string array'); + } + response.writeHead(200, { 'content-type': 'application/json' }); + response.end( + JSON.stringify({ + object: 'list', + model, + data: inputs.map((input, index) => ({ + object: 'embedding', + index, + embedding: deterministicVector(input), + })), + usage: { prompt_tokens: 0, total_tokens: 0 }, + }), + ); + } catch (error) { + response.writeHead(400, { 'content-type': 'application/json' }); + response.end( + JSON.stringify({ + error: { message: error instanceof Error ? error.message : String(error) }, + }), + ); + } + }); +}); + +await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); +}); +const address = server.address(); +if (!address || typeof address === 'string') throw new Error('embedding server has no TCP address'); +process.stdout.write(`EMBEDDING_SERVER=http://127.0.0.1:${address.port}/v1\n`); + +for (const signal of ['SIGINT', 'SIGTERM']) { + process.on(signal, () => server.close(() => process.exit(0))); +} diff --git a/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts b/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts index 36ec47cd72..24c29a60e5 100644 --- a/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts +++ b/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts @@ -14,6 +14,10 @@ import { getStoragePaths, loadMeta } from '../../src/storage/repo-manager.js'; const testDir = path.dirname(fileURLToPath(import.meta.url)); const childRunner = path.resolve(testDir, '../fixtures/large-incremental-child.mjs'); +const embeddingServerRunner = path.resolve( + testDir, + '../fixtures/deterministic-embedding-server.mjs', +); type HarnessResult = { stats: Record; @@ -28,18 +32,33 @@ type HarnessResult = { }; }; -const childEnv = (gitnexusHome: string): NodeJS.ProcessEnv => ({ - ...process.env, - CI: '1', - NODE_ENV: 'test', - GITNEXUS_HOME: gitnexusHome, - // The R09 contract requires FTS but must never install during the child - // process. CI and local setup preinstall the extension; load-only fails - // closed when that prerequisite is absent. - GITNEXUS_LBUG_EXTENSION_INSTALL: 'load-only', - GITNEXUS_WORKER_POOL_SIZE: '2', - GITNEXUS_PARSE_CHUNK_CONCURRENCY: '1', -}); +const childEnv = (gitnexusHome: string, embeddingUrl: string): NodeJS.ProcessEnv => { + const env = { ...process.env }; + for (const key of Object.keys(env)) { + if (key.startsWith('GITNEXUS_EMBEDDING_')) delete env[key]; + } + return { + ...env, + CI: '1', + NODE_ENV: 'test', + GITNEXUS_HOME: gitnexusHome, + // The R09 contract requires FTS but must never install during the child + // process. CI and local setup preinstall the extension; load-only fails + // closed when that prerequisite is absent. + GITNEXUS_LBUG_EXTENSION_INSTALL: 'load-only', + GITNEXUS_WORKER_POOL_SIZE: '2', + GITNEXUS_PARSE_CHUNK_CONCURRENCY: '1', + // Recovery may need to regenerate embeddings after a destructive phase. + // Keep that proof real but hermetic: a local deterministic endpoint means + // no model download, credentials, or external network can affect the test. + GITNEXUS_EMBEDDING_URL: embeddingUrl, + GITNEXUS_EMBEDDING_MODEL: 'gitnexus-test-deterministic-v1', + GITNEXUS_EMBEDDING_DIMS: '384', + GITNEXUS_EMBEDDING_MAX_ATTEMPTS: '1', + GITNEXUS_EMBEDDING_RETRY_CAP_MS: '1', + GITNEXUS_EMBEDDING_MIN_INTERVAL_MS: '0', + }; +}; const commitAll = (repoPath: string, message: string): void => { for (const args of [ @@ -96,11 +115,16 @@ const parseHarnessResult = (stdout: string): HarnessResult => { return JSON.parse(line.slice('HARNESS_RESULT='.length)) as HarnessResult; }; -const runChild = (repoPath: string, gitnexusHome: string, args: string[] = []): HarnessResult => { +const runChild = ( + repoPath: string, + gitnexusHome: string, + embeddingUrl: string, + args: string[] = [], +): HarnessResult => { const result = spawnSync(process.execPath, [childRunner, repoPath, ...args], { encoding: 'utf8', timeout: 300_000, - env: childEnv(gitnexusHome), + env: childEnv(gitnexusHome, embeddingUrl), stdio: ['ignore', 'pipe', 'pipe'], }); expect(result.error).toBeUndefined(); @@ -109,6 +133,42 @@ const runChild = (repoPath: string, gitnexusHome: string, args: string[] = []): return parseHarnessResult(result.stdout); }; +const startEmbeddingServer = async (): Promise<{ child: ChildProcess; url: string }> => { + const child = spawn(process.execPath, [embeddingServerRunner], { + env: { ...process.env, NODE_ENV: 'test' }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const url = await new Promise((resolve, reject) => { + let stdout = ''; + let stderr = ''; + const timer = setTimeout(() => reject(new Error('embedding server did not start')), 30_000); + child.stdout?.on('data', (chunk) => { + stdout += chunk.toString(); + const line = stdout + .split(/\r?\n/u) + .find((candidate) => candidate.startsWith('EMBEDDING_SERVER=')); + if (line) { + clearTimeout(timer); + resolve(line.slice('EMBEDDING_SERVER='.length)); + } + }); + child.stderr?.on('data', (chunk) => { + stderr += chunk.toString(); + }); + child.once('error', (error) => { + clearTimeout(timer); + reject(error); + }); + child.once('exit', (code, signal) => { + clearTimeout(timer); + reject( + new Error(`embedding server exited before ready: code=${code} signal=${signal} ${stderr}`), + ); + }); + }); + return { child, url }; +}; + const waitForFile = async (filePath: string, child: ChildProcess): Promise => { const deadline = Date.now() + 180_000; while (Date.now() < deadline) { @@ -153,9 +213,14 @@ const readEmbeddingDimensions = async (repoPath: string): Promise => { describe('large incremental analysis subprocess contract', () => { it('surfaces process failure and recovers add/edit/rename/delete plus importer closure and embeddings', async () => { expect(fs.existsSync(childRunner), `missing child runner: ${childRunner}`).toBe(true); + expect( + fs.existsSync(embeddingServerRunner), + `missing embedding server: ${embeddingServerRunner}`, + ).toBe(true); const fixture = setupLargeRepo(); + const embeddingServer = await startEmbeddingServer(); try { - const initial = runChild(fixture.repoPath, fixture.gitnexusHome); + const initial = runChild(fixture.repoPath, fixture.gitnexusHome, embeddingServer.url); expect(initial.stats.files).toBe(61); expect(initial.stats.nodes).toBeGreaterThan(61); @@ -172,10 +237,12 @@ describe('large incremental analysis subprocess contract', () => { ); commitAll(fixture.repoPath, 'exercise active FTS row deletion'); - const smallIncremental = runChild(fixture.repoPath, fixture.gitnexusHome, [ - '--fts-query', - 'r09SmallFtsNeedle', - ]); + const smallIncremental = runChild( + fixture.repoPath, + fixture.gitnexusHome, + embeddingServer.url, + ['--fts-query', 'r09SmallFtsNeedle'], + ); expect(smallIncremental.logs.join('\n')).not.toContain('switching to a full DB write'); expect(smallIncremental.stats.files).toBe(61); expect(smallIncremental.ftsSearch?.ftsAvailable).toBe(true); @@ -223,7 +290,7 @@ describe('large incremental analysis subprocess contract', () => { process.execPath, [childRunner, fixture.repoPath, '--pause-at', boundary, '--pause-ready', readyFile], { - env: childEnv(fixture.gitnexusHome), + env: childEnv(fixture.gitnexusHome, embeddingServer.url), stdio: ['ignore', 'pipe', 'pipe'], }, ); @@ -251,7 +318,7 @@ describe('large incremental analysis subprocess contract', () => { const dirtyMeta = await loadMeta(storagePath); expect(dirtyMeta?.incrementalInProgress?.phase).toBe(expectedDirtyPhase); - recovered = runChild(fixture.repoPath, fixture.gitnexusHome, [ + recovered = runChild(fixture.repoPath, fixture.gitnexusHome, embeddingServer.url, [ '--fts-query', 'r09FtsNeedle', ]); @@ -261,14 +328,21 @@ describe('large incremental analysis subprocess contract', () => { expect((await loadMeta(storagePath))?.incrementalInProgress).toBeUndefined(); const recoveredEmbeddingIds = await readEmbeddingNodeIds(fixture.repoPath); - const forced = runChild(fixture.repoPath, fixture.gitnexusHome, [ + expect( + recoveredEmbeddingIds.length, + `${boundary} recovery silently lost every preserved or regenerated embedding`, + ).toBeGreaterThanOrEqual(4); + expect(recovered.stats.embeddings).toBe(recoveredEmbeddingIds.length); + const forced = runChild(fixture.repoPath, fixture.gitnexusHome, embeddingServer.url, [ '--force', '--fts-query', 'r09FtsNeedle', ]); expect(forced.stats).toEqual(recovered.stats); expect(forced.ftsSearch).toEqual(recovered.ftsSearch); - expect(await readEmbeddingNodeIds(fixture.repoPath)).toEqual(recoveredEmbeddingIds); + expect((await readEmbeddingNodeIds(fixture.repoPath)).toSorted()).toEqual( + recoveredEmbeddingIds.toSorted(), + ); } if (!recovered) throw new Error('recovery matrix produced no successful run'); const finalRecovered = recovered; @@ -291,6 +365,9 @@ describe('large incremental analysis subprocess contract', () => { expect(survivingEmbeddingIds).not.toContain(seeded.get('src/spoke-002.ts')?.[0]); expect(await readEmbeddingDimensions(fixture.repoPath)).toEqual([384]); } finally { + if (embeddingServer.child.exitCode === null && embeddingServer.child.signalCode === null) { + await stopChild(embeddingServer.child); + } fs.rmSync(fixture.root, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } }, 2_400_000); From c1c030e50b0c734980299b5321763f9dab03c988 Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 05:59:50 +0700 Subject: [PATCH 11/16] test(incremental): harden recovery oracle --- .github/workflows/ci-tests.yml | 4 + .../test/fixtures/large-incremental-child.mjs | 4 +- gitnexus/test/helpers/embedding-seed.ts | 53 ++++- .../helpers/large-incremental-contract.ts | 208 ++++++++++++++++++ .../large-incremental-subprocess-e2e.test.ts | 120 +++++----- .../unit/large-incremental-contract.test.ts | 130 +++++++++++ 6 files changed, 449 insertions(+), 70 deletions(-) create mode 100644 gitnexus/test/helpers/large-incremental-contract.ts create mode 100644 gitnexus/test/unit/large-incremental-contract.test.ts diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 276245d305..34710997f2 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -204,6 +204,10 @@ jobs: env: GITNEXUS_REQUIRE_FTS: '1' GITNEXUS_E2E_CLI: dist + # Ubuntu coverage proves all four durable recovery boundaries. The OS + # matrix keeps the most destructive native filesystem boundary inside + # its 15-minute per-shard watchdog. + GITNEXUS_RECOVERY_BOUNDARIES: during-delete steps: # persist-credentials: false — runs tests only, never pushes (zizmor # credential-persistence / artipacked audit). diff --git a/gitnexus/test/fixtures/large-incremental-child.mjs b/gitnexus/test/fixtures/large-incremental-child.mjs index 8aee3a34a7..d53215458d 100644 --- a/gitnexus/test/fixtures/large-incremental-child.mjs +++ b/gitnexus/test/fixtures/large-incremental-child.mjs @@ -12,12 +12,13 @@ const bm25IndexUrl = pathToFileURL(path.join(packageRoot, 'dist/core/search/bm25 const [repoPath, ...args] = process.argv.slice(2); if (!repoPath) { throw new Error( - 'usage: large-incremental-child.mjs [--force] [--embeddings] [--pause-at --pause-ready ] [--fts-query ]', + 'usage: large-incremental-child.mjs [--force] [--embeddings] [--drop-embeddings] [--pause-at --pause-ready ] [--fts-query ]', ); } const force = args.includes('--force'); const embeddings = args.includes('--embeddings'); +const dropEmbeddings = args.includes('--drop-embeddings'); const pauseAtIndex = args.indexOf('--pause-at'); const pauseAt = pauseAtIndex >= 0 ? args[pauseAtIndex + 1] : undefined; const pauseReadyIndex = args.indexOf('--pause-ready'); @@ -65,6 +66,7 @@ try { skipSkills: true, force, embeddings, + dropEmbeddings, embeddingsNodeLimit: embeddings ? 0 : undefined, }, { onProgress: () => {}, onLog, onRecoveryBoundary }, diff --git a/gitnexus/test/helpers/embedding-seed.ts b/gitnexus/test/helpers/embedding-seed.ts index 4d337fb732..59e731319c 100644 --- a/gitnexus/test/helpers/embedding-seed.ts +++ b/gitnexus/test/helpers/embedding-seed.ts @@ -13,7 +13,8 @@ * close. Zero vectors need no VECTOR extension: the CodeEmbedding TABLE is * plain schema; only the HNSW index is extension-gated. */ -import { expect } from 'vitest'; +import { createHash } from 'node:crypto'; + import { getStoragePaths, loadMeta, @@ -105,11 +106,57 @@ export async function readEmbeddingNodeIds(repoPath: string): Promise } } +export interface EmbeddingRowFingerprint { + nodeId: string; + chunkIndex: number; + startLine: number; + endLine: number; + contentHash: string; + dimensions: number; + vectorSha256: string; +} + +/** Read a compact, deterministic fingerprint of every persisted embedding row. */ +export async function readEmbeddingRowFingerprints( + repoPath: string, +): Promise { + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + const { lbugPath } = getStoragePaths(repoPath); + await adapter.initLbug(lbugPath); + try { + const rows = (await adapter.executeQuery( + `MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.contentHash AS contentHash, e.embedding AS embedding`, + )) as Array>; + return rows + .map((row) => { + const rawVector = row.embedding ?? row[5]; + const vector = Array.isArray(rawVector) + ? rawVector.map(Number) + : Array.from(rawVector as Iterable).map(Number); + return { + nodeId: String(row.nodeId ?? row[0] ?? ''), + chunkIndex: Number(row.chunkIndex ?? row[1] ?? 0), + startLine: Number(row.startLine ?? row[2] ?? 0), + endLine: Number(row.endLine ?? row[3] ?? 0), + contentHash: String(row.contentHash ?? row[4] ?? ''), + dimensions: vector.length, + vectorSha256: createHash('sha256').update(JSON.stringify(vector)).digest('hex'), + }; + }) + .sort( + (left, right) => + left.nodeId.localeCompare(right.nodeId) || left.chunkIndex - right.chunkIndex, + ); + } finally { + await adapter.closeLbug(); + } +} + /** Tamper meta.stats.embeddings so deriveEmbeddingMode sees an embedded repo * (loadMeta → spread → saveMeta, same pattern as the dirty-flag tests). */ export async function stampEmbeddingCount(storagePath: string, embeddings: number): Promise { const meta = await loadMeta(storagePath); - expect(meta).not.toBeNull(); - const tampered: RepoMeta = { ...meta!, stats: { ...meta!.stats, embeddings } }; + if (!meta) throw new Error(`missing repository metadata at ${storagePath}`); + const tampered: RepoMeta = { ...meta, stats: { ...meta.stats, embeddings } }; await saveMeta(storagePath, tampered); } diff --git a/gitnexus/test/helpers/large-incremental-contract.ts b/gitnexus/test/helpers/large-incremental-contract.ts new file mode 100644 index 0000000000..0f6689a293 --- /dev/null +++ b/gitnexus/test/helpers/large-incremental-contract.ts @@ -0,0 +1,208 @@ +import { spawn, type ChildProcess } from 'node:child_process'; + +const PORTABLE_ENV_KEYS = [ + 'PATH', + 'SHELL', + 'TMPDIR', + 'TMP', + 'TEMP', + 'LANG', + 'LC_ALL', + 'LC_CTYPE', + 'TZ', +] as const; + +const WINDOWS_ENV_KEYS = ['SystemRoot', 'WINDIR', 'COMSPEC', 'PATHEXT'] as const; + +export const RECOVERY_BOUNDARY_CASES = [ + ['before-delete', 'escalated-full-write'], + ['during-delete', 'escalated-full-write'], + ['during-insert', 'escalated-load-graph'], + ['before-finalize', 'escalated-load-graph'], +] as const; + +export type RecoveryBoundary = (typeof RECOVERY_BOUNDARY_CASES)[number][0]; + +export const ALL_RECOVERY_BOUNDARIES: readonly RecoveryBoundary[] = RECOVERY_BOUNDARY_CASES.map( + ([boundary]) => boundary, +); + +export function createHermeticProcessEnv( + source: NodeJS.ProcessEnv, + home: string, + overrides: NodeJS.ProcessEnv = {}, + platform: NodeJS.Platform = process.platform, +): NodeJS.ProcessEnv { + if (!home) throw new Error('hermetic process environment requires an isolated home'); + const allowedKeys = + platform === 'win32' + ? ([...PORTABLE_ENV_KEYS, ...WINDOWS_ENV_KEYS] as const) + : PORTABLE_ENV_KEYS; + const env: NodeJS.ProcessEnv = {}; + for (const key of allowedKeys) { + if (typeof source[key] === 'string') env[key] = source[key]; + } + return { + ...env, + HOME: home, + USERPROFILE: home, + CI: '1', + GIT_TERMINAL_PROMPT: '0', + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: platform === 'win32' ? 'NUL' : '/dev/null', + ...overrides, + }; +} + +export function selectRecoveryBoundaries(raw: string | undefined): RecoveryBoundary[] { + if (raw === undefined) return [...ALL_RECOVERY_BOUNDARIES]; + const requested = raw + .split(',') + .map((value) => value.trim()) + .filter(Boolean); + if (requested.length === 0) { + throw new Error('GITNEXUS_RECOVERY_BOUNDARIES must name at least one boundary'); + } + const known = new Set(ALL_RECOVERY_BOUNDARIES); + const unknown = requested.filter((value) => !known.has(value)); + if (unknown.length > 0) { + throw new Error( + `GITNEXUS_RECOVERY_BOUNDARIES contains unknown value(s): ${unknown.join(', ')}`, + ); + } + return [...new Set(requested)] as RecoveryBoundary[]; +} + +export type ChildExit = { code: number | null; signal: NodeJS.Signals | null }; + +const currentExit = (child: ChildProcess): ChildExit | undefined => { + if (child.exitCode === null && child.signalCode === null) return undefined; + return { code: child.exitCode, signal: child.signalCode as NodeJS.Signals | null }; +}; + +export async function terminateChild( + child: ChildProcess, + graceMs = 30_000, + killWaitMs = 5_000, +): Promise { + const exited = currentExit(child); + if (exited) return exited; + + return new Promise((resolve, reject) => { + let killTimer: NodeJS.Timeout | undefined; + let settled = false; + + const cleanup = (): void => { + if (graceTimer) clearTimeout(graceTimer); + if (killTimer) clearTimeout(killTimer); + child.off('exit', onExit); + }; + const finish = (result: ChildExit): void => { + if (settled) return; + settled = true; + cleanup(); + resolve(result); + }; + const onExit = (code: number | null, signal: NodeJS.Signals | null): void => { + finish({ code, signal }); + }; + const graceTimer = setTimeout(() => { + if (settled) return; + child.kill('SIGKILL'); + killTimer = setTimeout(() => { + if (settled) return; + settled = true; + cleanup(); + reject(new Error('child did not terminate after SIGKILL')); + }, killWaitMs); + }, graceMs); + + child.once('exit', onExit); + const racedExit = currentExit(child); + if (racedExit) { + finish(racedExit); + return; + } + + child.kill('SIGTERM'); + }); +} + +export interface ReadyProcessOptions { + command: string; + args: string[]; + env: NodeJS.ProcessEnv; + readyPrefix: string; + timeoutMs?: number; + terminateGraceMs?: number; + cwd?: string; +} + +export async function startReadyProcess( + options: ReadyProcessOptions, +): Promise<{ child: ChildProcess; value: string }> { + const child = spawn(options.command, options.args, { + cwd: options.cwd, + env: options.env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + try { + const value = await new Promise((resolve, reject) => { + let stdout = ''; + let settled = false; + const timeoutMs = options.timeoutMs ?? 30_000; + const timer = setTimeout( + () => rejectOnce(new Error(`child did not become ready within ${timeoutMs}ms`)), + timeoutMs, + ); + + const cleanup = (): void => { + clearTimeout(timer); + child.stdout?.off('data', onStdout); + child.stderr?.off('data', onStderr); + child.off('error', onError); + child.off('exit', onExit); + }; + const rejectOnce = (error: Error): void => { + if (settled) return; + settled = true; + cleanup(); + reject(error); + }; + const onStdout = (chunk: Buffer | string): void => { + stdout += chunk.toString(); + const line = stdout + .split(/\r?\n/u) + .find((candidate) => candidate.startsWith(options.readyPrefix)); + if (!line || settled) return; + settled = true; + cleanup(); + resolve(line.slice(options.readyPrefix.length)); + }; + const onStderr = (chunk: Buffer | string): void => { + stderr = `${stderr}${chunk.toString()}`.slice(-8_192); + }; + const onError = (error: Error): void => rejectOnce(error); + const onExit = (code: number | null, signal: NodeJS.Signals | null): void => + rejectOnce( + new Error( + `child exited before ready: code=${code} signal=${signal}${stderr ? ` ${stderr}` : ''}`, + ), + ); + + child.stdout?.on('data', onStdout); + child.stderr?.on('data', onStderr); + child.once('error', onError); + child.once('exit', onExit); + }); + return { child, value }; + } catch (error) { + try { + await terminateChild(child, options.terminateGraceMs ?? 1_000); + } catch (cleanupError) { + throw new AggregateError([error, cleanupError], 'child startup and cleanup both failed'); + } + throw error; + } +} diff --git a/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts b/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts index 24c29a60e5..8fb3e95775 100644 --- a/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts +++ b/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts @@ -7,9 +7,17 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; import { readEmbeddingNodeIds, + readEmbeddingRowFingerprints, seedEmbeddingsForFiles, stampEmbeddingCount, } from '../helpers/embedding-seed.js'; +import { + createHermeticProcessEnv, + RECOVERY_BOUNDARY_CASES, + selectRecoveryBoundaries, + startReadyProcess, + terminateChild, +} from '../helpers/large-incremental-contract.js'; import { getStoragePaths, loadMeta } from '../../src/storage/repo-manager.js'; const testDir = path.dirname(fileURLToPath(import.meta.url)); @@ -33,12 +41,7 @@ type HarnessResult = { }; const childEnv = (gitnexusHome: string, embeddingUrl: string): NodeJS.ProcessEnv => { - const env = { ...process.env }; - for (const key of Object.keys(env)) { - if (key.startsWith('GITNEXUS_EMBEDDING_')) delete env[key]; - } - return { - ...env, + return createHermeticProcessEnv(process.env, gitnexusHome, { CI: '1', NODE_ENV: 'test', GITNEXUS_HOME: gitnexusHome, @@ -57,7 +60,7 @@ const childEnv = (gitnexusHome: string, embeddingUrl: string): NodeJS.ProcessEnv GITNEXUS_EMBEDDING_MAX_ATTEMPTS: '1', GITNEXUS_EMBEDDING_RETRY_CAP_MS: '1', GITNEXUS_EMBEDDING_MIN_INTERVAL_MS: '0', - }; + }); }; const commitAll = (repoPath: string, message: string): void => { @@ -83,12 +86,21 @@ const commitAll = (repoPath: string, message: string): void => { }; const setupLargeRepo = (): { root: string; repoPath: string; gitnexusHome: string } => { + const installedExtensions = path.join(os.homedir(), '.lbdb', 'extension'); + if (!fs.existsSync(installedExtensions)) { + throw new Error( + `large incremental test requires preinstalled extensions: ${installedExtensions}`, + ); + } const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-large-incremental-')); const repoPath = path.join(root, 'repo'); const gitnexusHome = path.join(root, 'home'); const src = path.join(repoPath, 'src'); fs.mkdirSync(src, { recursive: true }); fs.mkdirSync(gitnexusHome, { recursive: true }); + fs.cpSync(installedExtensions, path.join(gitnexusHome, '.lbdb', 'extension'), { + recursive: true, + }); fs.writeFileSync( path.join(src, 'hub.ts'), @@ -133,40 +145,16 @@ const runChild = ( return parseHarnessResult(result.stdout); }; -const startEmbeddingServer = async (): Promise<{ child: ChildProcess; url: string }> => { - const child = spawn(process.execPath, [embeddingServerRunner], { - env: { ...process.env, NODE_ENV: 'test' }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - const url = await new Promise((resolve, reject) => { - let stdout = ''; - let stderr = ''; - const timer = setTimeout(() => reject(new Error('embedding server did not start')), 30_000); - child.stdout?.on('data', (chunk) => { - stdout += chunk.toString(); - const line = stdout - .split(/\r?\n/u) - .find((candidate) => candidate.startsWith('EMBEDDING_SERVER=')); - if (line) { - clearTimeout(timer); - resolve(line.slice('EMBEDDING_SERVER='.length)); - } - }); - child.stderr?.on('data', (chunk) => { - stderr += chunk.toString(); - }); - child.once('error', (error) => { - clearTimeout(timer); - reject(error); - }); - child.once('exit', (code, signal) => { - clearTimeout(timer); - reject( - new Error(`embedding server exited before ready: code=${code} signal=${signal} ${stderr}`), - ); - }); +const startEmbeddingServer = async ( + isolatedHome: string, +): Promise<{ child: ChildProcess; url: string }> => { + const ready = await startReadyProcess({ + command: process.execPath, + args: [embeddingServerRunner], + env: createHermeticProcessEnv(process.env, isolatedHome, { NODE_ENV: 'test' }), + readyPrefix: 'EMBEDDING_SERVER=', }); - return { child, url }; + return { child: ready.child, url: ready.value }; }; const waitForFile = async (filePath: string, child: ChildProcess): Promise => { @@ -181,21 +169,6 @@ const waitForFile = async (filePath: string, child: ChildProcess): Promise throw new Error('timed out waiting for the incremental pause point'); }; -const stopChild = async ( - child: ChildProcess, -): Promise<{ code: number | null; signal: string | null }> => - new Promise((resolve, reject) => { - const timer = setTimeout( - () => reject(new Error('child did not terminate after SIGTERM')), - 30_000, - ); - child.once('exit', (code, signal) => { - clearTimeout(timer); - resolve({ code, signal }); - }); - child.kill('SIGTERM'); - }); - const readEmbeddingDimensions = async (repoPath: string): Promise => { const adapter = await import('../../src/core/lbug/lbug-adapter.js'); const { lbugPath } = getStoragePaths(repoPath); @@ -218,8 +191,9 @@ describe('large incremental analysis subprocess contract', () => { `missing embedding server: ${embeddingServerRunner}`, ).toBe(true); const fixture = setupLargeRepo(); - const embeddingServer = await startEmbeddingServer(); + let embeddingServer: Awaited> | undefined; try { + embeddingServer = await startEmbeddingServer(fixture.gitnexusHome); const initial = runChild(fixture.repoPath, fixture.gitnexusHome, embeddingServer.url); expect(initial.stats.files).toBe(61); expect(initial.stats.nodes).toBeGreaterThan(61); @@ -273,12 +247,12 @@ describe('large incremental analysis subprocess contract', () => { ); commitAll(fixture.repoPath, 'exercise incremental write set'); - const boundaries = [ - ['before-delete', 'escalated-full-write'], - ['during-delete', 'escalated-full-write'], - ['during-insert', 'escalated-load-graph'], - ['before-finalize', 'escalated-load-graph'], - ] as const; + const selectedBoundaries = new Set( + selectRecoveryBoundaries(process.env.GITNEXUS_RECOVERY_BOUNDARIES), + ); + const boundaries = RECOVERY_BOUNDARY_CASES.filter(([boundary]) => + selectedBoundaries.has(boundary), + ); let recovered: HarnessResult | undefined; for (const [index, [boundary, expectedDirtyPhase]] of boundaries.entries()) { if (index > 0) { @@ -312,13 +286,14 @@ describe('large incremental analysis subprocess contract', () => { expect(pauseState.details.table).toBeTruthy(); } - const stopped = await stopChild(interrupted); + const stopped = await terminateChild(interrupted); expect(stopped.code).not.toBe(0); expect(stopped.signal).toBe('SIGTERM'); const dirtyMeta = await loadMeta(storagePath); expect(dirtyMeta?.incrementalInProgress?.phase).toBe(expectedDirtyPhase); recovered = runChild(fixture.repoPath, fixture.gitnexusHome, embeddingServer.url, [ + '--embeddings', '--fts-query', 'r09FtsNeedle', ]); @@ -333,8 +308,11 @@ describe('large incremental analysis subprocess contract', () => { `${boundary} recovery silently lost every preserved or regenerated embedding`, ).toBeGreaterThanOrEqual(4); expect(recovered.stats.embeddings).toBe(recoveredEmbeddingIds.length); + const recoveredEmbeddingRows = await readEmbeddingRowFingerprints(fixture.repoPath); const forced = runChild(fixture.repoPath, fixture.gitnexusHome, embeddingServer.url, [ '--force', + '--embeddings', + '--drop-embeddings', '--fts-query', 'r09FtsNeedle', ]); @@ -343,6 +321,9 @@ describe('large incremental analysis subprocess contract', () => { expect((await readEmbeddingNodeIds(fixture.repoPath)).toSorted()).toEqual( recoveredEmbeddingIds.toSorted(), ); + expect(await readEmbeddingRowFingerprints(fixture.repoPath)).toEqual( + recoveredEmbeddingRows, + ); } if (!recovered) throw new Error('recovery matrix produced no successful run'); const finalRecovered = recovered; @@ -365,10 +346,17 @@ describe('large incremental analysis subprocess contract', () => { expect(survivingEmbeddingIds).not.toContain(seeded.get('src/spoke-002.ts')?.[0]); expect(await readEmbeddingDimensions(fixture.repoPath)).toEqual([384]); } finally { - if (embeddingServer.child.exitCode === null && embeddingServer.child.signalCode === null) { - await stopChild(embeddingServer.child); + try { + if ( + embeddingServer && + embeddingServer.child.exitCode === null && + embeddingServer.child.signalCode === null + ) { + await terminateChild(embeddingServer.child); + } + } finally { + fs.rmSync(fixture.root, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } - fs.rmSync(fixture.root, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } }, 2_400_000); }); diff --git a/gitnexus/test/unit/large-incremental-contract.test.ts b/gitnexus/test/unit/large-incremental-contract.test.ts new file mode 100644 index 0000000000..365b9682f4 --- /dev/null +++ b/gitnexus/test/unit/large-incremental-contract.test.ts @@ -0,0 +1,130 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { + ALL_RECOVERY_BOUNDARIES, + createHermeticProcessEnv, + selectRecoveryBoundaries, + startReadyProcess, + terminateChild, +} from '../helpers/large-incremental-contract.js'; + +const temporaryRoots: string[] = []; + +const makeTemporaryRoot = (): string => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-large-contract-')); + temporaryRoots.push(root); + return root; +}; + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +describe('large incremental subprocess isolation', () => { + it('passes only portable process keys into an isolated home', () => { + const home = makeTemporaryRoot(); + const env = createHermeticProcessEnv( + { + PATH: '/safe/bin', + LANG: 'C.UTF-8', + SSH_AUTH_SOCK: '/private/agent.sock', + GITHUB_TOKEN: 'must-not-leak', + NODE_OPTIONS: '--require private-hook.cjs', + HTTPS_PROXY: 'http://user:secret@example.invalid', + }, + home, + { NODE_ENV: 'test', GITNEXUS_HOME: path.join(home, '.gitnexus') }, + 'linux', + ); + + expect(env).toMatchObject({ + PATH: '/safe/bin', + LANG: 'C.UTF-8', + HOME: home, + USERPROFILE: home, + CI: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + NODE_ENV: 'test', + GITNEXUS_HOME: path.join(home, '.gitnexus'), + }); + expect(env).not.toHaveProperty('SSH_AUTH_SOCK'); + expect(env).not.toHaveProperty('GITHUB_TOKEN'); + expect(env).not.toHaveProperty('NODE_OPTIONS'); + expect(env).not.toHaveProperty('HTTPS_PROXY'); + }); + + it.skipIf(process.platform === 'win32')( + 'kills and reaps a child that never emits readiness', + async () => { + const home = makeTemporaryRoot(); + const pidFile = path.join(home, 'child.pid'); + const script = + "require('node:fs').writeFileSync(process.argv[1], String(process.pid)); " + + "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);"; + + await expect( + startReadyProcess({ + command: process.execPath, + args: ['-e', script, pidFile], + env: createHermeticProcessEnv(process.env, home, {}, process.platform), + readyPrefix: 'READY=', + timeoutMs: 100, + terminateGraceMs: 100, + }), + ).rejects.toThrow('did not become ready'); + + const pid = Number(fs.readFileSync(pidFile, 'utf8')); + expect(() => process.kill(pid, 0)).toThrow(); + }, + ); + + it.skipIf(process.platform === 'win32')( + 'escalates a signal-resistant child to SIGKILL', + async () => { + const home = makeTemporaryRoot(); + const readyScript = + "process.on('SIGTERM', () => {}); process.stdout.write('READY=ok\\n'); " + + 'setInterval(() => {}, 1000);'; + const ready = await startReadyProcess({ + command: process.execPath, + args: ['-e', readyScript], + env: createHermeticProcessEnv(process.env, home, {}, process.platform), + readyPrefix: 'READY=', + timeoutMs: 1_000, + }); + + await expect(terminateChild(ready.child, 100)).resolves.toEqual({ + code: null, + signal: 'SIGKILL', + }); + const pid = ready.child.pid; + expect(pid).toBeDefined(); + if (!pid) throw new Error('ready child did not expose a pid'); + expect(() => process.kill(pid, 0)).toThrow(); + }, + ); +}); + +describe('large incremental recovery boundary selection', () => { + it('runs every durable interruption boundary by default', () => { + expect(selectRecoveryBoundaries(undefined)).toEqual(ALL_RECOVERY_BOUNDARIES); + }); + + it('accepts an explicit bounded subset for the cross-platform matrix', () => { + expect(selectRecoveryBoundaries('during-delete,before-finalize,during-delete')).toEqual([ + 'during-delete', + 'before-finalize', + ]); + }); + + it('rejects empty or unknown boundary configuration', () => { + expect(() => selectRecoveryBoundaries(' ')).toThrow('must name at least one'); + expect(() => selectRecoveryBoundaries('during-delete,unknown')).toThrow('unknown'); + }); +}); From c231cbf6ae749b0fa62c57adfcd23a41e8ee60e0 Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 06:10:11 +0700 Subject: [PATCH 12/16] test(incremental): validate staged extensions --- .../helpers/large-incremental-contract.ts | 87 +++++++++++++++++++ .../large-incremental-subprocess-e2e.test.ts | 43 ++++----- .../unit/large-incremental-contract.test.ts | 51 +++++++++++ 3 files changed, 160 insertions(+), 21 deletions(-) diff --git a/gitnexus/test/helpers/large-incremental-contract.ts b/gitnexus/test/helpers/large-incremental-contract.ts index 0f6689a293..3c1e3b4aea 100644 --- a/gitnexus/test/helpers/large-incremental-contract.ts +++ b/gitnexus/test/helpers/large-incremental-contract.ts @@ -1,4 +1,6 @@ import { spawn, type ChildProcess } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; const PORTABLE_ENV_KEYS = [ 'PATH', @@ -27,6 +29,91 @@ export const ALL_RECOVERY_BOUNDARIES: readonly RecoveryBoundary[] = RECOVERY_BOU ([boundary]) => boundary, ); +export function setupDisposableRoot(prefix: string, setup: (root: string) => T): T { + const root = fs.mkdtempSync(prefix); + try { + return setup(root); + } catch (error) { + fs.rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + throw error; + } +} + +type RegularTreeEntry = { relativePath: string; directory: boolean; mode: number }; + +const collectRegularTree = (sourceRoot: string): RegularTreeEntry[] => { + const rootStat = fs.lstatSync(sourceRoot); + if (rootStat.isSymbolicLink()) { + throw new Error(`regular file tree source is a symbolic link: ${sourceRoot}`); + } + if (!rootStat.isDirectory()) { + throw new Error(`regular file tree source is not a directory: ${sourceRoot}`); + } + + const entries: RegularTreeEntry[] = []; + const visit = (directory: string, relativeDirectory: string): void => { + for (const name of fs.readdirSync(directory).sort()) { + const sourcePath = path.join(directory, name); + const relativePath = path.join(relativeDirectory, name); + const stat = fs.lstatSync(sourcePath); + if (stat.isSymbolicLink()) { + throw new Error(`regular file tree contains a symbolic link: ${sourcePath}`); + } + if (stat.isDirectory()) { + entries.push({ relativePath, directory: true, mode: stat.mode }); + visit(sourcePath, relativePath); + } else if (stat.isFile()) { + entries.push({ relativePath, directory: false, mode: stat.mode }); + } else { + throw new Error(`regular file tree contains a non-regular entry: ${sourcePath}`); + } + } + }; + visit(sourceRoot, ''); + return entries; +}; + +const assertContainedRealPath = (root: string, candidate: string): void => { + const realRoot = fs.realpathSync.native(root); + const realCandidate = fs.realpathSync.native(candidate); + if (realCandidate !== realRoot && !realCandidate.startsWith(`${realRoot}${path.sep}`)) { + throw new Error(`staged path escaped its destination root: ${candidate}`); + } +}; + +export function stageRegularFileTree(sourceRoot: string, destinationRoot: string): void { + if (fs.existsSync(destinationRoot)) { + throw new Error(`regular file tree destination already exists: ${destinationRoot}`); + } + const entries = collectRegularTree(sourceRoot); + try { + fs.mkdirSync(destinationRoot, { recursive: true, mode: 0o700 }); + assertContainedRealPath(destinationRoot, destinationRoot); + for (const entry of entries) { + const sourcePath = path.join(sourceRoot, entry.relativePath); + const destinationPath = path.join(destinationRoot, entry.relativePath); + if (entry.directory) { + fs.mkdirSync(destinationPath, { mode: entry.mode & 0o777 }); + } else { + const sourceStat = fs.lstatSync(sourcePath); + if (!sourceStat.isFile() || sourceStat.isSymbolicLink()) { + throw new Error(`regular file changed type during staging: ${sourcePath}`); + } + fs.mkdirSync(path.dirname(destinationPath), { recursive: true, mode: 0o700 }); + fs.copyFileSync(sourcePath, destinationPath, fs.constants.COPYFILE_EXCL); + fs.chmodSync(destinationPath, entry.mode & 0o777); + if (!fs.lstatSync(destinationPath).isFile()) { + throw new Error(`staged extension is not a regular file: ${destinationPath}`); + } + } + assertContainedRealPath(destinationRoot, destinationPath); + } + } catch (error) { + fs.rmSync(destinationRoot, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + throw error; + } +} + export function createHermeticProcessEnv( source: NodeJS.ProcessEnv, home: string, diff --git a/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts b/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts index 8fb3e95775..5c35036876 100644 --- a/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts +++ b/gitnexus/test/integration/large-incremental-subprocess-e2e.test.ts @@ -14,7 +14,9 @@ import { import { createHermeticProcessEnv, RECOVERY_BOUNDARY_CASES, + setupDisposableRoot, selectRecoveryBoundaries, + stageRegularFileTree, startReadyProcess, terminateChild, } from '../helpers/large-incremental-contract.js'; @@ -92,31 +94,30 @@ const setupLargeRepo = (): { root: string; repoPath: string; gitnexusHome: strin `large incremental test requires preinstalled extensions: ${installedExtensions}`, ); } - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-large-incremental-')); - const repoPath = path.join(root, 'repo'); - const gitnexusHome = path.join(root, 'home'); - const src = path.join(repoPath, 'src'); - fs.mkdirSync(src, { recursive: true }); - fs.mkdirSync(gitnexusHome, { recursive: true }); - fs.cpSync(installedExtensions, path.join(gitnexusHome, '.lbdb', 'extension'), { - recursive: true, - }); + return setupDisposableRoot(path.join(os.tmpdir(), 'gitnexus-large-incremental-'), (root) => { + const repoPath = path.join(root, 'repo'); + const gitnexusHome = path.join(root, 'home'); + const src = path.join(repoPath, 'src'); + fs.mkdirSync(src, { recursive: true }); + fs.mkdirSync(gitnexusHome, { recursive: true }); + stageRegularFileTree(installedExtensions, path.join(gitnexusHome, '.lbdb', 'extension')); - fs.writeFileSync( - path.join(src, 'hub.ts'), - 'export function hubValue(value: number): number {\n return value + 1;\n}\n', - ); - for (let index = 0; index < 60; index++) { - const suffix = String(index).padStart(3, '0'); fs.writeFileSync( - path.join(src, `spoke-${suffix}.ts`), - `import { hubValue } from './hub';\n\nexport function spoke${index}(): number {\n return hubValue(${index});\n}\n`, + path.join(src, 'hub.ts'), + 'export function hubValue(value: number): number {\n return value + 1;\n}\n', ); - } + for (let index = 0; index < 60; index++) { + const suffix = String(index).padStart(3, '0'); + fs.writeFileSync( + path.join(src, `spoke-${suffix}.ts`), + `import { hubValue } from './hub';\n\nexport function spoke${index}(): number {\n return hubValue(${index});\n}\n`, + ); + } - expect(spawnSync('git', ['init', '-q', '-b', 'main'], { cwd: repoPath }).status).toBe(0); - commitAll(repoPath, 'initial large fixture'); - return { root, repoPath, gitnexusHome }; + expect(spawnSync('git', ['init', '-q', '-b', 'main'], { cwd: repoPath }).status).toBe(0); + commitAll(repoPath, 'initial large fixture'); + return { root, repoPath, gitnexusHome }; + }); }; const parseHarnessResult = (stdout: string): HarnessResult => { diff --git a/gitnexus/test/unit/large-incremental-contract.test.ts b/gitnexus/test/unit/large-incremental-contract.test.ts index 365b9682f4..55611e8d2b 100644 --- a/gitnexus/test/unit/large-incremental-contract.test.ts +++ b/gitnexus/test/unit/large-incremental-contract.test.ts @@ -7,7 +7,9 @@ import { afterEach, describe, expect, it } from 'vitest'; import { ALL_RECOVERY_BOUNDARIES, createHermeticProcessEnv, + setupDisposableRoot, selectRecoveryBoundaries, + stageRegularFileTree, startReadyProcess, terminateChild, } from '../helpers/large-incremental-contract.js'; @@ -109,6 +111,55 @@ describe('large incremental subprocess isolation', () => { expect(() => process.kill(pid, 0)).toThrow(); }, ); + + it.skipIf(process.platform === 'win32')( + 'rejects extension symlinks before creating the isolated destination', + () => { + const root = makeTemporaryRoot(); + const source = path.join(root, 'source'); + const destination = path.join(root, 'destination'); + const outside = path.join(root, 'outside.bin'); + fs.mkdirSync(path.join(source, '0.18.0', 'osx_arm64', 'fts'), { recursive: true }); + fs.writeFileSync(outside, 'outside\n'); + fs.symlinkSync( + outside, + path.join(source, '0.18.0', 'osx_arm64', 'fts', 'libfts.lbug_extension'), + ); + + expect(() => stageRegularFileTree(source, destination)).toThrow('symbolic link'); + expect(fs.existsSync(destination)).toBe(false); + }, + ); + + it('copies only validated regular extension files', () => { + const root = makeTemporaryRoot(); + const source = path.join(root, 'source'); + const destination = path.join(root, 'destination'); + const relative = path.join('0.18.0', 'platform', 'fts', 'libfts.lbug_extension'); + fs.mkdirSync(path.dirname(path.join(source, relative)), { recursive: true }); + fs.writeFileSync(path.join(source, relative), 'extension-bytes\n'); + + stageRegularFileTree(source, destination); + + expect(fs.readFileSync(path.join(destination, relative), 'utf8')).toBe('extension-bytes\n'); + expect(fs.lstatSync(path.join(destination, relative)).isFile()).toBe(true); + }); + + it('removes an allocated disposable root when setup fails', () => { + const parent = makeTemporaryRoot(); + let allocated = ''; + + expect(() => + setupDisposableRoot(path.join(parent, 'fixture-'), (root) => { + allocated = root; + fs.writeFileSync(path.join(root, 'partial.txt'), 'partial\n'); + throw new Error('injected setup failure'); + }), + ).toThrow('injected setup failure'); + + expect(allocated).not.toBe(''); + expect(fs.existsSync(allocated)).toBe(false); + }); }); describe('large incremental recovery boundary selection', () => { From ca65489c990bff87427bb72d454213cd828dc253 Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 12:15:54 +0700 Subject: [PATCH 13/16] fix(scan): canonicalize traversal without order-sensitive Rust binding --- .../src/core/ingestion/filesystem-walker.ts | 5 +++ .../ingestion/languages/rust/range-binding.ts | 9 ++++- .../test/unit/filesystem-walker-order.test.ts | 39 +++++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 gitnexus/test/unit/filesystem-walker-order.test.ts diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index 0af28a9586..823b3670fd 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -83,6 +83,11 @@ export const walkRepositoryPaths = async ( } } + // Filesystem/glob traversal order is not stable across filesystems or repeated + // scans. Canonicalize once at the scan boundary so every downstream phase sees + // the same repository order. + entries.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0)); + if (skippedLarge > 0) { const isDefault = maxFileSizeBytes === DEFAULT_MAX_FILE_SIZE_BYTES; const isOverrideUnset = !process.env.GITNEXUS_MAX_FILE_SIZE; diff --git a/gitnexus/src/core/ingestion/languages/rust/range-binding.ts b/gitnexus/src/core/ingestion/languages/rust/range-binding.ts index 285b84d524..4c7224333d 100644 --- a/gitnexus/src/core/ingestion/languages/rust/range-binding.ts +++ b/gitnexus/src/core/ingestion/languages/rust/range-binding.ts @@ -89,6 +89,13 @@ export function populateRustRangeBindings( } } } + + // Publish per-type member bindings for the whole workspace before resolving + // assignments. Otherwise an importer processed before its defining file can + // miss a field or identity-method type solely because of file order. + const scopeMap = new Map(parsed.scopes.map((scope) => [scope.id, scope])); + processFieldTypeBindings(tree.rootNode, parsed, scopeMap); + processIdentityMethodBindings(parsed); } for (const parsed of parsedFiles) { @@ -122,8 +129,6 @@ export function populateRustRangeBindings( const moduleScope = parsed.scopes.find((s) => s.kind === 'Module'); if (moduleScope === undefined) continue; - processFieldTypeBindings(tree.rootNode, parsed, scopeMap); - processIdentityMethodBindings(parsed); processForLoops(tree.rootNode, parsed, scopeMap, moduleScope, allReturnTypes); processPatternBindings(tree.rootNode, parsed, scopeMap, moduleScope); processStructDestructuring(tree.rootNode, parsed, scopeMap, moduleScope, allFieldTypes); diff --git a/gitnexus/test/unit/filesystem-walker-order.test.ts b/gitnexus/test/unit/filesystem-walker-order.test.ts new file mode 100644 index 0000000000..7b75128a47 --- /dev/null +++ b/gitnexus/test/unit/filesystem-walker-order.test.ts @@ -0,0 +1,39 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { glob } from 'glob'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('glob', () => ({ glob: vi.fn() })); +vi.mock('../../src/config/ignore-service.js', () => ({ + createIgnoreFilter: vi.fn(async () => []), +})); + +import { walkRepositoryPaths } from '../../src/core/ingestion/filesystem-walker.js'; + +const temporaryRoots: string[] = []; + +afterEach(async () => { + vi.mocked(glob).mockReset(); + await Promise.all( + temporaryRoots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })), + ); +}); + +describe('walkRepositoryPaths ordering', () => { + it('returns accepted files in canonical path order when glob order is unstable', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-scan-order-')); + temporaryRoots.push(root); + await Promise.all( + ['zeta.ts', 'alpha.ts', 'middle.ts'].map((file) => + fs.writeFile(path.join(root, file), `export const ${file[0]} = true;\n`), + ), + ); + vi.mocked(glob).mockResolvedValue(['zeta.ts', 'alpha.ts', 'middle.ts']); + + const result = await walkRepositoryPaths(root); + + expect(result.map((entry) => entry.path)).toEqual(['alpha.ts', 'middle.ts', 'zeta.ts']); + }); +}); From 26d740e176f291101fe27b7f3da5be6cffac1d69 Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 12:43:54 +0700 Subject: [PATCH 14/16] fix(php): resolve function imports by declaring file --- .../scope-resolution/finalize-algorithm.ts | 8 ++++- gitnexus-shared/src/scope-resolution/types.ts | 5 +++ .../languages/php/import-decomposer.ts | 13 ++++++++ .../ingestion/languages/php/import-target.ts | 32 ++++++++++++++++++- .../core/ingestion/languages/php/interpret.ts | 11 +++++++ .../ingestion/languages/php/scope-resolver.ts | 4 +-- .../contract/scope-resolver.ts | 12 +++++++ .../scope-resolution/pipeline/run.ts | 7 ++-- .../test/integration/resolvers/php.test.ts | 5 +++ 9 files changed, 91 insertions(+), 6 deletions(-) diff --git a/gitnexus-shared/src/scope-resolution/finalize-algorithm.ts b/gitnexus-shared/src/scope-resolution/finalize-algorithm.ts index f5d3dd0bf4..f381bb12e3 100644 --- a/gitnexus-shared/src/scope-resolution/finalize-algorithm.ts +++ b/gitnexus-shared/src/scope-resolution/finalize-algorithm.ts @@ -93,6 +93,7 @@ export interface FinalizeHooks { targetRaw: string, fromFile: string, workspaceIndex: WorkspaceIndex, + parsedImport?: ParsedImport, ): string | readonly string[] | null; /** @@ -348,7 +349,12 @@ function makeEdgeDrafts( ]; } - const targetFile = hooks.resolveImportTarget(parsed.targetRaw ?? '', file.filePath, workspace); + const targetFile = hooks.resolveImportTarget( + parsed.targetRaw ?? '', + file.filePath, + workspace, + parsed, + ); // Edge is unresolvable at the file level — mark unresolved now. if (targetFile === null) { diff --git a/gitnexus-shared/src/scope-resolution/types.ts b/gitnexus-shared/src/scope-resolution/types.ts index bf837639e6..6012694cfa 100644 --- a/gitnexus-shared/src/scope-resolution/types.ts +++ b/gitnexus-shared/src/scope-resolution/types.ts @@ -105,6 +105,9 @@ export type ParsedImport = readonly localName: string; readonly importedName: string; readonly targetRaw: string; + /** Provider-specific imported symbol category when module and symbol + * namespaces have distinct resolution rules (for example PHP). */ + readonly importedSymbolKind?: 'type' | 'function' | 'const'; /** * Set by providers when `targetRaw` already names the imported symbol * rather than only its containing module. Consumers that compose @@ -127,6 +130,8 @@ export type ParsedImport = readonly alias: string; readonly targetRaw: string; /** See the same field on the `named` variant. */ + readonly importedSymbolKind?: 'type' | 'function' | 'const'; + /** See the same field on the `named` variant. */ readonly targetIncludesImportedName?: boolean; } /** diff --git a/gitnexus/src/core/ingestion/languages/php/import-decomposer.ts b/gitnexus/src/core/ingestion/languages/php/import-decomposer.ts index bcf57d99d5..f3d6cfc736 100644 --- a/gitnexus/src/core/ingestion/languages/php/import-decomposer.ts +++ b/gitnexus/src/core/ingestion/languages/php/import-decomposer.ts @@ -22,9 +22,11 @@ import type { Capture, CaptureMatch } from 'gitnexus-shared'; import { nodeToCapture, syntheticCapture, type SyntaxNode } from '../../utils/ast-helpers.js'; export type PhpImportKind = 'namespace' | 'alias' | 'function' | 'const'; +type PhpImportedSymbolKind = 'type' | 'function' | 'const'; interface PhpImportSpec { readonly kind: PhpImportKind; + readonly symbolKind: PhpImportedSymbolKind; /** Full backslash-separated path (backslashes intact): `Foo\Bar\Baz`. */ readonly source: string; /** Local binding name — last source segment for plain imports, the @@ -119,6 +121,7 @@ function parseUseClause(clause: SyntaxNode, qualifier: PhpImportKind): PhpImport if (alias !== '') { return { kind: 'alias', + symbolKind: symbolKindFor(qualifier), source, name: alias, alias, @@ -130,6 +133,7 @@ function parseUseClause(clause: SyntaxNode, qualifier: PhpImportKind): PhpImport return { kind: qualifier, + symbolKind: symbolKindFor(qualifier), source, name: lastSegment(source), atNode: clause, @@ -214,6 +218,7 @@ function parseInnerClause( if (alias !== '') { return { kind: 'alias', + symbolKind: symbolKindFor(qualifier), source, name: alias, alias, @@ -225,6 +230,7 @@ function parseInnerClause( return { kind: qualifier, + symbolKind: symbolKindFor(qualifier), source, name: lastSegment(innerPath), atNode: clause, @@ -237,6 +243,7 @@ function buildImportMatch(stmtNode: SyntaxNode, spec: PhpImportSpec): CaptureMat const m: Record = { '@import.statement': nodeToCapture('@import.statement', stmtNode), '@import.kind': syntheticCapture('@import.kind', spec.atNode, spec.kind), + '@import.symbol-kind': syntheticCapture('@import.symbol-kind', spec.atNode, spec.symbolKind), '@import.source': syntheticCapture('@import.source', spec.atNode, spec.source), '@import.name': syntheticCapture('@import.name', spec.atNode, spec.name), }; @@ -254,6 +261,12 @@ function lastSegment(path: string): string { return parts[parts.length - 1] ?? path; } +function symbolKindFor(kind: PhpImportKind): PhpImportedSymbolKind { + if (kind === 'function') return 'function'; + if (kind === 'const') return 'const'; + return 'type'; +} + /** Find the first named child with a given node type. */ function findNamedChild(node: SyntaxNode, type: string): SyntaxNode | null { for (let i = 0; i < node.namedChildCount; i++) { diff --git a/gitnexus/src/core/ingestion/languages/php/import-target.ts b/gitnexus/src/core/ingestion/languages/php/import-target.ts index ebf7938b32..1449bacb39 100644 --- a/gitnexus/src/core/ingestion/languages/php/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/php/import-target.ts @@ -16,6 +16,7 @@ */ import type { ParsedImport, WorkspaceIndex } from 'gitnexus-shared'; +import type { ImportResolutionContext } from '../../scope-resolution/contract/scope-resolver.js'; import { resolvePhpImportInternal } from '../../import-resolvers/php.js'; import type { ComposerConfig } from '../../language-config.js'; import { readFileSync } from 'node:fs'; @@ -117,6 +118,7 @@ export function resolvePhpImportTargetInternal( _fromFile: string, allFilePaths: ReadonlySet, resolutionConfig?: unknown, + context?: ImportResolutionContext, ): string | null { if (targetRaw === '') return null; @@ -129,7 +131,7 @@ export function resolvePhpImportTargetInternal( const normalizedFileList = [...allFiles].map((f) => f.replace(/\\/g, '/')); const allFileList = [...allFiles]; - return resolvePhpImportInternal( + const resolved = resolvePhpImportInternal( targetRaw, composerConfig, allFiles, @@ -137,4 +139,32 @@ export function resolvePhpImportTargetInternal( allFileList, undefined, ); + + const parsedImport = context?.parsedImport; + const symbolKind = + parsedImport?.kind === 'named' || parsedImport?.kind === 'alias' + ? parsedImport.importedSymbolKind + : undefined; + if (resolved === null || (symbolKind !== 'function' && symbolKind !== 'const')) return resolved; + + const importedName = targetRaw.replace(/\\/g, '/').split('/').filter(Boolean).at(-1); + if (importedName === undefined) return resolved; + + const normalizedResolved = resolved.replace(/\\/g, '/'); + const resolvedDirectory = normalizedResolved.slice(0, normalizedResolved.lastIndexOf('/') + 1); + const expectedType = symbolKind === 'function' ? 'Function' : 'Variable'; + const declaringFiles = context.parsedFiles.filter((parsed) => { + const normalizedPath = parsed.filePath.replace(/\\/g, '/'); + if (!normalizedPath.startsWith(resolvedDirectory)) return false; + if (normalizedPath.slice(resolvedDirectory.length).includes('/')) return false; + + return parsed.localDefs.some((def) => { + if (def.type !== expectedType) return false; + const simpleName = (def.qualifiedName ?? '').split(/[\\.]/).at(-1); + return simpleName === importedName; + }); + }); + + if (declaringFiles.length > 1) return null; + return declaringFiles.length === 1 ? declaringFiles[0].filePath : resolved; } diff --git a/gitnexus/src/core/ingestion/languages/php/interpret.ts b/gitnexus/src/core/ingestion/languages/php/interpret.ts index 8aa07a7367..7c920f9ef9 100644 --- a/gitnexus/src/core/ingestion/languages/php/interpret.ts +++ b/gitnexus/src/core/ingestion/languages/php/interpret.ts @@ -20,12 +20,19 @@ export function interpretPhpImport(captures: CaptureMatch): ParsedImport | null const sourceCap = captures['@import.source']; const nameCap = captures['@import.name']; const aliasCap = captures['@import.alias']; + const symbolKindCap = captures['@import.symbol-kind']; const kind = kindCap?.text; if (kind === undefined || sourceCap === undefined) return null; const source = sourceCap.text.trim(); if (source === '') return null; + const importedSymbolKind = + symbolKindCap?.text === 'function' || symbolKindCap?.text === 'const' + ? symbolKindCap.text + : kind === 'function' || kind === 'const' + ? kind + : 'type'; switch (kind) { case 'namespace': { @@ -39,6 +46,7 @@ export function interpretPhpImport(captures: CaptureMatch): ParsedImport | null localName, importedName: localName, targetRaw: source, + importedSymbolKind, }; } case 'alias': { @@ -53,6 +61,7 @@ export function interpretPhpImport(captures: CaptureMatch): ParsedImport | null importedName, alias, targetRaw: source, + importedSymbolKind, }; } case 'function': { @@ -64,6 +73,7 @@ export function interpretPhpImport(captures: CaptureMatch): ParsedImport | null localName, importedName: localName, targetRaw: source, + importedSymbolKind, }; } case 'const': { @@ -74,6 +84,7 @@ export function interpretPhpImport(captures: CaptureMatch): ParsedImport | null localName, importedName: localName, targetRaw: source, + importedSymbolKind, }; } default: diff --git a/gitnexus/src/core/ingestion/languages/php/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/php/scope-resolver.ts index 8b1fcf41b0..2b775e0e3a 100644 --- a/gitnexus/src/core/ingestion/languages/php/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/php/scope-resolver.ts @@ -354,8 +354,8 @@ const phpScopeResolver: ScopeResolver = { languageProvider: phpProvider, importEdgeReason: 'php-scope: use', - resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig) => - resolvePhpImportTargetInternal(targetRaw, fromFile, allFilePaths, resolutionConfig), + resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig, context) => + resolvePhpImportTargetInternal(targetRaw, fromFile, allFilePaths, resolutionConfig, context), loadResolutionConfig: (repoPath) => loadPhpComposerConfig(repoPath), diff --git a/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts b/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts index d10e67da91..e75b793d79 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts @@ -273,6 +273,7 @@ import type { Callsite, ConstraintContext, ParsedFile, + ParsedImport, ReferenceSite, ScopeId, SupportedLanguages, @@ -302,6 +303,11 @@ export type ReceiverMemberResolution = | { readonly kind: 'resolved'; readonly definition: SymbolDefinition } | { readonly kind: 'ambiguous'; readonly candidateIds: readonly string[] }; +export interface ImportResolutionContext { + readonly parsedFiles: readonly ParsedFile[]; + readonly parsedImport?: ParsedImport; +} + /** Re-exported for ScopeResolver consumers — same shape as * `RegistryProviders.constraintCompatibility`'s third parameter. */ export type { ConstraintContext } from 'gitnexus-shared'; @@ -340,12 +346,18 @@ export interface ScopeResolver { * orchestrator). TypeScript uses this to thread `tsconfig.json` path * aliases through to the standard resolver. Languages that don't * need any extra config ignore the parameter. + * + * `context.parsedFiles` is the complete, read-only language workspace. It is + * optional so resolvers that only need paths retain their existing shape. + * `context.parsedImport` is the exact import being finalized. PHP uses both + * when a PSR-4 import names a function instead of a file. */ resolveImportTarget( targetRaw: string, fromFile: string, allFilePaths: ReadonlySet, resolutionConfig?: unknown, + context?: ImportResolutionContext, ): string | readonly string[] | null; /** diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts index 62cb0083eb..5f11120b2b 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts @@ -561,8 +561,11 @@ export function runScopeResolution( const resolutionConfig = input.resolutionConfig; const finalized = finalizeScopeModel(parsedFiles, { hooks: { - resolveImportTarget: (targetRaw, fromFile) => - provider.resolveImportTarget(targetRaw, fromFile, allFilePaths, resolutionConfig), + resolveImportTarget: (targetRaw, fromFile, _workspaceIndex, parsedImport) => + provider.resolveImportTarget(targetRaw, fromFile, allFilePaths, resolutionConfig, { + parsedFiles, + parsedImport, + }), expandsWildcardTo: (targetModuleScope) => provider.expandsWildcardTo?.(targetModuleScope, parsedFiles) ?? [], mergeBindings: (existing, incoming, scopeId) => diff --git a/gitnexus/test/integration/resolvers/php.test.ts b/gitnexus/test/integration/resolvers/php.test.ts index 73b8f5babf..d25d7da67d 100644 --- a/gitnexus/test/integration/resolvers/php.test.ts +++ b/gitnexus/test/integration/resolvers/php.test.ts @@ -1621,6 +1621,11 @@ describe('PHP cross-file binding propagation', () => { (e) => e.sourceFilePath.includes('Main') && e.targetFilePath.includes('UserFactory'), ); expect(edge).toBeDefined(); + + const unrelatedEdge = imports.find( + (e) => e.sourceFilePath.includes('Main') && e.targetFilePath.endsWith('/Models/User.php'), + ); + expect(unrelatedEdge).toBeUndefined(); }); it('resolves $u->save() in run() to User#save via cross-file return type propagation', () => { From 90c8383d163d8a8014aec6aad19a08e0880c0e06 Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 12:51:20 +0700 Subject: [PATCH 15/16] test(bench): rebaseline PHP import capture shape --- gitnexus/bench/scope-capture/baselines.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitnexus/bench/scope-capture/baselines.json b/gitnexus/bench/scope-capture/baselines.json index d3383ae846..6ec633ac32 100644 --- a/gitnexus/bench/scope-capture/baselines.json +++ b/gitnexus/bench/scope-capture/baselines.json @@ -39,9 +39,9 @@ "_note": "PR #1934: F66/F68 let-binding pattern narrowing; F71 union (Struct-labeled, now materialized via legacy @definition.struct + resolvable); F72 macro FULLY WIRED \u2014 @declaration.macro/@reference.macro + MacroRegistry \u2192 USES edges to Macro nodes (never a same-named fn). + rust-macro / rust-union fixtures and merged with origin/main #1975 rust-scoped-impl; fingerprint re-baselined (scaling ~0.99, fixture_count 126). #1992: + rust-nested-tail-collision-generic and rust-generic-impl-same-method-name (F3) fixtures \u2014 pure fixture-corpus drift, no scope-extractor change; fixture_count 127->129, fingerprint 56ffc1c0->b00aea0f." }, "php": { - "fingerprint": "bc2c27c5ba26d5aea61142a2a99fb772222f5b969205260eb7a71b4c0bd73cdb", + "fingerprint": "31c9e3f3cb7094a2bf9021cf9db859036e002f8b44605cd993b470fc600e97cb", "scaling_budget": 1.5, - "_rebaselined": "#1956: heritage-bearing scale source (class extends Base + use trait); both forms gated at scale; linear (~1.04).", + "_rebaselined": "#1956: heritage-bearing scale source (class extends Base + use trait); both forms gated at scale; linear (~1.04). | #95: PHP imports carry a symbol-kind capture so function/constant imports resolve by declaring file; capture shape changes, scaling remains linear (~1.04).", "_note": "PR #1931: F53 import multi-clause, F54 enum_case, F55 anonymous_class \u2014 fixture count 138\u2192140, fingerprint drift expected." }, "ruby": { From 59ad8d906fea9858d6f5e31fbd2d5c90bbc20b8a Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 14 Jul 2026 13:12:11 +0700 Subject: [PATCH 16/16] fix(php): resolve symbol-named PSR-4 imports --- .../ingestion/languages/php/import-target.ts | 88 ++++++++++++++--- .../expected-captures.json | 62 ++++++------ .../php/php-import-target.test.ts | 94 +++++++++++++++++++ 3 files changed, 201 insertions(+), 43 deletions(-) create mode 100644 gitnexus/test/unit/scope-resolution/php/php-import-target.test.ts diff --git a/gitnexus/src/core/ingestion/languages/php/import-target.ts b/gitnexus/src/core/ingestion/languages/php/import-target.ts index 1449bacb39..25d3e53091 100644 --- a/gitnexus/src/core/ingestion/languages/php/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/php/import-target.ts @@ -27,6 +27,60 @@ export interface PhpResolveContext { readonly allFilePaths: ReadonlySet; } +function normalizePhpPath(value: string): string { + return value.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+$/, ''); +} + +function namespaceDirectories( + targetRaw: string, + composerConfig: ComposerConfig | null, + resolved: string | null, +): string[] { + const directories = new Set(); + if (resolved !== null) { + const normalizedResolved = normalizePhpPath(resolved); + const separator = normalizedResolved.lastIndexOf('/'); + if (separator >= 0) directories.add(normalizedResolved.slice(0, separator)); + } + + if (composerConfig === null) return [...directories]; + + const normalizedTarget = normalizePhpPath(targetRaw); + const mappings = [...composerConfig.psr4.entries()].sort((left, right) => { + const lengthDifference = right[0].length - left[0].length; + return lengthDifference !== 0 ? lengthDifference : left[0].localeCompare(right[0]); + }); + for (const [namespacePrefix, directoryPrefix] of mappings) { + const normalizedPrefix = normalizePhpPath(namespacePrefix); + if ( + normalizedTarget !== normalizedPrefix && + !normalizedTarget.startsWith(`${normalizedPrefix}/`) + ) { + continue; + } + + const remainder = normalizedTarget.slice(normalizedPrefix.length).replace(/^\//, ''); + const separator = remainder.lastIndexOf('/'); + const relativeNamespace = separator >= 0 ? remainder.slice(0, separator) : ''; + directories.add( + normalizePhpPath( + relativeNamespace === '' ? directoryPrefix : `${directoryPrefix}/${relativeNamespace}`, + ), + ); + break; + } + return [...directories]; +} + +function isDirectChild(filePath: string, directory: string): boolean { + const normalizedPath = normalizePhpPath(filePath); + const separator = normalizedPath.lastIndexOf('/'); + if (separator < 0) return directory === ''; + const parent = normalizedPath.slice(0, separator); + const normalizedDirectory = normalizePhpPath(directory); + return parent === normalizedDirectory || parent.endsWith(`/${normalizedDirectory}`); +} + // ─── loadResolutionConfig ────────────────────────────────────────────────── /** @@ -145,26 +199,36 @@ export function resolvePhpImportTargetInternal( parsedImport?.kind === 'named' || parsedImport?.kind === 'alias' ? parsedImport.importedSymbolKind : undefined; - if (resolved === null || (symbolKind !== 'function' && symbolKind !== 'const')) return resolved; + if ( + context === undefined || + parsedImport === undefined || + (symbolKind !== 'function' && symbolKind !== 'const') + ) { + return resolved; + } const importedName = targetRaw.replace(/\\/g, '/').split('/').filter(Boolean).at(-1); if (importedName === undefined) return resolved; - const normalizedResolved = resolved.replace(/\\/g, '/'); - const resolvedDirectory = normalizedResolved.slice(0, normalizedResolved.lastIndexOf('/') + 1); + const directories = namespaceDirectories(targetRaw, composerConfig, resolved); + const candidateFiles = context.parsedFiles.filter((parsed) => + directories.some((directory) => isDirectChild(parsed.filePath, directory)), + ); const expectedType = symbolKind === 'function' ? 'Function' : 'Variable'; - const declaringFiles = context.parsedFiles.filter((parsed) => { - const normalizedPath = parsed.filePath.replace(/\\/g, '/'); - if (!normalizedPath.startsWith(resolvedDirectory)) return false; - if (normalizedPath.slice(resolvedDirectory.length).includes('/')) return false; - - return parsed.localDefs.some((def) => { + const declaringFiles = candidateFiles.filter((parsed) => + parsed.localDefs.some((def) => { if (def.type !== expectedType) return false; const simpleName = (def.qualifiedName ?? '').split(/[\\.]/).at(-1); return simpleName === importedName; - }); - }); + }), + ); if (declaringFiles.length > 1) return null; - return declaringFiles.length === 1 ? declaringFiles[0].filePath : resolved; + if (declaringFiles.length === 1) return declaringFiles[0].filePath; + + // PHP constants are not currently emitted as local definitions. A single + // file in the namespace directory is still unambiguous; multiple files must + // fail closed rather than inheriting Set iteration order. + if (symbolKind === 'const' && candidateFiles.length === 1) return candidateFiles[0].filePath; + return resolved; } diff --git a/gitnexus/test/fixtures/php-captures-golden/expected-captures.json b/gitnexus/test/fixtures/php-captures-golden/expected-captures.json index ecbe428c8a..06ba547053 100644 --- a/gitnexus/test/fixtures/php-captures-golden/expected-captures.json +++ b/gitnexus/test/fixtures/php-captures-golden/expected-captures.json @@ -5,11 +5,11 @@ }, "php-abstract-dispatch/src/Repositories/SqlRepository.php": { "captureGroups": 14, - "digest": "5905bb450b29d4e186d74b50f70f07a54c8e0d4b8c3748c998c1579e72283215" + "digest": "584da0b8d38ba3b2e45513e24ee17e0366bcfe48e353c274daed367250a0ccd9" }, "php-abstract-dispatch/src/app.php": { "captureGroups": 10, - "digest": "52ce761f1d53a56034124fa5e674863bce9092c1b523f6d71139c4f335e8b9f0" + "digest": "af85999f2ddf2bf718cc419553efd09425919835460bd6739277cc570cb2d3c4" }, "php-alias-imports/app/Models/Repo.php": { "captureGroups": 14, @@ -21,7 +21,7 @@ }, "php-alias-imports/app/Services/Main.php": { "captureGroups": 15, - "digest": "bf8e1b8079956e7ef9ebfcb3ca38f547e9760da5c83856e93163efe6961067b9" + "digest": "685798f8164a494d7dc794cf6076ff5ccade0e79fe570947a824be0099011739" }, "php-ambiguous/app/Models/Dispatchable.php": { "captureGroups": 6, @@ -41,7 +41,7 @@ }, "php-ambiguous/app/Services/UserHandler.php": { "captureGroups": 12, - "digest": "1c47e7020939a6cd6bb18f3732b51a16f4c75ded0fe9883a87d6706a2c624d1f" + "digest": "87bbd866d8748b7cd4932aea209abc1e17f9eca518c0a5bb0dc9572cc5cd95be" }, "php-anonymous-class/anon.php": { "captureGroups": 5, @@ -61,15 +61,15 @@ }, "php-app/app/Models/BaseModel.php": { "captureGroups": 18, - "digest": "be7ca5be7e28417afdef2e45c8cfed9e04594b341b0d676e7a00fdd84c94e42a" + "digest": "7d5a013f34f909846e3a91cef4b3daa8638034e5c6939d90ccd7dae34c8f9272" }, "php-app/app/Models/User.php": { "captureGroups": 27, - "digest": "b6b08be1af66cbd757cfd715389725952c0e2a8e5e910ed715594c0b07570a37" + "digest": "718b9ba0238e2139ff34027d68cb135c4698a2fc401a4f85200864b1e27a9a78" }, "php-app/app/Services/UserService.php": { "captureGroups": 38, - "digest": "80b8557e183052f25222cfda8879e08b1613752001f58670ce7b3dbf4b8bee28" + "digest": "a911c5af5042738b1ca9aeb71dad06f4918d8c41e84b6a27ffaa0eadfe56ca22" }, "php-app/app/Traits/HasTimestamps.php": { "captureGroups": 11, @@ -89,7 +89,7 @@ }, "php-assignment-chain/app/Services/AppService.php": { "captureGroups": 15, - "digest": "36c5358d7f724cc0906e087828c86d9a29fe0629e2bad6105967fe8d671aa6fe" + "digest": "0e9a9ef52a987dbd12c9478a75d5019c2a0059ab4b862924597d2ecd32f03aa7" }, "php-call-result-binding/App.php": { "captureGroups": 23, @@ -97,7 +97,7 @@ }, "php-calls/app/Services/UserService.php": { "captureGroups": 7, - "digest": "94f1cdfb0c444dc9e8116d1bbf824dc88584046539b2cbc48297729eb49e41a1" + "digest": "84e2e65cbb4026ef9e67ac2710d1304c8dffe335f2b1b1cb70123157a04acf50" }, "php-calls/app/Utils/OneArg/log.php": { "captureGroups": 5, @@ -109,7 +109,7 @@ }, "php-child-extends-parent/src/App.php": { "captureGroups": 11, - "digest": "0c8a7c7edaa20009b71f22fef98230119a4738d2a21c8b88d64047c3c932fd36" + "digest": "6e8b1d8f2e9c02eebcf8e2b6cacc15dfbd48c9fc03011c03d70afd265c38f3dc" }, "php-child-extends-parent/src/Child.php": { "captureGroups": 5, @@ -125,7 +125,7 @@ }, "php-constructor-calls/app.php": { "captureGroups": 8, - "digest": "23ef0f05446126892e598872a0b3c3d2f96e3b0dc50e301f1fa784a56ebc81d4" + "digest": "0f3e6cf60248ae02ca0a744d3fa4f67b2d8421858c29243c4fcaf60763c7ac3d" }, "php-constructor-promotion-fields/Models.php": { "captureGroups": 22, @@ -145,7 +145,7 @@ }, "php-constructor-type-inference/app/Services/AppService.php": { "captureGroups": 15, - "digest": "26181127fe9e9bf04431a7cc801624bde9dd284049627bb7ca0a4c9234141eec" + "digest": "5aee8297c1f3e032523fe871571ee4410422b7ee932b259e9f36b0cab4d79c8f" }, "php-coverage/enum-and-anon.php": { "captureGroups": 9, @@ -153,7 +153,7 @@ }, "php-coverage/multi-import.php": { "captureGroups": 7, - "digest": "4d72fbfba40f2e083aa55d1c48bb500aeb7dd615de18b48b00eaca5b26a2001e" + "digest": "e3746c5f7a68dc4dd9d658159077eb272573e27dfc29fedeadfbd6ab7ef35a10" }, "php-deep-field-chain/Models.php": { "captureGroups": 26, @@ -245,7 +245,7 @@ }, "php-fqn-cross-namespace/app/Services/Service.php": { "captureGroups": 15, - "digest": "94a2bec57ccde7aa663ce2027c7f36151d18ee257830663365bc14f9d4d5f703" + "digest": "e711548813d38a0db268d89a7edff1d31bbaa9ff0399898e09f8e93bdbc1c785" }, "php-grandparent-resolution/app/Models/A.php": { "captureGroups": 9, @@ -265,7 +265,7 @@ }, "php-grandparent-resolution/app/Services/App.php": { "captureGroups": 12, - "digest": "5d0c3524e1ca41ee57bd05fd0a2831804598fca8079ea4ece38045d9c4bb1113" + "digest": "7273ba524f587c01ca3ba9ededfdfef635c20fd9c6ade95225e81aff07ace6a6" }, "php-grouped-imports/app/Models/Repo.php": { "captureGroups": 7, @@ -277,11 +277,11 @@ }, "php-grouped-imports/app/Services/Main.php": { "captureGroups": 15, - "digest": "b315d87f85f0899ed3883ec529b86a57f2d1a88d41b2b615e707307e2889c049" + "digest": "aef9ddf30722a053664637b7fefdfcdd541baf09b24e65a3a0d80fe920189828" }, "php-local-shadow/app/Services/Main.php": { "captureGroups": 9, - "digest": "b4b3e35399501d98521dc9d9281a4e5c94449b580cc6cb9b3ed4187e79ee2c07" + "digest": "52549cd82f7e2e69fb8bb81bca4e5e3487e216f4305023b77579de92374b9ee3" }, "php-local-shadow/app/Utils/Logger.php": { "captureGroups": 5, @@ -293,7 +293,7 @@ }, "php-member-calls/app/Services/UserService.php": { "captureGroups": 11, - "digest": "6f6d5d34edd1cd4e32ea77db7e4b07b73de9ca09bb658123455820af8228d0cc" + "digest": "dfe3ef69a8d6dbd95cf5a6bdc754914973b7208ad0c9575bf2595a0f84ac22cf" }, "php-method-chain-binding/App.php": { "captureGroups": 48, @@ -309,7 +309,7 @@ }, "php-method-enrichment/src/app.php": { "captureGroups": 11, - "digest": "52791f6945c8c4ae083b816bd3af239bce44f0e97cbf80cdc119f4f366015138" + "digest": "ebfa35aa3eb065977f922ae72f353d5bf31dd49956c3e9aaef349718e555b7a7" }, "php-mro-arity-mismatch/app/Models/ChildModel.php": { "captureGroups": 16, @@ -325,11 +325,11 @@ }, "php-mro-arity-mismatch/app/Services/Caller.php": { "captureGroups": 22, - "digest": "d51fdbcf226f31f9220a506cceb9f538642ff99d238a1828b6b43fce6193f664" + "digest": "20f76e10fc598152597ba8df71deccb698761938bfde8cf7c3437044c8fcdcf1" }, "php-namespace-fallback-isolation/src/App/Caller.php": { "captureGroups": 13, - "digest": "d5040d068fd8227388da7f25cc471f154adb37cd6bbfb78b5cac00f44bf45734" + "digest": "48f132404e2d00efbf49302a69e5ab31540eb0eac71cd2fca43a8a0eb4533d98" }, "php-namespace-fallback-isolation/src/App/Utils/Caller.php": { "captureGroups": 8, @@ -353,7 +353,7 @@ }, "php-nullable-receiver/app/Services/AppService.php": { "captureGroups": 13, - "digest": "cea6ae3f9e32a3e0448a279034bcf039b1d5af6334ab1b1ae43d9c55b6ca094d" + "digest": "52e6db35e8fa4174ce698fec802b9b034dcda8f49fd7ab98cd9e884a6bd9777f" }, "php-overload-dispatch/src/Services/Formatter.php": { "captureGroups": 7, @@ -365,7 +365,7 @@ }, "php-overload-dispatch/src/app.php": { "captureGroups": 10, - "digest": "d11621fbaec9f5015e61f35b5c2747b9f2da090a346adc5fcc10f191af61f048" + "digest": "d29646b0c2d1e072ee0265acf641f99f3c443b57859c2582b782d5a485bfb089" }, "php-parent-resolution/app/Models/BaseModel.php": { "captureGroups": 7, @@ -421,7 +421,7 @@ }, "php-receiver-resolution/app/Services/AppService.php": { "captureGroups": 13, - "digest": "59783c7af75e9075f00f425984ca1ba7a4c556bb0301e2eb2c3fdcaa94cd547f" + "digest": "5e92226af09405c7fc4a02d3aeafcedc8462f3e525b8f67b8509f38091488505" }, "php-response-shapes/api/items.php": { "captureGroups": 11, @@ -445,7 +445,7 @@ }, "php-return-type/app/Services/UserService.php": { "captureGroups": 17, - "digest": "fcc2d78ac4bdde1ac5179e6623a35299465bcbf9bb016375100bcb636624d043" + "digest": "ac4851815d7a450ab92f7f68ccadabf219d2534b99ef1b65ced0a344188f2a69" }, "php-self-this-resolution/app/Models/Repo.php": { "captureGroups": 7, @@ -481,7 +481,7 @@ }, "php-transitive-traits/app/Models/Consumer.php": { "captureGroups": 18, - "digest": "c4524f4fa18f6e7f0fd76a11920a6a65ab547ab4cf0fa5799b42b7678e14db08" + "digest": "19b4d42452d84a4da0d1f05a03880561cee4d77ae0efd382351933875787ad96" }, "php-transitive-traits/app/Traits/TraitA.php": { "captureGroups": 8, @@ -501,7 +501,7 @@ }, "php-typed-properties/app/Services/UserService.php": { "captureGroups": 12, - "digest": "2ec84482a3e2332f3f15ebb3f2d8d01d1d56e62da256127ece571a8c2e0c070c" + "digest": "9f0540a4f7f322ee96dad4437c4e41e087f29a6993d2659fc48bd26e3277e963" }, "php-typed-property-dedup/app/Models/UserRepo.php": { "captureGroups": 7, @@ -509,7 +509,7 @@ }, "php-typed-property-dedup/app/Services/Mixed.php": { "captureGroups": 14, - "digest": "9f4ba6bd183c20acaad547b6a713e80498eb70eaabaa7b416650bb64098bde0d" + "digest": "97d09b46f6e66bef700f4686e5988c98372853203885b9a2d99dafce6fdc8343" }, "php-unresolved-receiver-arity/app/Models/Handler.php": { "captureGroups": 21, @@ -529,7 +529,7 @@ }, "php-use-function-const/app/Services/Calculator.php": { "captureGroups": 15, - "digest": "d6996da68f917c3ae6b52b530d166e8266f5053ce6c9b76e1643cb61edcafa38" + "digest": "b2206861c8550b6d2c7610ff167c42f4236c7249c87584b418e05f687233ad22" }, "php-use-function-const/app/Utils/helpers.php": { "captureGroups": 6, @@ -537,7 +537,7 @@ }, "php-variadic-arity-minimum/app/Services/Caller.php": { "captureGroups": 29, - "digest": "d5669fe609abae6b7db19c15c0cb795859e0b4b0570ae83fbe80a392f3775ca8" + "digest": "4ac7be9ff21c52b76630cdcfde0eb41af43e126c2de5db327e9f82c46c3aabbc" }, "php-variadic-arity-minimum/app/Utils/Logger.php": { "captureGroups": 18, @@ -545,7 +545,7 @@ }, "php-variadic-resolution/app/Services/AppService.php": { "captureGroups": 9, - "digest": "8b5298358fba8f578b2470f9d93ffbc1089d1ef3208c1142185880b4dc174751" + "digest": "6591007d2f4ba85b25a59c11c3ae200452fbad23cacb1ebc04291a074b534dd8" }, "php-variadic-resolution/app/Utils/Logger.php": { "captureGroups": 7, diff --git a/gitnexus/test/unit/scope-resolution/php/php-import-target.test.ts b/gitnexus/test/unit/scope-resolution/php/php-import-target.test.ts new file mode 100644 index 0000000000..b3a7393648 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/php/php-import-target.test.ts @@ -0,0 +1,94 @@ +import type { ParsedFile, ParsedImport, SymbolDefinition } from 'gitnexus-shared'; +import { describe, expect, it } from 'vitest'; + +import type { ComposerConfig } from '../../../../src/core/ingestion/language-config.js'; +import { resolvePhpImportTargetInternal } from '../../../../src/core/ingestion/languages/php/import-target.js'; + +const composerConfig: ComposerConfig = { psr4: new Map([['App', 'app']]) }; + +function parsedFile(filePath: string, definitions: readonly SymbolDefinition[]): ParsedFile { + return { filePath, localDefs: definitions } as ParsedFile; +} + +function definition( + filePath: string, + type: SymbolDefinition['type'], + name: string, +): SymbolDefinition { + return { + nodeId: `def:${filePath}:${type}:${name}`, + filePath, + type, + qualifiedName: name, + }; +} + +const functionImport: ParsedImport = { + kind: 'named', + localName: 'getUser', + importedName: 'getUser', + targetRaw: 'App\\Models\\getUser', + importedSymbolKind: 'function', +}; + +describe('resolvePhpImportTargetInternal declaration selection', () => { + it('finds a unique function declaration when the symbol name is not a filename', () => { + const user = '/repo/app/Models/User.php'; + const factory = '/repo/app/Models/UserFactory.php'; + const parsedFiles = [ + parsedFile(user, [definition(user, 'Class', 'User')]), + parsedFile(factory, [definition(factory, 'Function', 'getUser')]), + ]; + + expect( + resolvePhpImportTargetInternal( + functionImport.targetRaw, + '/repo/app/Main.php', + new Set(parsedFiles.map((parsed) => parsed.filePath)), + composerConfig, + { parsedFiles, parsedImport: functionImport }, + ), + ).toBe(factory); + }); + + it('fails closed when the namespace has duplicate function declarations', () => { + const first = '/repo/app/Models/First.php'; + const second = '/repo/app/Models/Second.php'; + const parsedFiles = [ + parsedFile(first, [definition(first, 'Function', 'getUser')]), + parsedFile(second, [definition(second, 'Function', 'getUser')]), + ]; + + expect( + resolvePhpImportTargetInternal( + functionImport.targetRaw, + '/repo/app/Main.php', + new Set(parsedFiles.map((parsed) => parsed.filePath)), + composerConfig, + { parsedFiles, parsedImport: functionImport }, + ), + ).toBeNull(); + }); + + it('resolves a constant only when its namespace directory has one candidate file', () => { + const constants = '/repo/app/Config/constants.php'; + const parsedFiles = [parsedFile(constants, [])]; + const parsedImport: ParsedImport = { + kind: 'named', + localName: 'MAX_RETRIES', + importedName: 'MAX_RETRIES', + targetRaw: 'App\\Config\\MAX_RETRIES', + importedSymbolKind: 'const', + }; + + expect( + resolvePhpImportTargetInternal( + parsedImport.targetRaw, + '/repo/app/Main.php', + new Set([constants]), + composerConfig, + { parsedFiles, parsedImport }, + ), + ).toBe(constants); + }); +});