From cfc7cf2c93cdb1a66cbaf1082bdafe944df028e8 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Tue, 14 Jul 2026 00:25:34 +0100 Subject: [PATCH 01/42] fix(cache): resolve FileSlice via unit_path in checkpoint allowlist (#1870) The #1757 batch-scoping followup built the per-chunk allowlist by reading FileSlice.rel, which does not exist (a FileSlice carries its parent file in .path). So every chunk containing a sliced oversized document leaked the FileSlice object into the allowlist, save_semantic_cache raised TypeError on Path(FileSlice), and the best-effort except swallowed it: extraction finished but those chunks were never checkpointed, so a re-run or a crash/rate-limit resume re-billed them. Resolve each unit through the canonical unit_path() helper so a slice maps to its parent file. Adds a regression test that slices a real oversized .md and asserts the checkpoint writes without swallowing a TypeError. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 ++++ graphify/llm.py | 9 ++++++--- pyproject.toml | 2 +- tests/test_chunking.py | 38 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 076d602c8..228a5b3c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.9.16 (unreleased) + +- Fix: the incremental semantic-cache checkpoint no longer fails on oversized (sliced) documents (#1870). The 0.9.14 batch-scoping fix built its per-chunk allowlist by reading `FileSlice.rel`, an attribute that does not exist (a `FileSlice` carries its parent file in `.path`), so every chunk containing a sliced document leaked the `FileSlice` object into the allowlist, `save_semantic_cache` raised `TypeError`, and the best-effort handler swallowed it: extraction still finished but those chunks were never checkpointed, so a re-run or a run resumed after a crash/rate-limit re-billed them. The allowlist now resolves each unit through `unit_path`, so a slice maps to its parent file and the checkpoint writes as intended. + ## 0.9.15 (2026-07-13) - Fix: detection now honors nested `.gitignore`/`.graphifyignore` files below the scan root, not just those at the scan root and above (#1847, thanks @Mohak-Agrawal). git applies a `.gitignore` to everything under its own directory, but graphify only loaded ignore files from the VCS-root-down-to-scan-root chain — so a `vendor/sub/.gitignore` deeper in the tree was never read and its exclusions leaked into the graph. Each directory's own ignore files are now read during the walk and anchored to that directory, preserving last-match-wins precedence (nearer files win, including over `.git/info/exclude`) and the parent-exclusion rule. diff --git a/graphify/llm.py b/graphify/llm.py index 23b30204f..05e2186ea 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -1918,9 +1918,12 @@ def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: # (#1757). The model can attribute a node's source_file to another # corpus file; without this bound, that stray node would clobber the # other file's complete cache entry (or, with merge_existing, pollute - # it). A FileSlice reports its file via `.rel`; a bare Path is the - # relative source_file itself. - allowed = [getattr(item, "rel", None) or item for item in chunk] + # it). Use unit_path so a FileSlice (one slice of an oversized doc) + # resolves to its parent file; a bare Path passes through. (#1870: the + # old `.rel` attribute does not exist on FileSlice, so every sliced + # chunk leaked the FileSlice object into the allowlist and the write + # raised TypeError, silently defeating the checkpoint.) + allowed = [unit_path(item) for item in chunk] _scs( result.get("nodes", []), result.get("edges", []), diff --git a/pyproject.toml b/pyproject.toml index 1b2b818d1..3e5662a7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.9.15" +version = "0.9.16" description = "AI coding assistant skill (Claude Code, CodeBuddy, Codex, OpenCode, Kilo Code, Cursor, Gemini CLI, Aider, OpenClaw, Factory Droid, Trae, Hermes, Kiro, Pi, Devin CLI, Google Antigravity) - turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } diff --git a/tests/test_chunking.py b/tests/test_chunking.py index b39262adf..42b4651a5 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -326,6 +326,44 @@ def stray(chunk, **kwargs): assert a_cache and any(n["id"] == "a_ok" for n in a_cache["nodes"]) +def test_checkpoint_caches_sliced_document_chunks(tmp_path, capsys): + """#1870: the checkpoint's allowlist must resolve a FileSlice to its parent + path (via unit_path), not read a non-existent `.rel`. An oversized doc is + split into FileSlice units; before the fix each sliced chunk leaked the + FileSlice object into the allowlist, so save_semantic_cache raised TypeError, + the best-effort except swallowed it, and the slice was never checkpointed.""" + from graphify.llm import extract_corpus_parallel, expand_oversized_files, _FILE_CHAR_CAP + from graphify.file_slice import FileSlice + from graphify.cache import load_cached + + doc = tmp_path / "big.md" + doc.write_text("# Title\n" + ("word " * 12000) + "\n## Section\n" + ("more " * 12000)) + # sanity: the doc really does slice into FileSlice units + units = expand_oversized_files([doc], _FILE_CHAR_CAP) + assert len(units) > 1 and all(isinstance(u, FileSlice) for u in units) + + def sliced(chunk, **kwargs): + assert any(isinstance(c, FileSlice) for c in chunk) + return { + "nodes": [{"id": "big_title", "source_file": "big.md", "file_type": "document"}], + "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1, + } + + with patch("graphify.llm.extract_files_direct", side_effect=sliced): + extract_corpus_parallel( + [doc], backend="kimi", root=tmp_path, + token_budget=None, chunk_size=1, max_concurrency=1, + ) + + assert "incremental cache checkpoint failed" not in capsys.readouterr().err, ( + "checkpoint raised on a FileSlice chunk (#1870)" + ) + cached = load_cached(doc, tmp_path, kind="semantic") + assert cached and any(n["id"] == "big_title" for n in cached["nodes"]), ( + "sliced document was never checkpointed (#1870)" + ) + + def test_corpus_parallel_legacy_mode_when_token_budget_is_none(tmp_path): """token_budget=None should fall back to legacy fixed-count chunking.""" from graphify.llm import extract_corpus_parallel From 20405a84bf2489b7762dec48cbc95d4e050b790d Mon Sep 17 00:00:00 2001 From: thejesh Date: Mon, 13 Jul 2026 10:48:27 -0700 Subject: [PATCH 02/42] fix(dedup): report fuzzy count in summary when exact_merges is 0 (#1857) --- graphify/dedup.py | 13 +++++++++---- tests/test_dedup.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/graphify/dedup.py b/graphify/dedup.py index e0ddd2e3b..47c4eff8f 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -417,11 +417,16 @@ def deduplicate_entities( total = len(remap) msg = f"[graphify] Deduplicated {total} node(s)" + # Both counters are reported when non-zero. Previous form nested the fuzzy + # branch inside `if exact_merges`, silently dropping the fuzzy count on + # doc/semantic-heavy runs where Pass 1 finds nothing (#1857). + parts: list[str] = [] if exact_merges: - msg += f" ({exact_merges} exact" - if fuzzy_merges: - msg += f", {fuzzy_merges} fuzzy" - msg += ")" + parts.append(f"{exact_merges} exact") + if fuzzy_merges: + parts.append(f"{fuzzy_merges} fuzzy") + if parts: + msg += f" ({', '.join(parts)})" print(msg + ".", flush=True) deduped_nodes = [n for n in unique_nodes if n["id"] not in remap] diff --git a/tests/test_dedup.py b/tests/test_dedup.py index 267388e57..8d8028589 100644 --- a/tests/test_dedup.py +++ b/tests/test_dedup.py @@ -404,3 +404,39 @@ def test_same_id_same_source_file_no_warning(capsys): assert len(result_nodes) == 1 captured = capsys.readouterr() assert "WARNING" not in captured.err + + +# ── #1857: dedup summary log breakdown ──────────────────────────────────────── + +def test_dedup_summary_prints_fuzzy_count_when_no_exact_merges(capsys): + """A fuzzy-only run must still report the fuzzy count (#1857). + + Two long, high-entropy, non-code labels on different files. Pass 1 (exact, + same-file) finds nothing; Pass 2 (Jaro-Winkler cross-file) merges them. + Previously the fuzzy count was nested inside `if exact_merges`, so this + line printed as `Deduplicated 1 node(s).` with no breakdown. + """ + nodes = [ + {"id": "g1", "label": "GraphExtractor", "file_type": "concept", "source_file": "a.md"}, + {"id": "g2", "label": "Graph Extractor", "file_type": "concept", "source_file": "b.md"}, + ] + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) + assert len(result_nodes) == 1 + captured = capsys.readouterr() + assert "Deduplicated" in captured.out + assert "fuzzy" in captured.out + assert "exact" not in captured.out + + +def test_dedup_summary_still_reports_exact_only(capsys): + """Non-regression: an exact-only run still prints `(N exact)` and no fuzzy.""" + # Same file + same normalized label → Pass 1 exact merge. + nodes = [ + {"id": "u1", "label": "User Service", "file_type": "concept", "source_file": "svc.md"}, + {"id": "u2", "label": "user service", "file_type": "concept", "source_file": "svc.md"}, + ] + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) + assert len(result_nodes) == 1 + captured = capsys.readouterr() + assert "exact" in captured.out + assert "fuzzy" not in captured.out From e32be08c2a279be987873365f5c504a8864d8c38 Mon Sep 17 00:00:00 2001 From: thejesh Date: Mon, 13 Jul 2026 10:53:03 -0700 Subject: [PATCH 03/42] fix(detect): legacy-manifest branch must `!=`-compare mtime to catch reverse jumps (#1859) --- graphify/detect.py | 10 ++++++-- tests/test_detect.py | 61 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index 6b54d8913..5af06e381 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -1604,9 +1604,15 @@ def detect_incremental( except Exception: current_mtime = 0 - # Legacy manifest: plain float value — treat as ast_hash only + # Legacy manifest: plain float value stores only mtime. + # Compare with `!=` so backwards mtime motion (git checkout of an + # older commit, tarball restore, rsync --times) still triggers a + # re-extract; the previous `>` silently kept the stale cache and + # the graph drifted from disk (#1859). No stored hash means we + # cannot verify content — any mtime delta forces a re-extract, + # and the next save promotes the entry into the dict schema. if isinstance(stored, (int, float)): - changed = stored is None or current_mtime > stored + changed = current_mtime != stored elif isinstance(stored, dict): # Normalise legacy {mtime, hash} to new schema if "hash" in stored and "ast_hash" not in stored: diff --git a/tests/test_detect.py b/tests/test_detect.py index 37ffc6fb7..a229cc6bd 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -449,6 +449,67 @@ def test_detect_incremental_survives_dict_valued_mtime(tmp_path, monkeypatch): assert not any("mod.py" in f for f in result["unchanged_files"]["code"]) +def test_detect_incremental_legacy_float_reextracts_on_backwards_mtime(tmp_path, monkeypatch): + """Legacy float manifests must re-extract when mtime moves BACKWARDS (#1859). + + Pre-fix the legacy branch used `current_mtime > stored`, which silently kept + the cached entry after operations that restore older mtimes: `git checkout` + of an older commit, `tar -xf` restore, or `rsync --times`. The graph then + reflected the newer content while disk held the older content. The dict + branch has always used `!=`; this test pins the legacy branch to the same + contract. + """ + import json + + monkeypatch.chdir(tmp_path) + + src = tmp_path / "mod.py" + src.write_text("def old_content():\n return 1\n", encoding="utf-8") + current_mtime = os.stat(src).st_mtime + + manifest_dir = tmp_path / "graphify-out" + manifest_dir.mkdir() + manifest_path = str(manifest_dir / "manifest.json") + + # Legacy schema (pre-dict-migration): the value is a bare float mtime. + # Store a mtime FROM THE FUTURE, simulating a checkout of an older + # revision that restored the file to an earlier timestamp. + future_mtime = current_mtime + 3600 + legacy = {str(src.resolve()): future_mtime} + Path(manifest_path).write_text(json.dumps(legacy), encoding="utf-8") + + result = detect_incremental(tmp_path, manifest_path) + + assert any("mod.py" in f for f in result["new_files"]["code"]), ( + "backwards-moving mtime on a legacy manifest entry must trigger re-extract" + ) + assert not any("mod.py" in f for f in result["unchanged_files"]["code"]) + + +def test_detect_incremental_legacy_float_skips_when_mtime_matches(tmp_path, monkeypatch): + """Non-regression for the fix above: legacy float branch still skips when + the stored mtime equals the current mtime.""" + import json + + monkeypatch.chdir(tmp_path) + + src = tmp_path / "mod.py" + src.write_text("def stable():\n return 1\n", encoding="utf-8") + + manifest_dir = tmp_path / "graphify-out" + manifest_dir.mkdir() + manifest_path = str(manifest_dir / "manifest.json") + + # Legacy schema with the exact current mtime → no change → skip. + legacy = {str(src.resolve()): os.stat(src).st_mtime} + Path(manifest_path).write_text(json.dumps(legacy), encoding="utf-8") + + result = detect_incremental(tmp_path, manifest_path) + + assert not any("mod.py" in f for f in result["new_files"]["code"]) + assert any("mod.py" in f for f in result["unchanged_files"]["code"]) + + def test_classify_video_extensions(): """Video and audio file extensions should classify as VIDEO.""" from graphify.detect import FileType From 368407874f75c53a0596eb46c0c5af63a520c27b Mon Sep 17 00:00:00 2001 From: safishamsi Date: Tue, 14 Jul 2026 00:26:13 +0100 Subject: [PATCH 04/42] docs(changelog): record #1862 and #1860 in the unreleased 0.9.16 section Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 228a5b3c0..8b26df9bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.16 (unreleased) +- Fix: `detect_incremental` re-extracts a legacy-manifest file when its mtime moves backwards, not just forwards (#1859 / #1862, thanks @thejesh23). The legacy float-schema branch used a strict `current_mtime > stored`, so a file restored to an older timestamp (a `git checkout` of an older commit, a tarball restore, `rsync --times`) was treated as unchanged and never re-extracted, leaving the graph reflecting newer content than the corpus on disk. It now compares with `!=`, matching the dict-schema branch; with no stored hash to verify, any mtime delta forces a re-extract, and the next save promotes the entry to the hash-verified dict schema. + +- Fix: the dedup summary line reports the fuzzy-merge count even when there were no exact merges (#1857 / #1860, thanks @thejesh23). The fuzzy branch was nested inside `if exact_merges`, so a doc- or semantic-heavy run that merged only via the cross-file fuzzy pass printed a bare `Deduplicated N node(s).` with no breakdown. Both counts are now reported whenever non-zero. + - Fix: the incremental semantic-cache checkpoint no longer fails on oversized (sliced) documents (#1870). The 0.9.14 batch-scoping fix built its per-chunk allowlist by reading `FileSlice.rel`, an attribute that does not exist (a `FileSlice` carries its parent file in `.path`), so every chunk containing a sliced document leaked the `FileSlice` object into the allowlist, `save_semantic_cache` raised `TypeError`, and the best-effort handler swallowed it: extraction still finished but those chunks were never checkpointed, so a re-run or a run resumed after a crash/rate-limit re-billed them. The allowlist now resolves each unit through `unit_path`, so a slice maps to its parent file and the checkpoint writes as intended. ## 0.9.15 (2026-07-13) From fb4d45226e9ac3e7103c5f413d31b71c0f7de741 Mon Sep 17 00:00:00 2001 From: Alwyn93 Date: Tue, 14 Jul 2026 14:01:58 +0400 Subject: [PATCH 05/42] fix(detect): scope nested ignore patterns to their own subtree (#1873) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Non-anchored patterns from a nested .gitignore/.graphifyignore were matched against root-relative paths first, so a nested ignore file's patterns leaked outside its directory. In the wild, .hypothesis/.gitignore (a bare "*" auto-written by the hypothesis library) ignored the entire repository and detect() returned 0 files. Per gitignore semantics, patterns from A/.gitignore apply only to paths under A. Match every pattern against the path relative to its own anchor (the anchor dir itself exempt — an ignore file governs its directory's contents, not the directory), and skip patterns whose anchor does not contain the target. Regression introduced with nested-ignore support in 8a5287a (#1206). tests/test_detect.py: 143 passed, including the existing nested-ignore and nested-negation tests plus two new regressions. Fixes #1873 Co-Authored-By: Claude Fable 5 --- graphify/detect.py | 29 +++++++++++------------------ tests/test_detect.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index 5af06e381..6e9d283a5 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -955,25 +955,18 @@ def _matches(rel: str, p: str, anchored: bool) -> bool: if not p: continue + # gitignore semantics: patterns from A/.gitignore apply ONLY to paths + # under A. Matching non-anchored patterns against root-relative paths + # let e.g. .hypothesis/.gitignore's bare "*" ignore the ENTIRE repo + # (detect() returned 0 files). The anchor dir itself is exempt — an + # ignore file governs its directory's contents, not the directory. matched = False - if anchored: - try: - rel_anchor = str(target.relative_to(anchor)).replace(os.sep, "/") - matched = _matches(rel_anchor, p, anchored=True) - except ValueError: - pass - else: - try: - rel = str(target.relative_to(root)).replace(os.sep, "/") - matched = _matches(rel, p, anchored=False) - except ValueError: - pass - if not matched and anchor != root: - try: - rel_anchor = str(target.relative_to(anchor)).replace(os.sep, "/") - matched = _matches(rel_anchor, p, anchored=False) - except ValueError: - pass + try: + rel_anchor = str(target.relative_to(anchor)).replace(os.sep, "/") + except ValueError: + continue # target outside this pattern's anchor: cannot match + if rel_anchor != ".": + matched = _matches(rel_anchor, p, anchored=anchored) if matched: result = not negated # last match wins; ! flips to un-ignore diff --git a/tests/test_detect.py b/tests/test_detect.py index a229cc6bd..2c9160303 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -1815,3 +1815,35 @@ def test_detect_surfaces_unreadable_dir_instead_of_silent_skip(tmp_path, capsys) assert any(f.endswith("a.py") for f in code) # rest of tree still enumerated assert len(res["walk_errors"]) >= 1 assert "could not scan" in capsys.readouterr().err + + +def test_nested_gitignore_star_does_not_ignore_outside_its_dir(tmp_path): + """A nested .gitignore containing a bare `*` (auto-written by e.g. the + hypothesis library into .hypothesis/) must ignore ONLY that directory's + contents — matching it against root-relative paths ignored the entire + corpus (detect() returned 0 files on a real repo). Regression for #1873.""" + (tmp_path / "README.md").write_text("# hello") + (tmp_path / "main.py").write_text("x = 1") + hyp = tmp_path / ".hypothesis" + hyp.mkdir() + (hyp / ".gitignore").write_text("*\n") + (hyp / "cached.py").write_text("y = 2") + + result = detect(tmp_path) + + assert result["total_files"] == 2 # README.md + main.py survive; .hypothesis/* ignored + + +def test_nested_gitignore_patterns_still_apply_inside_their_dir(tmp_path): + """Counterpart guard: the anchor-scoped fix must not stop nested ignore + files from working WITHIN their own subtree.""" + (tmp_path / "main.py").write_text("x = 1") + sub = tmp_path / "sub" + sub.mkdir() + (sub / ".gitignore").write_text("*.log\n") + (sub / "keep.py").write_text("y = 2") + (sub / "noise.log").write_text("z") + + result = detect(tmp_path) + + assert result["total_files"] == 2 # main.py + sub/keep.py; sub/noise.log ignored From 40eae3cea2c88b2c3c206a72b4296e7a9bd8639a Mon Sep 17 00:00:00 2001 From: safishamsi Date: Tue, 14 Jul 2026 13:18:28 +0100 Subject: [PATCH 06/42] test(watch): pin update-with-nested-gitignore (#1880); changelog for #1873/#1880 #1880 is the update-layer symptom of the #1873/#1887 nested-ignore subtree-scoping regression (fixed in fb4d452, @Alwyn93): a nested broad `.gitignore` zeroed update's re-scan, so it built 0 nodes and the shrink-guard refused to overwrite. Adds a _rebuild_code regression test asserting update sees the real files (and still scopes the nested ignore correctly) so this can't recur at the update layer. Records both regressions in the unreleased 0.9.16 changelog. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 ++++ tests/test_watch.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b26df9bc..a52dba716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.16 (unreleased) +- Fix (regression from 0.9.15): a nested `.gitignore`/`.graphifyignore` no longer applies its patterns outside its own directory (#1873 / #1887 / #1885, thanks @Alwyn93). When 0.9.15 started reading nested ignore files, a non-anchored pattern was still matched against the path relative to the scan root, so a nested bare `*` (a common "ignore this scratch dir" idiom, e.g. what the hypothesis library writes into `.hypothesis/`) matched every file and `detect()` returned zero files, silently producing an empty graph. Patterns are now matched relative to the directory whose ignore file defined them and only apply within that subtree; the anchor directory itself is exempt. + +- Fix (regression from 0.9.15): `graphify update` no longer emits 0 nodes and refuses to overwrite an existing `graph.json` when the source tree contains a nested broad `.gitignore` (#1880). This was the update-layer symptom of the scoping bug above: the zeroed re-scan produced an empty rebuild, which the shrink-guard then correctly refused. With subtree scoping fixed the rebuild sees the real files again; the shrink-guard is unchanged. + - Fix: `detect_incremental` re-extracts a legacy-manifest file when its mtime moves backwards, not just forwards (#1859 / #1862, thanks @thejesh23). The legacy float-schema branch used a strict `current_mtime > stored`, so a file restored to an older timestamp (a `git checkout` of an older commit, a tarball restore, `rsync --times`) was treated as unchanged and never re-extracted, leaving the graph reflecting newer content than the corpus on disk. It now compares with `!=`, matching the dict-schema branch; with no stored hash to verify, any mtime delta forces a re-extract, and the next save promotes the entry to the hash-verified dict schema. - Fix: the dedup summary line reports the fuzzy-merge count even when there were no exact merges (#1857 / #1860, thanks @thejesh23). The fuzzy branch was nested inside `if exact_merges`, so a doc- or semantic-heavy run that merged only via the cross-file fuzzy pass printed a bare `Deduplicated N node(s).` with no breakdown. Both counts are now reported whenever non-zero. diff --git a/tests/test_watch.py b/tests/test_watch.py index 67c68639d..8047499da 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -188,6 +188,37 @@ def test_rebuild_code_writes_community_name(tmp_path): ) +def test_update_rebuilds_with_nested_star_gitignore(tmp_path): + """#1880: `graphify update` must not emit 0 nodes (and then refuse to + overwrite) just because the source tree has a nested `.gitignore` with a + broad pattern. This was the 0.9.15 symptom of the #1847/#1873 subtree-scoping + bug: a nested bare `*` zeroed the re-scan, update built 0 nodes, and the + shrink-guard refused. With scoping fixed the rebuild sees the real files.""" + import json + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + (corpus / "src").mkdir(parents=True) + (corpus / "src" / "a.py").write_text( + "from src.b import Base\nclass App(Base):\n def run(self): return 1\n", encoding="utf-8" + ) + (corpus / "src" / "b.py").write_text("class Base: pass\n", encoding="utf-8") + (corpus / "main.py").write_text("def top(): return 2\n", encoding="utf-8") + # a common scratch-dir idiom deeper in the tree: ignore everything HERE only + (corpus / "scratch").mkdir() + (corpus / "scratch" / ".gitignore").write_text("*\n", encoding="utf-8") + (corpus / "scratch" / "junk.py").write_text("x = 1\n", encoding="utf-8") + + assert _rebuild_code(corpus, acquire_lock=False) is True + + graph = json.loads((corpus / "graphify-out" / "graph.json").read_text(encoding="utf-8")) + sources = {n.get("source_file", "") for n in graph["nodes"]} + assert graph["nodes"], "update produced 0 nodes on a tree with a nested '*' gitignore (#1880)" + assert any("src/a.py" in s for s in sources) and any("main.py" in s for s in sources) + # the nested-ignored scratch file stays out (scoped correctly, not tree-wide) + assert not any("scratch/junk.py" in s for s in sources) + + def test_graphify_root_preserves_absolute_when_user_supplied(tmp_path): """When the caller supplies an absolute path, ``.graphify_root`` stores that absolute form verbatim — preserving explicit-absolute intent.""" From 1aca158ada2ee23765ebfcae123b37104d791703 Mon Sep 17 00:00:00 2001 From: thejesh Date: Mon, 13 Jul 2026 10:50:33 -0700 Subject: [PATCH 07/42] fix(cargo): honor `package = "..."` rename when resolving internal deps (#1858) --- graphify/cargo_introspect.py | 15 +++++- tests/test_cargo_introspect.py | 92 ++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/graphify/cargo_introspect.py b/graphify/cargo_introspect.py index 3bda96126..fa03ed309 100644 --- a/graphify/cargo_introspect.py +++ b/graphify/cargo_introspect.py @@ -78,8 +78,19 @@ def introspect_cargo(root: str | Path) -> dict[str, Any]: if not isinstance(dependencies, dict): continue source_file = manifest.relative_to(root_path).as_posix() - for dependency_name in sorted(dependencies): - target = crates.get(dependency_name) + for dep_key, spec in sorted(dependencies.items()): + # Cargo lets a dep table entry rename the crate via `package = "..."`: + # db = { path = "../storage", package = "internal-storage" } + # The key `db` is the name used in `use db::…;`; the actual crate + # published under `[package].name = "internal-storage"` is what + # `crates` is keyed by. Without honoring `package`, every renamed + # workspace-internal dep silently drops its edge (#1858). + real_name = dep_key + if isinstance(spec, dict): + pkg_override = spec.get("package") + if isinstance(pkg_override, str) and pkg_override: + real_name = pkg_override + target = crates.get(real_name) if target is None: continue edges.append( diff --git a/tests/test_cargo_introspect.py b/tests/test_cargo_introspect.py index e80d8c16a..9abf82655 100644 --- a/tests/test_cargo_introspect.py +++ b/tests/test_cargo_introspect.py @@ -363,3 +363,95 @@ def test_cargo_introspect_large_workspace_dependency_chain(tmp_path): "source_file": "crates/crate_198/Cargo.toml", "source_location": "L1", } in result["edges"] + + +def test_cargo_introspect_honors_package_rename_on_internal_dep(tmp_path): + """Renamed workspace-internal deps still produce a `crate_depends_on` edge (#1858). + + Cargo's `package = "..."` inside a dep table entry lets the key used in + `use db::…;` differ from the crate's real `[package].name`. Looking up + `crates` by the raw dep-table key misses the rename and silently drops + the edge. + """ + _write_manifest( + tmp_path / "Cargo.toml", + """ +[workspace] +members = ["app", "storage"] +""", + ) + app = tmp_path / "app" + storage = tmp_path / "storage" + app.mkdir() + storage.mkdir() + _write_manifest( + app / "Cargo.toml", + """ +[package] +name = "app" +version = "0.1.0" +edition = "2021" + +[dependencies] +db = { path = "../storage", package = "internal-storage" } +""", + ) + _write_manifest( + storage / "Cargo.toml", + """ +[package] +name = "internal-storage" +version = "0.1.0" +edition = "2021" +""", + ) + + result = introspect_cargo(tmp_path) + + node_ids = {node["id"] for node in result["nodes"]} + assert node_ids == {"crate:app", "crate:internal-storage"} + assert { + "source": "crate:app", + "target": "crate:internal-storage", + "relation": "crate_depends_on", + "context": "cargo_dependency", + "weight": 1.0, + "confidence": "EXTRACTED", + "source_file": "app/Cargo.toml", + "source_location": "L1", + } in result["edges"] + + +def test_cargo_introspect_package_rename_falls_through_when_unresolved(tmp_path): + """A rename pointing at an external (non-workspace) crate stays a no-op. + + Guards against the fix silently emitting edges to registry crates: the + resolved name still has to appear in `crates` (the workspace-internal + index) for an edge to be produced. + """ + _write_manifest( + tmp_path / "Cargo.toml", + """ +[workspace] +members = ["app"] +""", + ) + app = tmp_path / "app" + app.mkdir() + _write_manifest( + app / "Cargo.toml", + """ +[package] +name = "app" +version = "0.1.0" +edition = "2021" + +[dependencies] +tokio_rt = { version = "1", package = "tokio" } +""", + ) + + result = introspect_cargo(tmp_path) + + assert {node["id"] for node in result["nodes"]} == {"crate:app"} + assert result["edges"] == [] From 4ac5f07e88334d0b9f040fa7ac745e3e0bb117bb Mon Sep 17 00:00:00 2001 From: xor_xe Date: Tue, 14 Jul 2026 01:04:33 +0400 Subject: [PATCH 08/42] fix(watch): preserve semantic edges of re-extracted sources (#1865) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A full graphify update evicted every edge whose source_file was a re-extracted source, including LLM semantic edges (e.g. semantically_similar_to) from Markdown docs that also have an AST extractor. The AST pass cannot regenerate those edges, so they were silently lost while their concept nodes survived — node eviction is provenance-aware via _origin (#1116), edge eviction was not. Tag AST-extracted edges with the same _origin=ast marker nodes already carry, and scope rebuild-driven edge eviction to that tier: re-extraction replaces a source's AST edges, while its semantic edges survive until a semantic re-extraction supersedes them. Deletion-driven eviction stays provenance-blind, so edges of deleted or excluded sources are still purged regardless of tier. Edges from graphs built before this change lack the marker; a stale AST edge from a file changed exactly once between the old and new version can linger until its source is deleted — the same migration trade-off the #1116 node marker made. --- graphify/extract.py | 7 ++++- graphify/watch.py | 18 +++++++++---- tests/test_watch.py | 66 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 6 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index a4ac6cbbd..4435e2bfd 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4874,9 +4874,14 @@ def _has_import_evidence(candidate_id: str) -> bool: # Tag AST provenance so the incremental watch rebuild can distinguish # AST-extracted nodes from semantic/LLM nodes. On a full re-extraction # the watcher drops any AST-marked node missing from the fresh output - # even when its source file still exists (#1116). + # even when its source file still exists (#1116). Edges carry the same + # marker so edge eviction can be tier-scoped: re-extracting a source + # replaces its AST edges without evicting the semantic edges the AST + # pass cannot regenerate (#1865). for n in all_nodes: n["_origin"] = "ast" + for e in all_edges: + e["_origin"] = "ast" return { "nodes": all_nodes, diff --git a/graphify/watch.py b/graphify/watch.py index 14454bf08..e32b1c977 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -408,11 +408,11 @@ def _reconcile_existing_graph( } node_evicted_source_identities = set(deleted_source_identities) hyperedge_evicted_source_identities = set(deleted_source_identities) + # Deletion evicts edges regardless of tier; re-extraction only owns a + # source's AST-tier edges (checked per-edge below, #1865). + edge_evicted_source_identities = set(deleted_source_identities) if not full_rebuild: node_evicted_source_identities.update(rebuilt_source_identities) - edge_evicted_source_identities = ( - node_evicted_source_identities | rebuilt_source_identities - ) # Reconcile every rebuild against the current watched corpus. Hook change # lists can contain only a rename destination, so explicit paths alone @@ -485,14 +485,22 @@ def _reconcile_existing_graph( ] all_ids = new_ast_ids | {node["id"] for node in preserved_nodes} - # Edges are owned by source_file. Re-extraction must replace an owner's - # previous edges, while edges from unchanged or semantic sources survive. + # Edges are owned by source_file, but ownership is tier-scoped: the AST + # pass replaces a re-extracted source's AST edges, while that source's + # semantic/LLM edges — which the AST pass cannot regenerate — survive + # until a semantic re-extraction supersedes them. Same provenance rule + # the node reconciliation above applies via _origin (#1865). Deletion + # eviction stays provenance-blind. preserved_edges = [ edge for edge in existing.get("links", existing.get("edges", [])) if edge.get("source") in all_ids and edge.get("target") in all_ids and not source_paths.is_evicted(edge, edge_evicted_source_identities) + and not ( + edge.get("_origin") == "ast" + and source_paths.is_evicted(edge, rebuilt_source_identities) + ) ] new_hyperedge_ids = { diff --git a/tests/test_watch.py b/tests/test_watch.py index 8047499da..a5511ed9e 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -386,6 +386,72 @@ def test_rebuild_code_preserves_hyperedges_for_rebuilt_surviving_source( }] +@pytest.mark.parametrize( + "changed_paths", + [None, [Path("auth.md")]], + ids=["full-update", "incremental-doc-update"], +) +def test_rebuild_code_preserves_semantic_edges_from_reextracted_doc( + tmp_path, changed_paths +): + """#1865: AST-only updates must not evict semantic edges whose source_file + is a re-extracted document; only that source's AST-tier edges are replaced.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "auth.md").write_text( + "# Token Validation\n\nVerifies bearer tokens.\n", encoding="utf-8" + ) + (corpus / "login.md").write_text( + "# Session Verification\n\nVerifies login sessions.\n", encoding="utf-8" + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + node_ids = {n["id"] for n in data["nodes"]} + assert {"auth_token_validation", "login_session_verification"} <= node_ids + + data["links"].extend([ + { + "source": "auth_token_validation", + "target": "login_session_verification", + "relation": "semantically_similar_to", + "confidence": "INFERRED", + "source_file": "auth.md", + }, + # A stale AST-tier edge of the same source must still be evicted. + { + "source": "auth_token_validation", + "target": "login_session_verification", + "relation": "references", + "_origin": "ast", + "source_file": "auth.md", + }, + ]) + graph_path.write_text(json.dumps(data), encoding="utf-8") + + assert _rebuild_code( + corpus, + changed_paths=changed_paths, + no_cluster=True, + acquire_lock=False, + ) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + relations = { + (e.get("source"), e.get("target"), e.get("relation")) + for e in after["links"] + } + assert ( + "auth_token_validation", "login_session_verification", "semantically_similar_to" + ) in relations, "semantic edge from a re-extracted doc must survive an AST-only update" + assert ( + "auth_token_validation", "login_session_verification", "references" + ) not in relations, "stale AST-tier edge of a re-extracted source must be evicted" + + @pytest.mark.parametrize( "changed_paths", [None, [Path("only.py")]], From 2210d69bc466d7f3ef8aadc0ce03f85f231cff78 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Tue, 14 Jul 2026 13:21:29 +0100 Subject: [PATCH 09/42] docs(changelog): record #1865 and #1858 in the unreleased 0.9.16 section Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a52dba716..3cf6f7baa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ Full release notes with details on each version: [GitHub Releases](https://githu - Fix (regression from 0.9.15): `graphify update` no longer emits 0 nodes and refuses to overwrite an existing `graph.json` when the source tree contains a nested broad `.gitignore` (#1880). This was the update-layer symptom of the scoping bug above: the zeroed re-scan produced an empty rebuild, which the shrink-guard then correctly refused. With subtree scoping fixed the rebuild sees the real files again; the shrink-guard is unchanged. +- Fix: a full `graphify update` no longer evicts the LLM semantic edges of a re-extracted document (#1865 / #1868, thanks @xor-xe). Node reconciliation was already provenance-aware (semantic nodes lack the AST `_origin` marker and are preserved), but edge reconciliation evicted by `source_file` alone, so a Markdown/doc file that also has an AST extractor had its `semantically_similar_to`/`conceptually_related_to` edges dropped on an AST-only rebuild that cannot regenerate them, leaving orphaned concept nodes. Edges now carry the same `_origin` marker, and re-extraction replaces only a source's AST-tier edges while its semantic edges survive until a semantic re-extraction supersedes them. Deletion still evicts every tier. + +- Fix: `--cargo` no longer drops a workspace-internal dependency edge that uses Cargo's `package = "..."` rename (#1858 / #1861, thanks @thejesh23). The dependency loop looked up crates by the `[dependencies]` table key, but a renamed entry (`db = { path = "../storage", package = "internal-storage" }`) keys on `db` while the target crate is published as `internal-storage`, so `crate_depends_on` silently never appeared. The lookup now honors the `package` override. + - Fix: `detect_incremental` re-extracts a legacy-manifest file when its mtime moves backwards, not just forwards (#1859 / #1862, thanks @thejesh23). The legacy float-schema branch used a strict `current_mtime > stored`, so a file restored to an older timestamp (a `git checkout` of an older commit, a tarball restore, `rsync --times`) was treated as unchanged and never re-extracted, leaving the graph reflecting newer content than the corpus on disk. It now compares with `!=`, matching the dict-schema branch; with no stored hash to verify, any mtime delta forces a re-extract, and the next save promotes the entry to the hash-verified dict schema. - Fix: the dedup summary line reports the fuzzy-merge count even when there were no exact merges (#1857 / #1860, thanks @thejesh23). The fuzzy branch was nested inside `if exact_merges`, so a doc- or semantic-heavy run that merged only via the cross-file fuzzy pass printed a bare `Deduplicated N node(s).` with no breakdown. Both counts are now reported whenever non-zero. From 70bc9ca7b2a41da6c42db37da05bddba0a748af7 Mon Sep 17 00:00:00 2001 From: bchan84 Date: Mon, 13 Jul 2026 21:11:36 +0800 Subject: [PATCH 10/42] fix(dedup): let the defining file win an ID collision, and warn only about real loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Node IDs are _, so a doc that merely references an entity mints the entity's own ID and collides with the entity's node by construction. The pre-dedup pass kept whichever arrived first, so chunk order decided whether an entity kept its own attributes or a passing cross-reference's, and the warning that fired (#1504) told the user a same-name-different-directory clash had lost their data — while the one drop that really is lossy, two labels for one ID from the same file, was silent because the warning was gated on source_file differing. The survivor is now the node whose source_file is the file its ID encodes (any trailing slice of the path, so absolute, repo-relative, and pre-#1504 bare-stem IDs all resolve), falling back to first-seen. Reporting follows what the drop actually costs: a cross-reference folding into the node it references loses nothing (edges are keyed by ID and rewire to the survivor) and is silent; a same-file relabel notes the discarded label; two files that both encode the ID are distinct entities and still WARNING, now stating the ID-scheme cause rather than a filename clash. Refs #1851 --- graphify/dedup.py | 102 +++++++++++++++++++++++++++++++++++++------- tests/test_dedup.py | 84 +++++++++++++++++++++++++++++++++++- 2 files changed, 169 insertions(+), 17 deletions(-) diff --git a/graphify/dedup.py b/graphify/dedup.py index 47c4eff8f..9da26b30b 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -187,6 +187,76 @@ def _is_code(node: dict) -> bool: return node.get("file_type") == "code" +# ── ID collisions ───────────────────────────────────────────────────────────── + +_ID_SEGMENT = re.compile(r"[^a-z0-9]+") +_EXTENSION = re.compile(r"\.[^./]+$") + + +def _id_prefixes(source_file: str) -> set[str]: + """The ID prefixes a node extracted from ``source_file`` may legitimately mint. + + An ID is ``_``, where the path is the extension-stripped source + path, each segment slugified and joined with ``_``. Every trailing slice of the + path counts as a prefix: the stored path may be absolute or repo-relative, and + graphs built under the pre-#1504 scheme keyed off the bare filename stem. + """ + stem = _EXTENSION.sub("", source_file.replace("\\", "/")) + segments = [s for s in (_ID_SEGMENT.sub("_", p.casefold()).strip("_") + for p in stem.split("/")) if s] + return {"_".join(segments[i:]) for i in range(len(segments))} + + +def _defines_id(node: dict) -> bool: + """True when the node's own source_file is the file its ID encodes. + + A doc that *references* an entity mints the ID of the entity's own file, not one + derived from the doc's path — so the referencing node collides with the defining + node by construction. This separates the two: the definer owns the ID. + """ + nid = node.get("id") or "" + source_file = node.get("source_file") or "" + if not nid or not source_file: + return False + return any(nid.startswith(f"{prefix}_") for prefix in _id_prefixes(source_file)) + + +def _report_id_collision(nid: str, survivor: dict, losers: list[dict]) -> None: + """Report an ID collision in proportion to what dropping the loser actually costs. + + Cross-reference to a defining node: same entity, edges are keyed by ID and rewire + to the survivor — nothing is lost, so say nothing. Same file, different labels: the + extractor emitted two labels for one entity and one is discarded — note it. Two + files that both encode this ID: they are distinct entities and one is genuinely + lost — warn, and point at the extraction split that keeps them apart (#1504). + """ + keep_file = survivor.get("source_file") or "" + keep_label = survivor.get("label") or "" + for loser in losers: + lose_file = loser.get("source_file") or "" + lose_label = loser.get("label") or "" + if lose_file == keep_file: + if _norm(lose_label) != _norm(keep_label): + print( + f"[graphify] note: node '{nid}' was extracted twice from " + f"'{keep_file}' under different labels — keeping '{keep_label}', " + f"dropping '{lose_label}'.", + file=sys.stderr, + ) + elif _defines_id(survivor) and not _defines_id(loser): + continue # the loser only references the entity the survivor defines + else: + print( + f"[graphify] WARNING: node '{nid}' is minted by two different files — " + f"keeping '{keep_label}' from '{keep_file}', dropping '{lose_label}' " + f"from '{lose_file}'. An ID is derived from the source path plus the " + f"entity name, so this one does not identify a single entity and the " + f"dropped node is lost. To keep them distinct, run 'graphify extract' " + f"per subfolder and merge with 'graphify merge-graphs'.", + file=sys.stderr, + ) + + # ── main entry point ────────────────────────────────────────────────────────── def deduplicate_entities( @@ -220,29 +290,29 @@ def deduplicate_entities( if len(nodes) <= 1: return nodes, edges - # Pre-deduplicate: keep first occurrence of each id. - # Warn when two nodes share an ID but originate from different source files — - # this indicates a cross-chunk ID collision (#1504) where silent data loss occurs. + # Pre-deduplicate: one node per ID. The survivor is the node that *defines* the + # ID (its source_file is the file the ID encodes), not merely the first seen — + # otherwise chunk order decides whether an entity keeps its own attributes or a + # passing cross-reference's. Warnings are then emitted for what is actually lost + # (#1504); a same-entity merge costs nothing and stays quiet. seen_ids: dict[str, dict] = {} + dropped: dict[str, list[dict]] = defaultdict(list) for node in nodes: nid = node.get("id", "") if not nid: continue - if nid not in seen_ids: + incumbent = seen_ids.get(nid) + if incumbent is None: seen_ids[nid] = node + elif _defines_id(node) and not _defines_id(incumbent): + seen_ids[nid] = node + dropped[nid].append(incumbent) else: - existing_sf = seen_ids[nid].get("source_file") or "" - new_sf = node.get("source_file") or "" - if existing_sf != new_sf: - print( - f"[graphify] WARNING: node '{nid}' from '{new_sf}' collides with " - f"node from '{existing_sf}' — the second node will be dropped. " - f"This is a cross-chunk ID collision caused by two files with the " - f"same name in different directories. To avoid data loss, run " - f"'graphify extract' per subfolder and merge with " - f"'graphify merge-graphs'.", - file=sys.stderr, - ) + dropped[nid].append(node) + + for nid, losers in dropped.items(): + _report_id_collision(nid, seen_ids[nid], losers) + unique_nodes = list(seen_ids.values()) if len(unique_nodes) <= 1: diff --git a/tests/test_dedup.py b/tests/test_dedup.py index 8d8028589..ee69cae0f 100644 --- a/tests/test_dedup.py +++ b/tests/test_dedup.py @@ -1,7 +1,7 @@ """Tests for graphify/dedup.py entity deduplication pipeline.""" from __future__ import annotations import pytest -from graphify.dedup import deduplicate_entities, _entropy, _shingles +from graphify.dedup import deduplicate_entities, _defines_id, _entropy, _shingles # ── entropy gate ───────────────────────────────────────────────────────────── @@ -440,3 +440,85 @@ def test_dedup_summary_still_reports_exact_only(capsys): captured = capsys.readouterr() assert "exact" in captured.out assert "fuzzy" not in captured.out + + +# ── ID collisions: definition vs cross-reference ────────────────────────────── + +# The defining node and a doc that merely mentions the entity. Both mint the ID +# encoded from the *defining* file's path, so they collide by construction. +_DEFINING = {"id": "agents_make_batch_fixtures_make_batch_fixtures", + "label": "make-batch-fixtures agent", "file_type": "concept", + "source_file": "agents/make-batch-fixtures.md"} +_REFERENCING = {"id": "agents_make_batch_fixtures_make_batch_fixtures", + "label": "make-batch-fixtures", "file_type": "concept", + "source_file": "available/diagnose-issue/SKILL.md"} + + +@pytest.mark.parametrize("nodes", [ + [_DEFINING, _REFERENCING], + [_REFERENCING, _DEFINING], +], ids=["definition-first", "reference-first"]) +def test_defining_file_wins_over_referencing_file(nodes, capsys): + """The node whose source_file is the file its ID encodes survives, whichever + chunk order the nodes arrive in — the survivor must not depend on it.""" + result_nodes, _ = deduplicate_entities(list(nodes), [], communities={}) + + assert len(result_nodes) == 1 + assert result_nodes[0]["source_file"] == "agents/make-batch-fixtures.md" + assert result_nodes[0]["label"] == "make-batch-fixtures agent" + + +def test_reference_collision_is_silent(capsys): + """A cross-reference collapsing into the entity it references loses nothing — + edges are keyed by ID and rewire to the survivor — so it must not be reported.""" + edges = _make_edges("agents_make_batch_fixtures_make_batch_fixtures", "other") + result_nodes, result_edges = deduplicate_entities( + [_DEFINING, _REFERENCING], edges, communities={}) + + assert len(result_nodes) == 1 + assert len(result_edges) == 1 + captured = capsys.readouterr() + assert "WARNING" not in captured.err + assert "note:" not in captured.err + + +def test_absolute_source_path_still_defines_id(capsys): + """source_file is absolute in some pipelines and repo-relative in others; the + defining file is recognised either way.""" + absolute = dict(_DEFINING, source_file="/home/u/proj/agents/make-batch-fixtures.md") + result_nodes, _ = deduplicate_entities([_REFERENCING, absolute], [], communities={}) + + assert len(result_nodes) == 1 + assert result_nodes[0]["label"] == "make-batch-fixtures agent" + assert "WARNING" not in capsys.readouterr().err + + +def test_same_file_relabel_is_noted(capsys): + """Two labels for one ID from one file: the loser's label is discarded, which is + the one drop that used to be silent. It is a note, not a collision warning.""" + nodes = [ + {"id": "agents_make_batch_fixtures_make_batch_fixtures", + "label": "make-batch-fixtures agent", "file_type": "concept", + "source_file": "agents/make-batch-fixtures.md"}, + {"id": "agents_make_batch_fixtures_make_batch_fixtures", + "label": "make-batch-fixtures helper agent", "file_type": "concept", + "source_file": "agents/make-batch-fixtures.md"}, + ] + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) + + assert len(result_nodes) == 1 + captured = capsys.readouterr() + assert "note:" in captured.err + assert "make-batch-fixtures helper agent" in captured.err + assert "WARNING" not in captured.err + + +def test_defines_id_helper(): + assert _defines_id(_DEFINING) + assert not _defines_id(_REFERENCING) + # Pre-#1504 IDs keyed off the bare filename stem. + assert _defines_id({"id": "readme_booking_service", + "source_file": "module-a/README.md"}) + # A path that is merely a string-prefix of the ID's path does not define it. + assert not _defines_id({"id": "agents_foo", "source_file": "agent/foo.md"}) + assert not _defines_id({"id": "docs_intro_foo", "source_file": ""}) From fd54f0ef0b11eb139b3139f632bc12e8380f06ec Mon Sep 17 00:00:00 2001 From: safishamsi Date: Tue, 14 Jul 2026 13:51:42 +0100 Subject: [PATCH 11/42] fix(dedup): deterministic collision survivor + bare-file definer (#1851 followup) Two tweaks on top of @bchan84x's #1852: 1. The definer heuristic decided the survivor pairwise, so when several same-file nodes co-defined one ID the surviving label still depended on arrival order (the exact 3-node case #1851 reports). Choose the survivor by a total order (definer first, then shorter/canonical label, then lexically) via a min over _collision_rank, so it is order-independent. Direction preserves #1504 (lexically-first source path wins). 2. _defines_id now also recognizes a bare file-level node whose id is exactly the slugified path (nid == prefix), not only _. Adds an order-independence test over all 6 permutations of definer + same-file relabel + cross-file reference, and a bare-file-node test. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 ++ graphify/dedup.py | 30 ++++++++++++++++++++++++++++-- tests/test_dedup.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cf6f7baa..032eb1044 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.16 (unreleased) +- Fix: the dedup ID-collision warning now reports in proportion to what is actually lost, and an ID collision resolves to a deterministic survivor (#1851 / #1852, thanks @bchan84x). Two nodes can mint the same `_` ID when a document merely references an entity defined elsewhere; the old code kept whichever arrived first and always printed a scary "two files with the same name" warning that overstated the loss (edges rewire to the survivor, so a reference collapse loses nothing). Now the node whose `source_file` actually defines the ID wins, a harmless reference collapse is silent, a same-file relabel is a quiet note, and only a genuine cross-file collision warns. The survivor is chosen by a total order (definer first, then the shorter/canonical label, then lexically) so it no longer depends on chunk arrival order, including when several same-file nodes co-define one ID. + - Fix (regression from 0.9.15): a nested `.gitignore`/`.graphifyignore` no longer applies its patterns outside its own directory (#1873 / #1887 / #1885, thanks @Alwyn93). When 0.9.15 started reading nested ignore files, a non-anchored pattern was still matched against the path relative to the scan root, so a nested bare `*` (a common "ignore this scratch dir" idiom, e.g. what the hypothesis library writes into `.hypothesis/`) matched every file and `detect()` returned zero files, silently producing an empty graph. Patterns are now matched relative to the directory whose ignore file defined them and only apply within that subtree; the anchor directory itself is exempt. - Fix (regression from 0.9.15): `graphify update` no longer emits 0 nodes and refuses to overwrite an existing `graph.json` when the source tree contains a nested broad `.gitignore` (#1880). This was the update-layer symptom of the scoping bug above: the zeroed re-scan produced an empty rebuild, which the shrink-guard then correctly refused. With subtree scoping fixed the rebuild sees the real files again; the shrink-guard is unchanged. diff --git a/graphify/dedup.py b/graphify/dedup.py index 9da26b30b..79a1bfb8a 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -218,7 +218,30 @@ def _defines_id(node: dict) -> bool: source_file = node.get("source_file") or "" if not nid or not source_file: return False - return any(nid.startswith(f"{prefix}_") for prefix in _id_prefixes(source_file)) + # `nid == prefix` covers a bare file-level node whose id is exactly the + # slugified path with no `_entity` suffix (a semantic node for the file + # itself); `startswith(prefix + "_")` covers the usual `_` id. + return any(nid == prefix or nid.startswith(f"{prefix}_") + for prefix in _id_prefixes(source_file)) + + +def _collision_rank(node: dict) -> tuple: + """A total order for choosing the survivor of an ID collision, independent of + the order the colliding nodes arrive in. + + The winner is the node with the SMALLEST rank. A node whose ``source_file`` + defines the ID always outranks a mere reference; among equally-(non-)defining + nodes it prefers the shorter, more canonical label over a longer qualified + variant, then breaks any remaining tie lexically on label and then source_file + (so the lexically-first path wins) — fully deterministic regardless of order. + """ + label = node.get("label") or "" + return ( + not _defines_id(node), # definers (False) sort before references (True) + len(label), # shorter, more canonical label first + label, # lexical tiebreak + node.get("source_file") or "", # lexically-first source path wins + ) def _report_id_collision(nid: str, survivor: dict, losers: list[dict]) -> None: @@ -304,7 +327,10 @@ def deduplicate_entities( incumbent = seen_ids.get(nid) if incumbent is None: seen_ids[nid] = node - elif _defines_id(node) and not _defines_id(incumbent): + elif _collision_rank(node) < _collision_rank(incumbent): + # Smallest-ranked node wins; the min over a total order is independent + # of the order nodes arrive in, so the survivor no longer depends on + # chunk ordering (#1851). seen_ids[nid] = node dropped[nid].append(incumbent) else: diff --git a/tests/test_dedup.py b/tests/test_dedup.py index ee69cae0f..57fe6c192 100644 --- a/tests/test_dedup.py +++ b/tests/test_dedup.py @@ -513,6 +513,35 @@ def test_same_file_relabel_is_noted(capsys): assert "WARNING" not in captured.err +def test_collision_survivor_is_order_independent(): + """#1851: definer + same-file relabel + cross-file reference. Across every + insertion order the SAME node (source_file AND label) must survive — the + definer heuristic alone left the label order-dependent among co-definers.""" + import itertools + nid = "agents_make_batch_fixtures_make_batch_fixtures" + definer = {"id": nid, "label": "make-batch-fixtures agent", + "file_type": "concept", "source_file": "agents/make-batch-fixtures.md"} + relabel = {"id": nid, "label": "make-batch-fixtures helper agent", + "file_type": "concept", "source_file": "agents/make-batch-fixtures.md"} + xref = {"id": nid, "label": "make-batch-fixtures", "file_type": "concept", + "source_file": "available/diagnose-issue/SKILL.md"} + survivors = set() + for perm in itertools.permutations([definer, relabel, xref]): + out, _ = deduplicate_entities([dict(n) for n in perm], [], communities={}) + assert len(out) == 1 + survivors.add((out[0]["source_file"], out[0]["label"])) + assert survivors == {("agents/make-batch-fixtures.md", "make-batch-fixtures agent")}, ( + f"non-deterministic collision survivor: {survivors}" + ) + + +def test_bare_file_node_defines_its_own_id(): + """A file-level semantic node whose id is exactly the slugified path (no + `_entity` suffix) must be recognised as defining its id (#1851 tweak).""" + assert _defines_id({"id": "agents_make_batch_fixtures", + "source_file": "agents/make-batch-fixtures.md"}) + + def test_defines_id_helper(): assert _defines_id(_DEFINING) assert not _defines_id(_REFERENCING) From 2795a18bda4126715a330513b4196e619d755a97 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Tue, 14 Jul 2026 14:03:20 +0100 Subject: [PATCH 12/42] fix(exclude): persist --exclude so rebuilds re-apply it (#1886) --exclude was honored only by the initial extract scan; update/watch/hook rebuilds re-ran detect() without it and silently re-indexed excluded paths. extract now writes the patterns to a graphify-out/.graphify_build .json sidecar, and _rebuild_code reads and re-applies them on every rebuild (layered after the ignore files, so they still win). Pre-existing graphs without the sidecar behave as before. Adds a regression test. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 ++ graphify/cli.py | 4 ++++ graphify/watch.py | 47 ++++++++++++++++++++++++++++++++++++++++++++- tests/test_watch.py | 26 +++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 032eb1044..19a0d6c2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.16 (unreleased) +- Fix: `--exclude` patterns now survive into `update`/`watch`/git-hook rebuilds instead of applying only to the initial scan (#1886). The patterns were never persisted, so the first rebuild re-ran `detect()` without them and silently re-indexed the excluded paths. `extract` now records them in a `graphify-out/.graphify_build.json` sidecar and `_rebuild_code` re-applies them on every rebuild (they still layer after `.gitignore`/`.graphifyignore`/`.git/info/exclude`, so they keep winning). Graphs built before this release simply have no sidecar and behave as before. + - Fix: the dedup ID-collision warning now reports in proportion to what is actually lost, and an ID collision resolves to a deterministic survivor (#1851 / #1852, thanks @bchan84x). Two nodes can mint the same `_` ID when a document merely references an entity defined elsewhere; the old code kept whichever arrived first and always printed a scary "two files with the same name" warning that overstated the loss (edges rewire to the survivor, so a reference collapse loses nothing). Now the node whose `source_file` actually defines the ID wins, a harmless reference collapse is silent, a same-file relabel is a quiet note, and only a genuine cross-file collision warns. The survivor is chosen by a total order (definer first, then the shorter/canonical label, then lexically) so it no longer depends on chunk arrival order, including when several same-file nodes co-define one ID. - Fix (regression from 0.9.15): a nested `.gitignore`/`.graphifyignore` no longer applies its patterns outside its own directory (#1873 / #1887 / #1885, thanks @Alwyn93). When 0.9.15 started reading nested ignore files, a non-anchored pattern was still matched against the path relative to the scan root, so a nested bare `*` (a common "ignore this scratch dir" idiom, e.g. what the hypothesis library writes into `.hypothesis/`) matched every file and `detect()` returned zero files, silently producing an empty graph. Patterns are now matched relative to the directory whose ignore file defined them and only apply within that subtree; the anchor directory itself is exempt. diff --git a/graphify/cli.py b/graphify/cli.py index 0bb124777..92f05875d 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -2071,6 +2071,10 @@ def _parse_float(name: str, raw: str) -> float: out_root = (out_dir.resolve() if out_dir else target) graphify_out = out_root / _GRAPHIFY_OUT graphify_out.mkdir(parents=True, exist_ok=True) + # Persist --exclude so later update/watch/hook rebuilds re-apply it + # instead of silently re-including the excluded paths (#1886). + from graphify.watch import _write_build_config as _write_build_cfg + _write_build_cfg(graphify_out, excludes=cli_excludes or None) stages = _StageTimer(cli_timing) diff --git a/graphify/watch.py b/graphify/watch.py index e32b1c977..df911ef8f 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -69,6 +69,44 @@ def _drain_pending(out_dir: Path) -> list[Path]: return out +# Build options that must survive into later rebuilds. The initial `extract` +# scan honours `--exclude`, but `update`/`watch`/hook rebuilds re-run detect() +# and would silently re-include excluded paths unless the patterns are persisted +# (#1886). We store them beside the graph so any rebuild driver can re-apply them. +_BUILD_CONFIG_FILENAME = ".graphify_build.json" + + +def _write_build_config(out_dir: Path, *, excludes: "list[str] | None") -> None: + """Persist build options (currently ``--exclude`` patterns) under ``out_dir``. + + Best-effort and non-clobbering: with no excludes it leaves any existing file + untouched, so a plain rebuild never erases patterns a prior extract recorded. + """ + if not excludes: + return + try: + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / _BUILD_CONFIG_FILENAME).write_text( + json.dumps({"excludes": list(excludes)}), encoding="utf-8" + ) + except OSError: + pass + + +def _read_build_excludes(out_dir: Path) -> list[str]: + """Return the persisted ``--exclude`` patterns for this graph, or [].""" + try: + path = out_dir / _BUILD_CONFIG_FILENAME + if path.is_file(): + cfg = json.loads(path.read_text(encoding="utf-8")) + ex = cfg.get("excludes") if isinstance(cfg, dict) else None + if isinstance(ex, list): + return [str(x) for x in ex if isinstance(x, str) and x] + except (OSError, json.JSONDecodeError): + pass + return [] + + def _merge_changed_paths(*sources: "list[Path] | None") -> list[Path]: """Concatenate path lists, preserving order and dropping duplicates. @@ -827,7 +865,14 @@ def _rebuild_code( from graphify.export import to_json, to_html from graphify.security import check_graph_file_size_cap - detected = detect(watch_path, follow_symlinks=follow_symlinks) + # Re-apply the excludes the initial extract recorded, so an update/watch/ + # hook rebuild does not silently re-include deliberately excluded paths + # (#1886). + _persisted_excludes = _read_build_excludes(out) + detected = detect( + watch_path, follow_symlinks=follow_symlinks, + extra_excludes=_persisted_excludes or None, + ) code_files = [Path(f) for f in detected['files']['code']] # Include document files that have AST extractors (e.g. .md, .mdx, .qmd) diff --git a/tests/test_watch.py b/tests/test_watch.py index a5511ed9e..7c115c1d8 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -219,6 +219,32 @@ def test_update_rebuilds_with_nested_star_gitignore(tmp_path): assert not any("scratch/junk.py" in s for s in sources) +def test_rebuild_honors_persisted_excludes(tmp_path): + """#1886: `--exclude` recorded at extract time must survive into update/watch/ + hook rebuilds. Before the fix only the initial scan applied the excludes, so + the first rebuild silently re-indexed the excluded paths. _rebuild_code now + reads the persisted build config and re-applies them.""" + import json + from graphify.watch import _rebuild_code, _write_build_config + + corpus = tmp_path / "corpus" + (corpus / "src").mkdir(parents=True) + (corpus / "vendor").mkdir() + (corpus / "src" / "app.py").write_text("def keep(): return 1\n", encoding="utf-8") + (corpus / "main.py").write_text("def top(): return 2\n", encoding="utf-8") + (corpus / "vendor" / "lib.py").write_text("def vendored(): pass\n", encoding="utf-8") + _write_build_config(corpus / "graphify-out", excludes=["vendor"]) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + graph = json.loads((corpus / "graphify-out" / "graph.json").read_text(encoding="utf-8")) + sources = {n.get("source_file", "") for n in graph["nodes"]} + assert any("src/app.py" in s for s in sources) and any("main.py" in s for s in sources) + assert not any("vendor/lib.py" in s for s in sources), ( + "rebuild silently re-included an excluded path (#1886)" + ) + + def test_graphify_root_preserves_absolute_when_user_supplied(tmp_path): """When the caller supplies an absolute path, ``.graphify_root`` stores that absolute form verbatim — preserving explicit-absolute intent.""" From 060dd6317f70254d615a812162dba95088e58eee Mon Sep 17 00:00:00 2001 From: safishamsi Date: Tue, 14 Jul 2026 23:19:06 +0100 Subject: [PATCH 13/42] fix(extract): resolve Python module-qualified calls to calls edges (#1883) `module.function()` where `module` is imported produced no calls edge, while bare-name calls did. The member call was captured, but the shared cross-file pass skips all member calls and _resolve_python_member_calls only handled capitalized class receivers, so a lowercase module receiver fell through. Add a module arm: a lowercase receiver naming a module imported into the caller's file resolves to the single callable that module contains. Keyed on stable node ids (imports edge source is the caller file node; contains maps file -> children), not source_file strings, so it holds under the cache_root id-remap relativization. Also drop the no-classes early return that skipped the whole resolver when a corpus had functions but no class methods. Guards: only modules imported into the caller's own file match (so self/obj/local instances never match), and a single-definition god-node guard on the callable. Adds a positive test and a false-edge negative. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 + graphify/extract.py | 100 ++++++++++++++++++++++++++++++------------ tests/test_extract.py | 45 +++++++++++++++++++ 3 files changed, 118 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19a0d6c2f..3dec27fae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.16 (unreleased) +- Fix: Python qualified calls to an imported module (`module.function()`) now produce a `calls` edge, matching bare-name calls (#1883). The call was captured correctly but fell through a gap: the shared cross-file pass skips every member call (to avoid god-node blowups from bare method names), and the Python member-call resolver only handled capitalized class receivers (`ClassName.method()`), so a lowercase module receiver was dropped. The resolver now also resolves a lowercase receiver that names a module imported into the caller's file, linking to the single callable that module contains (guarded so `self`/`obj`/local instances and ambiguous names never create a false edge). + - Fix: `--exclude` patterns now survive into `update`/`watch`/git-hook rebuilds instead of applying only to the initial scan (#1886). The patterns were never persisted, so the first rebuild re-ran `detect()` without them and silently re-indexed the excluded paths. `extract` now records them in a `graphify-out/.graphify_build.json` sidecar and `_rebuild_code` re-applies them on every rebuild (they still layer after `.gitignore`/`.graphifyignore`/`.git/info/exclude`, so they keep winning). Graphs built before this release simply have no sidecar and behave as before. - Fix: the dedup ID-collision warning now reports in proportion to what is actually lost, and an ID collision resolves to a deterministic survivor (#1851 / #1852, thanks @bchan84x). Two nodes can mint the same `_` ID when a document merely references an entity defined elsewhere; the old code kept whichever arrived first and always printed a scary "two files with the same name" warning that overstated the loss (edges rewire to the survivor, so a reference collapse loses nothing). Now the node whose `source_file` actually defines the ID wins, a harmless reference collapse is silent, a same-file relabel is a quiet note, and only a genuine cross-file collision warns. The survivor is chosen by a total order (definer first, then the shorter/canonical label, then lexically) so it no longer depends on chunk arrival order, including when several same-file nodes co-define one ID. diff --git a/graphify/extract.py b/graphify/extract.py index 4435e2bfd..a82b074fb 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -2162,9 +2162,9 @@ def _key(label: str) -> str: tnode = node_by_id.get(tgt) if tnode is not None: method_index[(src, _key(tnode.get("label", "")))] = tgt - if not class_def_nids: - return - # A class with N methods produced N entries; collapse to a unique set. + # A class with N methods produced N entries; collapse to a unique set. (No + # early return when there are no classes: the module arm below resolves + # `module.func()` where the callable is a plain function, not a method.) for k in list(class_def_nids): class_def_nids[k] = sorted(set(class_def_nids[k])) @@ -2172,35 +2172,46 @@ def _key(label: str) -> str: for result in per_file: all_raw_calls.extend(result.get("raw_calls", [])) + # Module-alias arm index (#1883): `module.func()` where `module` is imported. + # Key on stable node ids, not source_file strings (source_file is relativized + # by the CLI id-remap pass but raw_calls keep their original path, so a string + # join would miss under an explicit cache_root). The `imports` edge's source + # is the caller's own file node; `contains` maps a file node to its children. + contains_children: dict[str, dict[str, list[str]]] = {} + file_of_node: dict[str, str] = {} + for e in all_edges: + if e.get("relation") == "contains": + src, tgt = e.get("source"), e.get("target") + tnode = node_by_id.get(tgt) + if tnode is not None: + contains_children.setdefault(src, {}).setdefault( + _key(tnode.get("label", "")), []).append(tgt) + file_of_node[tgt] = src + imported_by_filenode: dict[str, set[str]] = {} + for e in all_edges: + if e.get("relation") in ("imports", "imports_from"): + imported_by_filenode.setdefault(e.get("source"), set()).add(e.get("target")) + + def _module_stem_key(nid: str) -> str: + n = node_by_id.get(nid) + if not n: + return "" + sf = n.get("source_file") or "" + stem = Path(sf).stem if sf else "" + return _key(stem or n.get("label", "")) + existing_pairs = {(e.get("source"), e.get("target")) for e in all_edges} - for rc in all_raw_calls: - if not rc.get("is_member_call"): - continue - receiver = rc.get("receiver") - callee = rc.get("callee") - caller = rc.get("caller_nid") - if not receiver or not callee or not caller: - continue - # Only a capitalized receiver is treated as a class reference, so an - # instance/module (`self`, `obj`, `config`) never collides with a - # same-spelled class via the case-folding key. - if not receiver[:1].isupper(): - continue - class_nids = class_def_nids.get(_key(receiver), []) - if len(class_nids) != 1: # absent or ambiguous -> bail (god-node guard) - continue - method_nid = method_index.get((class_nids[0], _key(callee))) - if not method_nid or method_nid == caller: - continue - if (caller, method_nid) in existing_pairs: - continue - existing_pairs.add((caller, method_nid)) - # EXTRACTED: a qualified `ClassName.method()` is an explicit, unambiguous - # static reference (unlike a bare instance member call), and the class - # resolved to exactly one definition that owns the method. + + def _emit_call(caller: str, target_nid: "str | None", rc: dict) -> None: + if not target_nid or target_nid == caller or (caller, target_nid) in existing_pairs: + return + existing_pairs.add((caller, target_nid)) + # EXTRACTED: a qualified call (`ClassName.method()` or `module.func()`) is + # an explicit, unambiguous static reference resolved to exactly one + # definition (each arm applies a single-definition god-node guard). all_edges.append({ "source": caller, - "target": method_nid, + "target": target_nid, "relation": "calls", "context": "call", "confidence": "EXTRACTED", @@ -2210,6 +2221,37 @@ def _key(label: str) -> str: "weight": 1.0, }) + for rc in all_raw_calls: + if not rc.get("is_member_call"): + continue + receiver = rc.get("receiver") + callee = rc.get("callee") + caller = rc.get("caller_nid") + if not receiver or not callee or not caller: + continue + if receiver[:1].isupper(): + # Class arm (#1446): a capitalized receiver is a class reference; an + # instance (`self`, `obj`) never collides with a same-spelled class. + class_nids = class_def_nids.get(_key(receiver), []) + if len(class_nids) != 1: # absent or ambiguous -> bail (god-node guard) + continue + _emit_call(caller, method_index.get((class_nids[0], _key(callee))), rc) + else: + # Module arm (#1883): a lowercase receiver may be an imported module. + # Resolve it against the modules imported into the caller's own file + # (so `self`/`obj`/local instances, which are not imported modules, + # never match), then to the single callable that module contains. + rkey = _key(receiver) + caller_file = file_of_node.get(caller) + mods = [t for t in imported_by_filenode.get(caller_file, ()) + if t in contains_children and _module_stem_key(t) == rkey] + if len(mods) != 1: # not an imported module, or ambiguous -> bail + continue + children = contains_children[mods[0]].get(_key(callee), []) + if len(children) != 1: # absent or ambiguous callable -> bail + continue + _emit_call(caller, children[0], rc) + def _resolve_typescript_member_calls( per_file: list[dict], diff --git a/tests/test_extract.py b/tests/test_extract.py index dfb28d6d1..d588b2151 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -970,6 +970,51 @@ def test_python_qualified_class_method_call_resolves_extracted(tmp_path): assert call_edges[0]["confidence"] == "EXTRACTED" +def test_python_module_qualified_call_resolves_extracted(tmp_path): + """`module.func()` where `module` is imported resolves to the callable that + module contains, with an EXTRACTED `calls` edge (#1883). A lowercase module + receiver was previously dropped alongside instance calls.""" + mathlib = tmp_path / "mathlib.py" + caller = tmp_path / "caller.py" + mathlib.write_text("def compute(x):\n return x * 2\n") + caller.write_text( + "import mathlib\n\n" + "def use_qualified(n):\n" + " return mathlib.compute(n)\n" + ) + result = extract([caller, mathlib], cache_root=tmp_path) + nodes = {n["id"]: n for n in result["nodes"]} + edges = [ + e for e in result["edges"] + if e["relation"] == "calls" + and "use_qualified" in nodes[e["source"]]["label"] + and "compute" in nodes[e["target"]]["label"] + and "mathlib.py" in (nodes[e["target"]].get("source_file") or "") + ] + assert len(edges) == 1, f"expected one use_qualified->compute edge, got {edges}" + assert edges[0]["confidence"] == "EXTRACTED" + + +def test_python_module_qualified_call_requires_the_import(tmp_path): + """A `module.func()` call must resolve only against a module the caller's own + file imports — a local instance `o.compute()` (o is a parameter) must NOT be + linked to a same-named function in some other module (#1883 false-edge guard).""" + mathlib = tmp_path / "mathlib.py" + caller = tmp_path / "caller.py" + mathlib.write_text("def compute(x):\n return x * 2\n") + # no `import mathlib`; `o` is just a parameter that happens to expose compute() + caller.write_text("def via_obj(o):\n return o.compute(3)\n") + result = extract([caller, mathlib], cache_root=tmp_path) + nodes = {n["id"]: n for n in result["nodes"]} + bad = [ + e for e in result["edges"] + if e["relation"] == "calls" + and "via_obj" in nodes[e["source"]]["label"] + and "compute" in nodes[e["target"]]["label"] + ] + assert bad == [], f"non-imported receiver must not link cross-file: {bad}" + + def test_python_qualified_call_resolves_when_method_name_collides_with_caller(tmp_path): """The real #1446 shape: a viewset action `approve()` delegates to a SERVICE action of the SAME name via `Service.approve()`. The bare-name in-file lookup From f80e4fac351036a317a63f24bfbaf2f7d1e0f7a7 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Tue, 14 Jul 2026 23:20:36 +0100 Subject: [PATCH 14/42] test(watch): pin update discovers newly-added files/dirs (#1837) #1837 (update silently fails to discover new files) was the same nested-gitignore scoping bug (#1873/#1880), already fixed. The existing test only does a single build; add the build -> add new file+dir -> update -> discovered sequence the report describes, with a nested `*` scratch dir as a scoping guard. No production change. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_watch.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_watch.py b/tests/test_watch.py index 7c115c1d8..8df9c1b84 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -219,6 +219,38 @@ def test_update_rebuilds_with_nested_star_gitignore(tmp_path): assert not any("scratch/junk.py" in s for s in sources) +def test_update_discovers_newly_added_files_and_dirs(tmp_path): + """#1837: after an initial build, a plain `graphify update` (full re-scan, no + change-list) must discover brand-new files AND new directories. The reported + silent no-op was the #1873 nested-gitignore scoping bug zeroing the re-scan; + this pins the build -> add -> update -> discovered sequence the earlier test + (single build) did not cover, with a nested `*` scratch dir as a guard.""" + import json + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + (corpus / "src").mkdir(parents=True) + (corpus / "src" / "a.py").write_text("def alpha(): return 1\n", encoding="utf-8") + assert _rebuild_code(corpus, acquire_lock=False) is True + + # Add a brand-new file and a brand-new nested directory after the first build, + # plus a scratch dir that ignores only itself. + (corpus / "src" / "new.py").write_text("def added(): return 2\n", encoding="utf-8") + (corpus / "monitor").mkdir() + (corpus / "monitor" / "dash.py").write_text("def board(): return 3\n", encoding="utf-8") + (corpus / "scratch").mkdir() + (corpus / "scratch" / ".gitignore").write_text("*\n", encoding="utf-8") + (corpus / "scratch" / "junk.py").write_text("x = 1\n", encoding="utf-8") + + assert _rebuild_code(corpus, acquire_lock=False) is True + + sources = {n.get("source_file", "") for n in + json.loads((corpus / "graphify-out" / "graph.json").read_text())["nodes"]} + assert any("src/new.py" in s for s in sources), "new file not discovered by update (#1837)" + assert any("monitor/dash.py" in s for s in sources), "new directory not discovered (#1837)" + assert not any("scratch/junk.py" in s for s in sources) + + def test_rebuild_honors_persisted_excludes(tmp_path): """#1886: `--exclude` recorded at extract time must survive into update/watch/ hook rebuilds. Before the fix only the initial scan applied the excludes, so From 4f6106a2f1fbf1628307464ef4a4fe66432c7841 Mon Sep 17 00:00:00 2001 From: mzt006 Date: Tue, 14 Jul 2026 19:51:56 +0500 Subject: [PATCH 15/42] fix(cli): send missing-skill warning to stderr --- graphify/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphify/__main__.py b/graphify/__main__.py index 82e422e0d..c5151b8e3 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -174,7 +174,7 @@ def _check_skill_version(skill_dst: Path) -> None: except OSError: return if not skill_exists: - print(" warning: skill dir exists but SKILL.md is missing. Run 'graphify install' to repair.") + print(" warning: skill dir exists but SKILL.md is missing. Run 'graphify install' to repair.", file=sys.stderr) return # A progressive SKILL.md links to its references/ sidecar. If the body points # at references/ but the dir is gone (manual delete, partial upgrade), the From 8361a81f7315cf34f53d489199eec22cf4d45a39 Mon Sep 17 00:00:00 2001 From: xkam7ar Date: Mon, 13 Jul 2026 22:51:17 -0700 Subject: [PATCH 16/42] fix(extract): recognize uppercase TypeScript --- graphify/extract.py | 5 +++-- tests/test_typescript_module_extensions.py | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index a82b074fb..51e249077 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -1051,9 +1051,10 @@ def extract_python(path: Path) -> dict: def extract_js(path: Path) -> dict: """Extract classes, functions, arrow functions, and imports from a .js/.ts/.tsx/.mts/.cts file.""" - if path.suffix == ".tsx": + suffix = path.suffix.lower() + if suffix == ".tsx": config = _TSX_CONFIG - elif path.suffix in (".ts", ".mts", ".cts"): + elif suffix in (".ts", ".mts", ".cts"): config = _TS_CONFIG else: config = _JS_CONFIG diff --git a/tests/test_typescript_module_extensions.py b/tests/test_typescript_module_extensions.py index 41091a552..33c8493d2 100644 --- a/tests/test_typescript_module_extensions.py +++ b/tests/test_typescript_module_extensions.py @@ -78,6 +78,13 @@ def test_cts_uses_the_typescript_grammar(tmp_path): } +def test_uppercase_typescript_extensions_use_typescript_grammar(tmp_path): + for ext in (".TS", ".TSX", ".MTS", ".CTS"): + labels = _labels(_extract(tmp_path, ext)) + assert any("Mode" in label for label in labels), f"TS `type` alias missing for {ext}" + assert any("Options" in label for label in labels), f"TS `interface` missing for {ext}" + + def test_mts_cts_route_to_extract_js(): from graphify.extract import _DISPATCH, extract_js assert _DISPATCH.get(".mts") is extract_js From 5c0a04c9917406d09468fcdb87d9ec15ef6b04d7 Mon Sep 17 00:00:00 2001 From: kebwlmbhee Date: Tue, 14 Jul 2026 10:14:30 +0800 Subject: [PATCH 17/42] fix(extract): filter Kotlin builtin/stdlib types from references graph _kotlin_collect_type_refs had no equivalent of _JAVA_BUILTIN_TYPES / _PYTHON_ANNOTATION_NOISE / _GO_PREDECLARED_TYPES, so every Boolean/String/ Int/List/... type annotation in a .kt file became a real graph node/edge. Unrelated files sharing only these language-level types were getting merged into the same community by cluster-only. Add _KOTLIN_BUILTIN_TYPES (kotlin.* scalars, collections, throwables - deliberately not Android/JVM framework types like Context/View/Bundle, matching how _JAVA_BUILTIN_TYPES doesn't filter framework types either) and check it plus _JAVA_BUILTIN_TYPES (Kotlin freely references java.util.Date/UUID/etc) at the three call sites that previously appended unconditionally. Deliberately excludes "Result" from the filter list even though kotlin.Result exists - it collides with the very common user-defined sealed-class name (confirmed against this repo's own sample.kt fixture, which defines its own Result). Verified on a ~400-file Android/Kotlin project: 3693->3239 nodes (-12%), 6481->5221 edges (-19%), with a previously merged 5-file grab-bag community cleanly splitting out an unrelated screen-dimension utility. --- graphify/extractors/engine.py | 35 ++++++++++++++++++++++++++++++++--- tests/test_languages.py | 21 +++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/graphify/extractors/engine.py b/graphify/extractors/engine.py index 90c95cc49..723bef7ca 100644 --- a/graphify/extractors/engine.py +++ b/graphify/extractors/engine.py @@ -607,6 +607,35 @@ def _php_method_return_type_node(method_node): return c return None +# Kotlin stdlib scalar/collection/core types that appear constantly as type +# annotations but carry no useful semantic meaning as graph nodes (mirrors +# _JAVA_BUILTIN_TYPES / _PYTHON_ANNOTATION_NOISE / _GO_PREDECLARED_TYPES). +# Kotlin compiles to the JVM and freely references java.* types too, so this +# is combined with _JAVA_BUILTIN_TYPES at the call site rather than duplicated. +_KOTLIN_BUILTIN_TYPES = frozenset({ + # kotlin — scalars & core + "Any", "Unit", "Nothing", "Boolean", "Byte", "Short", "Int", "Long", + "Float", "Double", "Char", "String", "CharSequence", "Number", + "Comparable", "Enum", "Annotation", "Pair", "Triple", "Lazy", + "Function", + # kotlin — throwables + "Throwable", "Exception", "RuntimeException", "Error", + "IllegalArgumentException", "IllegalStateException", "NullPointerException", + "IndexOutOfBoundsException", "ClassCastException", "NumberFormatException", + "ArithmeticException", "UnsupportedOperationException", + "NoSuchElementException", "ConcurrentModificationException", + "StackOverflowError", "OutOfMemoryError", "AssertionError", + "InterruptedException", + # kotlin.collections + "Array", "List", "MutableList", "ArrayList", "Set", "MutableSet", + "HashSet", "LinkedHashSet", "Map", "MutableMap", "HashMap", + "LinkedHashMap", "Collection", "MutableCollection", "Iterable", + "MutableIterable", "Iterator", "MutableIterator", "ListIterator", + "MutableListIterator", "Sequence", "Comparator", + # kotlin.text + "Regex", "MatchResult", "StringBuilder", +}) + def _kotlin_user_type_name(user_type_node, source: bytes) -> str | None: """Return the head identifier text from a Kotlin user_type node (without generics).""" if user_type_node is None: @@ -636,14 +665,14 @@ def _kotlin_collect_type_refs(node, source: bytes, generic: bool, out: list[tupl for c in node.children: if c.type in ("identifier", "type_identifier"): text = _read_text(c, source) - if text: + if text and text not in _KOTLIN_BUILTIN_TYPES and text not in _JAVA_BUILTIN_TYPES: out.append((text, "generic_arg" if generic else "type")) break if c.type == "simple_user_type": for sub in c.children: if sub.type in ("identifier", "type_identifier"): text = _read_text(sub, source) - if text: + if text and text not in _KOTLIN_BUILTIN_TYPES and text not in _JAVA_BUILTIN_TYPES: out.append((text, "generic_arg" if generic else "type")) break break @@ -659,7 +688,7 @@ def _kotlin_collect_type_refs(node, source: bytes, generic: bool, out: list[tupl return if t in ("identifier", "type_identifier"): text = _read_text(node, source) - if text: + if text and text not in _KOTLIN_BUILTIN_TYPES and text not in _JAVA_BUILTIN_TYPES: out.append((text, "generic_arg" if generic else "type")) return if t in ("nullable_type", "parenthesized_type", "type_reference"): diff --git a/tests/test_languages.py b/tests/test_languages.py index f610fbc40..f36ea7603 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -656,6 +656,27 @@ def test_kotlin_parameter_return_generic_and_field_contexts(): assert ("run", "DataProcessor") in _edge_labels(r, "references", "generic_arg") assert ("DataProcessor", "Result") in _edge_labels(r, "references", "field") +def test_kotlin_builtin_types_not_emitted_as_references(): + # kotlin.* scalar/collection/core types used as parameter, return, or field + # types carry no useful graph meaning: they never resolve to a project node, + # so emitting `references` edges to them is pure noise (mirrors the Java + # _JAVA_BUILTIN_TYPES / Python _PYTHON_ANNOTATION_NOISE handling). + r = extract_kotlin(FIXTURES / "sample.kt") + ref_targets = {target for (_, target) in _edge_labels(r, "references")} + for builtin in ("String", "Int"): + assert builtin not in ref_targets, ( + f"builtin type {builtin!r} should not be a references target" + ) + +def test_kotlin_user_types_still_emit_references(): + # Guard against over-filtering: a user-defined class sharing its name with a + # common domain-modeling identifier (Result) must still resolve to a real + # edge - the builtin filter is a fixed name list, so it must stay narrow + # enough not to swallow common user-chosen names like a sealed-class "Result". + r = extract_kotlin(FIXTURES / "sample.kt") + assert ("DataProcessor", "Result") in _edge_labels(r, "references", "field") + assert ("run", "DataProcessor") in _edge_labels(r, "references", "parameter_type") + # ── Scala ───────────────────────────────────────────────────────────────────── From e1b8a17dc8dd825601e6d73021ec12e1ddc26592 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Tue, 14 Jul 2026 23:32:28 +0100 Subject: [PATCH 18/42] docs(changelog): record #1881, #1876, #1893 in the unreleased 0.9.16 section Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dec27fae..f57b1682b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.16 (unreleased) +- Fix: uppercase TypeScript extensions (`.TS`/`.TSX`/`.MTS`/`.CTS`) are now parsed with the TypeScript grammar instead of falling through to the JavaScript grammar, which silently dropped interfaces and type aliases (#1881, thanks @xkam7ar). Detection and dispatch already lowercased, but the grammar selection inside `extract_js` compared the suffix case-sensitively. + +- Fix: Kotlin builtin/stdlib types (`String`, `Int`, `List`, ...) are no longer emitted as `references` edges, matching the existing Java/Python/Go builtin filtering (#1876, thanks @kebwlmbhee). They created false coupling and split clusters on real projects. User types that legitimately share a name (`Result`, framework types) are deliberately not filtered, consistent with the other languages. + +- Fix: the stale/missing-skill version warning now prints to stderr, not stdout, so it no longer pollutes the machine-readable output of `graphify query`/`path` (#1805 / #1893, thanks @Mzt00). Every sibling warning in that check already used stderr. + - Fix: Python qualified calls to an imported module (`module.function()`) now produce a `calls` edge, matching bare-name calls (#1883). The call was captured correctly but fell through a gap: the shared cross-file pass skips every member call (to avoid god-node blowups from bare method names), and the Python member-call resolver only handled capitalized class receivers (`ClassName.method()`), so a lowercase module receiver was dropped. The resolver now also resolves a lowercase receiver that names a module imported into the caller's file, linking to the single callable that module contains (guarded so `self`/`obj`/local instances and ambiguous names never create a false edge). - Fix: `--exclude` patterns now survive into `update`/`watch`/git-hook rebuilds instead of applying only to the initial scan (#1886). The patterns were never persisted, so the first rebuild re-ran `detect()` without them and silently re-indexed the excluded paths. `extract` now records them in a `graphify-out/.graphify_build.json` sidecar and `_rebuild_code` re-applies them on every rebuild (they still layer after `.gitignore`/`.graphifyignore`/`.git/info/exclude`, so they keep winning). Graphs built before this release simply have no sidecar and behave as before. From b3dc15b839da0aa1a7f899150cd020bb3944669f Mon Sep 17 00:00:00 2001 From: safishamsi Date: Tue, 14 Jul 2026 23:42:06 +0100 Subject: [PATCH 19/42] fix(extract): close residual absolute-path/username leaks (#1899) Completes #1789. Two residual leaks into committed graph.json: (a) Out-of-root reference targets (a .csproj ProjectReference, .sln project, or bash `source` pointing outside the scan root) kept an absolute source_file and an absolute-derived id, because the relativization post-passes only handled paths under root and silently left out-of-root ones absolute. They now get a portable walk-up relative source_file and an `ext_`-namespaced id (basename fallback for far-outside or cross-drive targets), with edge endpoints remapped. (b) A symbol whose name normalizes to nothing (minified `$`, a JSONC `"//"` key) collapsed _make_id(stem, name) to the bare absolute file stem, leaking the path and colliding with the file node. Guard at the three mint sites (json_config keys, engine function names) to skip these no-signal symbols. Adds regression tests: an out-of-root ProjectReference stays portable (no scan-path in any id/source_file/edge), and a minified `$` is dropped while the real function survives. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 ++ graphify/extract.py | 42 +++++++++++++++++++++++++++++- graphify/extractors/engine.py | 11 ++++++++ graphify/extractors/json_config.py | 7 +++++ tests/test_dotnet.py | 29 +++++++++++++++++++++ tests/test_extract.py | 17 ++++++++++++ 6 files changed, 107 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f57b1682b..b276d1ea4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.16 (unreleased) +- Fix: close two residual paths where an absolute scan path (including the OS username) still leaked into a committed `graph.json`, completing #1789 (#1899). (a) A reference target outside the scan root (an out-of-root `.csproj` ProjectReference, `.sln` project, or bash `source`) kept its absolute `source_file` and an absolute-derived id, because the relativization post-passes silently skipped anything `relative_to(root)` could not handle; such targets now get a portable walk-up relative path and an `ext_`-namespaced id (bare basename when the target is far outside the corpus or on another drive). (b) A symbol whose name normalizes to nothing (a minified `$` function, a JSONC `"//"` comment key) collapsed `_make_id(stem, name)` down to the bare absolute file stem; those no-signal symbols are now skipped at mint time. + - Fix: uppercase TypeScript extensions (`.TS`/`.TSX`/`.MTS`/`.CTS`) are now parsed with the TypeScript grammar instead of falling through to the JavaScript grammar, which silently dropped interfaces and type aliases (#1881, thanks @xkam7ar). Detection and dispatch already lowercased, but the grammar selection inside `extract_js` compared the suffix case-sensitively. - Fix: Kotlin builtin/stdlib types (`String`, `Int`, `List`, ...) are no longer emitted as `references` edges, matching the existing Java/Python/Go builtin filtering (#1876, thanks @kebwlmbhee). They created false coupling and split clusters on real projects. User types that legitimately share a name (`Result`, framework types) are deliberately not filtered, consistent with the other languages. diff --git a/graphify/extract.py b/graphify/extract.py index 51e249077..d5faa5235 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4891,7 +4891,31 @@ def _has_import_evidence(candidate_id: str) -> bool: # a new language plugs in without editing this body (#1356 Swift, #1446 Python). run_language_resolvers(paths, per_file, all_nodes, all_edges) - # Relativize source_file fields so paths are portable across machines (#555) + # Relativize source_file fields so paths are portable across machines (#555). + # A target OUTSIDE the scan root (an out-of-root ProjectReference/.sln/bash + # `source`) can't be made relative to root; leaving it absolute leaked the + # scan path including the OS username into a committed graph.json (#1899). + # Fall back to a walk-up relative form, or the bare basename when that would + # still embed foreign path segments (a far-away or cross-drive target). When + # the node's id was itself minted from the absolute path, remap it to a + # portable id and rewrite the edge endpoints that reference it. + def _portable_out_of_root_sf(p: Path) -> str: + try: + rel = os.path.relpath(str(p), str(root)).replace("\\", "/") + except ValueError: + return p.name # different Windows drive: no relative path exists + updepth = 0 + for seg in rel.split("/"): + if seg == "..": + updepth += 1 + else: + break + # More than a couple of walk-ups means the target lives well outside the + # corpus; its ancestor dirs would embed foreign (possibly user-named) + # segments, so collapse to the basename. + return p.name if updepth > 3 else rel + + ext_id_remap: dict[str, str] = {} for item in all_nodes + all_edges: sf = item.get("source_file") if not sf: @@ -4901,8 +4925,24 @@ def _has_import_evidence(candidate_id: str) -> bool: continue try: item["source_file"] = sf_path.relative_to(root).as_posix() + continue except ValueError: pass + portable = _portable_out_of_root_sf(sf_path) + # A node whose id was minted from this absolute path also leaks it. + if "id" in item and item.get("id") == _make_id(str(sf_path)): + ext_id_remap[item["id"]] = _make_id("ext", portable) + item["source_file"] = portable + + if ext_id_remap: + for n in all_nodes: + if n.get("id") in ext_id_remap: + n["id"] = ext_id_remap[n["id"]] + for e in all_edges: + if e.get("source") in ext_id_remap: + e["source"] = ext_id_remap[e["source"]] + if e.get("target") in ext_id_remap: + e["target"] = ext_id_remap[e["target"]] # origin_file is an internal disambiguation hint (#1462): the colliding-id pass # above reads it to keep same-named cross-file stubs distinct, after which nothing diff --git a/graphify/extractors/engine.py b/graphify/extractors/engine.py index 723bef7ca..edc624a8e 100644 --- a/graphify/extractors/engine.py +++ b/graphify/extractors/engine.py @@ -4,6 +4,7 @@ import hashlib import importlib from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS, _file_stem, _make_id, _read_text +from graphify.ids import normalize_id from graphify.extractors.models import LanguageConfig from graphify.extractors.resolution import _resolve_js_import_target from graphify.security import sanitize_metadata @@ -1760,6 +1761,11 @@ def _js_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str, if name_node: func_name = _read_text(name_node, source) line = child.start_point[0] + 1 + # A name that normalizes to nothing (e.g. minified `$`) + # would collapse the id to the absolute file-stem and + # leak the scan path (#1899); skip it (no graph signal). + if not normalize_id(func_name): + continue func_nid = _make_id(stem, func_name) add_node_fn(func_nid, f"{func_name}()", line) add_edge_fn(file_nid, func_nid, "contains", line) @@ -3124,6 +3130,11 @@ def _emit_java_parent_type(type_node, rel: str, at_line: int) -> None: if not func_name: return + # A name that normalizes to nothing collapses `_make_id(prefix, name)` + # onto the (absolute-path-derived) prefix, leaking the scan path and + # colliding with the file/class node (#1899). No graph signal; skip. + if not normalize_id(func_name): + return line = node.start_point[0] + 1 if parent_class_nid: diff --git a/graphify/extractors/json_config.py b/graphify/extractors/json_config.py index 90dab918e..6a9b641a9 100644 --- a/graphify/extractors/json_config.py +++ b/graphify/extractors/json_config.py @@ -4,6 +4,7 @@ from pathlib import Path from graphify.extractors.base import _file_stem, _make_id, _read_text +from graphify.ids import normalize_id _CONFIG_JSON_NAMES = frozenset({ @@ -140,6 +141,12 @@ def walk_object(obj_node, parent_nid: str, parent_key: str | None, key = _key_text(child) if not key: continue + # A key that normalizes to nothing (a JSONC `"//"` comment key, say) + # would collapse `_make_id(stem, key)` down to the bare file-stem id, + # which is absolute-path-derived and leaks the scan path (#1899). Such + # keys carry no graph signal, so drop them. + if not normalize_id(key): + continue key_nid = _make_id(stem, *(([parent_key] if parent_key else []) + [key])) if not key_nid: continue diff --git a/tests/test_dotnet.py b/tests/test_dotnet.py index 37e0ba511..fac4fd0e6 100644 --- a/tests/test_dotnet.py +++ b/tests/test_dotnet.py @@ -126,6 +126,35 @@ def test_csproj_project_references(): assert len(imports) == 6 # 4 packages + 2 project refs +def test_csproj_out_of_root_reference_id_is_portable(tmp_path): + """#1899: a ProjectReference to a project OUTSIDE the scan root must not leak + the absolute scan path (including the OS username) into the node id or + source_file. The out-of-root target gets a portable, `ext_`-namespaced id and + a walk-up relative source_file rather than the absolute-derived form.""" + web = tmp_path / "WebApi"; web.mkdir() + core = tmp_path / "Core"; core.mkdir() + (core / "Core.csproj").write_text( + '' + 'net8.0' + ) + (web / "WebApi.csproj").write_text( + '' + '' + ) + result = extract([web / "WebApi.csproj"], cache_root=web) + marker = str(tmp_path) + for n in result["nodes"]: + assert marker not in n["id"], f"absolute path leaked into id: {n}" + assert marker not in (n.get("source_file") or ""), f"leaked into source_file: {n}" + for e in result["edges"]: + for f in ("source", "target", "source_file"): + assert marker not in str(e.get(f, "")), f"leaked into edge {f}: {e}" + core_ref = [n for n in result["nodes"] if "core" in n["id"].lower()] + assert core_ref, "out-of-root Core reference node missing" + assert core_ref[0]["id"].startswith("ext_") + assert core_ref[0]["source_file"] == "../Core/Core.csproj" + + def test_csproj_target_framework(): r = extract_csproj(FIXTURES / "sample.csproj") assert "net8.0" in _labels(r) diff --git a/tests/test_extract.py b/tests/test_extract.py index d588b2151..32c8e8e13 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -970,6 +970,23 @@ def test_python_qualified_class_method_call_resolves_extracted(tmp_path): assert call_edges[0]["confidence"] == "EXTRACTED" +def test_degenerate_symbol_name_does_not_leak_absolute_id(tmp_path): + """#1899 variant B: a symbol whose name normalizes to nothing (a minified `$` + function, a JSONC `"//"` key) must not be minted — `_make_id(stem, "")` + collapses to the bare, absolute-path-derived file stem, leaking the scan path + and colliding with the file node. Such nodes carry no graph signal.""" + (tmp_path / "vendor.js").write_text( + "function $(){return 1}\nfunction real(){return 2}\n", encoding="utf-8" + ) + result = extract([tmp_path / "vendor.js"], cache_root=tmp_path) + marker = str(tmp_path) + for n in result["nodes"]: + assert marker not in n["id"], f"absolute path leaked into id: {n}" + labels = {n.get("label") for n in result["nodes"]} + assert "real()" in labels, "the real function must still be extracted" + assert "$()" not in labels, "the degenerate `$` symbol must be dropped (#1899)" + + def test_python_module_qualified_call_resolves_extracted(tmp_path): """`module.func()` where `module` is imported resolves to the callable that module contains, with an EXTRACTED `calls` edge (#1883). A lowercase module From a4ab6ed3f6986451ee2e5653ee5aa0388865ae5a Mon Sep 17 00:00:00 2001 From: safishamsi Date: Tue, 14 Jul 2026 23:45:46 +0100 Subject: [PATCH 20/42] fix(llm): reconcile dispatched vs returned files in semantic extract (#1890) A semantic chunk can return a clean, non-empty response that omits some of the documents it was given. Those docs vanished from the graph with no node, no warning, and no cache/manifest stamp, so they were silently re-dispatched (and typically re-omitted) on every run. extract_corpus_ parallel now diffs the dispatched file set against the source_files that returned, records the gap in merged["uncovered_files"], and prints a loud warning naming the omitted files. Smallest visibility guard; routing docs through the deterministic extractor for a guaranteed file node is a separate, larger change. Adds a regression test: a chunk that omits odd-numbered docs is caught and warned, not silently dropped. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 ++ graphify/llm.py | 27 +++++++++++++++++++++++++++ tests/test_chunking.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b276d1ea4..00d0b3f6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.16 (unreleased) +- Fix: semantic extraction now reconciles dispatched files against returned results, so a document the model silently omits is no longer lost without a trace (#1890). A chunk can return a clean, non-empty response that simply leaves out some of the documents it was given; those docs previously produced no node, no warning, and no cache/manifest stamp, so they were re-dispatched and re-omitted on every run. `extract_corpus_parallel` now diffs the dispatched file set against the `source_file`s that came back, records the gap in `uncovered_files`, and prints a loud warning listing the omitted files. (This is the visibility guard; routing documents through the deterministic extractor so they always get at least a file node is tracked separately.) + - Fix: close two residual paths where an absolute scan path (including the OS username) still leaked into a committed `graph.json`, completing #1789 (#1899). (a) A reference target outside the scan root (an out-of-root `.csproj` ProjectReference, `.sln` project, or bash `source`) kept its absolute `source_file` and an absolute-derived id, because the relativization post-passes silently skipped anything `relative_to(root)` could not handle; such targets now get a portable walk-up relative path and an `ext_`-namespaced id (bare basename when the target is far outside the corpus or on another drive). (b) A symbol whose name normalizes to nothing (a minified `$` function, a JSONC `"//"` comment key) collapsed `_make_id(stem, name)` down to the bare absolute file stem; those no-signal symbols are now skipped at mint time. - Fix: uppercase TypeScript extensions (`.TS`/`.TSX`/`.MTS`/`.CTS`) are now parsed with the TypeScript grammar instead of falling through to the JavaScript grammar, which silently dropped interfaces and type aliases (#1881, thanks @xkam7ar). Detection and dispatch already lowercased, but the grammar selection inside `extract_js` compared the suffix case-sensitively. diff --git a/graphify/llm.py b/graphify/llm.py index 05e2186ea..d019b704c 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -1987,6 +1987,33 @@ def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: " — see errors above. Partial results returned.", file=sys.stderr, ) + + # Dispatch/return reconciliation (#1890). A chunk can return a clean, non-empty + # response that simply omits some of the documents it was given; those docs then + # vanish from the graph with no node, no warning, and no cache/manifest stamp, so + # they are silently re-dispatched (and re-omitted) forever. Diff the files we + # dispatched against the source_files that actually came back and surface the gap. + dispatched = {unit_path(f) for chunk in chunks for f in chunk} + covered: set[Path] = set() + for n in merged.get("nodes", []): + sf = n.get("source_file") + if sf: + p = Path(sf) + covered.add(p if p.is_absolute() else (root / p)) + uncovered = sorted( + p for p in dispatched + if p.resolve() not in {c.resolve() for c in covered} + ) + merged["uncovered_files"] = [str(p) for p in uncovered] + if uncovered: + shown = ", ".join(p.name for p in uncovered[:5]) + more = f" (+{len(uncovered) - 5} more)" if len(uncovered) > 5 else "" + print( + f"[graphify] WARNING: {len(uncovered)}/{len(dispatched)} dispatched file(s) " + f"produced no nodes and are absent from the graph: {shown}{more}. The model " + "returned a response but omitted them; a re-run will retry them.", + file=sys.stderr, + ) return merged diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 42b4651a5..f0ec4f52e 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -326,6 +326,40 @@ def stray(chunk, **kwargs): assert a_cache and any(n["id"] == "a_ok" for n in a_cache["nodes"]) +def test_omitted_documents_are_reconciled_and_warned(tmp_path, capsys): + """#1890: a chunk can return a clean, non-empty response that omits some of the + documents it was given. Those docs must not vanish silently — the run reports + them in `uncovered_files` and warns, instead of dropping them with no signal.""" + from graphify.llm import extract_corpus_parallel + + docs = [] + for i in range(4): + f = tmp_path / f"doc{i}.md" + f.write_text(f"# Doc {i}\n\nsome content\n", encoding="utf-8") + docs.append(f) + + def omit_odd(chunk, **kwargs): + # Return nodes only for even-numbered docs; a clean response, not a failure. + nodes = [] + for u in chunk: + name = getattr(u, "path", u).name + idx = int(name[len("doc")]) + if idx % 2 == 0: + nodes.append({"id": f"n{idx}", "source_file": name, "file_type": "document"}) + return {"nodes": nodes, "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1} + + with patch("graphify.llm.extract_files_direct", side_effect=omit_odd): + result = extract_corpus_parallel( + docs, backend="kimi", root=tmp_path, + token_budget=None, chunk_size=1, max_concurrency=1, + ) + + uncovered = {Path(p).name for p in result.get("uncovered_files", [])} + assert uncovered == {"doc1.md", "doc3.md"}, f"reconciliation missed omissions: {uncovered}" + err = capsys.readouterr().err + assert "produced no nodes" in err and "doc1.md" in err + + def test_checkpoint_caches_sliced_document_chunks(tmp_path, capsys): """#1870: the checkpoint's allowlist must resolve a FileSlice to its parent path (via unit_path), not read a non-existent `.rel`. An oversized doc is From a0e4a1c6bd3a99edfdd84ad30927003f51face6a Mon Sep 17 00:00:00 2001 From: safishamsi Date: Tue, 14 Jul 2026 23:57:12 +0100 Subject: [PATCH 21/42] docs(changelog): date 0.9.16 for release Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00d0b3f6f..43e7b658e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) -## 0.9.16 (unreleased) +## 0.9.16 (2026-07-14) - Fix: semantic extraction now reconciles dispatched files against returned results, so a document the model silently omits is no longer lost without a trace (#1890). A chunk can return a clean, non-empty response that simply leaves out some of the documents it was given; those docs previously produced no node, no warning, and no cache/manifest stamp, so they were re-dispatched and re-omitted on every run. `extract_corpus_parallel` now diffs the dispatched file set against the `source_file`s that came back, records the gap in `uncovered_files`, and prints a loud warning listing the omitted files. (This is the visibility guard; routing documents through the deterministic extractor so they always get at least a file node is tracked separately.) From 75b7279d92e957bfbbe8dfb2774e9d1ae9c59290 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 00:00:56 +0100 Subject: [PATCH 22/42] docs(readme): point star-history at Graphify-Labs org; X handle to @graphify Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0a883e75b..81685cf72 100644 --- a/README.md +++ b/README.md @@ -835,8 +835,8 @@ See [ARCHITECTURE.md](ARCHITECTURE.md) for module responsibilities and how to ad ---

- - Star History Chart + + Star History Chart

@@ -846,7 +846,7 @@ See [ARCHITECTURE.md](ARCHITECTURE.md) for module responsibilities and how to ad

Discord - X + X Sponsor The Memory Layer

From ef8771e8c120f7427676c3c065fd85f02104e769 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 00:33:15 +0100 Subject: [PATCH 23/42] fix: correct postgres install hint to graphifyy (#1906); classify .skill files as documents (#1901) - pg_introspect: the psycopg-missing ImportError suggested pip install 'graphify[postgres]', but the PyPI package is graphifyy (double-y) - detect: add .skill (Markdown with YAML frontmatter agent files) to DOC_EXTENSIONS so they are no longer dropped as unclassified - extract: route .skill through extract_markdown for the structural quick-scan Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/detect.py | 2 +- graphify/extract.py | 1 + graphify/pg_introspect.py | 2 +- tests/test_detect.py | 4 ++++ tests/test_pg_introspect.py | 4 +++- 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index 6e9d283a5..bc73fbde7 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -28,7 +28,7 @@ class FileType(str, Enum): _MANIFEST_PATH = str(out_path("manifest.json")) CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} -DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.txt', '.rst', '.html', '.yaml', '.yml'} +DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} OFFICE_EXTENSIONS = {'.docx', '.xlsx'} diff --git a/graphify/extract.py b/graphify/extract.py index d5faa5235..0343a5673 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -3896,6 +3896,7 @@ def add_existing_edge(edge: dict) -> None: ".md": extract_markdown, ".mdx": extract_markdown, ".qmd": extract_markdown, + ".skill": extract_markdown, ".pas": extract_pascal, ".pp": extract_pascal, ".dpr": extract_pascal, diff --git a/graphify/pg_introspect.py b/graphify/pg_introspect.py index dc7b2bbf8..2a2cca628 100644 --- a/graphify/pg_introspect.py +++ b/graphify/pg_introspect.py @@ -15,7 +15,7 @@ def introspect_postgres(dsn: str | None = None) -> dict: except ModuleNotFoundError: raise ImportError( "psycopg is required for --postgres. " - "Install with: pip install 'graphify[postgres]'" + "Install with: pip install 'graphifyy[postgres]'" ) try: diff --git a/tests/test_detect.py b/tests/test_detect.py index 2c9160303..374b2552f 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -23,6 +23,10 @@ def test_classify_powershell_manifest(): def test_classify_markdown(): assert classify_file(Path("README.md")) == FileType.DOCUMENT +def test_classify_skill(): + # #1901: .skill agent files (Markdown with YAML frontmatter) were dropped as unclassified. + assert classify_file(Path("10_Orchestrator.skill")) == FileType.DOCUMENT + def test_classify_pdf(): assert classify_file(Path("paper.pdf")) == FileType.PAPER diff --git a/tests/test_pg_introspect.py b/tests/test_pg_introspect.py index 919d7ba40..fb4749b3c 100644 --- a/tests/test_pg_introspect.py +++ b/tests/test_pg_introspect.py @@ -313,8 +313,10 @@ class FakeOperationalError(Exception): def test_pg_introspect_import_error(): """If psycopg is missing, introspect_postgres raises ImportError.""" with patch.dict("sys.modules", {"psycopg": None}): - with pytest.raises(ImportError, match="psycopg is required"): + with pytest.raises(ImportError, match="psycopg is required") as exc_info: introspect_postgres("postgresql://localhost/db") + # #1906: the PyPI package is graphifyy (double-y), so the install hint must match + assert "graphifyy[postgres]" in str(exc_info.value) def test_pg_introspect_uri_forward_slashes(): From 107cea044c6afe123592d4c14a398cf87068d4a6 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 00:35:59 +0100 Subject: [PATCH 24/42] fix(serve): drop German and Romance question stopwords from query terms (#1900) Full-sentence non-English queries picked wrong BFS seeds because _QUERY_STOPWORDS was English-only: in a mostly-English code corpus, fillers like wie/die/das/funktioniert are rare, get high IDF weight, and out-seed the real keyword (~175x score collapse). Extend the frozenset with a curated German set plus trimmed French/ Spanish/Portuguese/Italian question/filler words, diacritics intact. English-collision words (war, bald, comment, come, son, sin, con, pour, des) are deliberately omitted; die/hat are kept since the all-stopword fallback and unfiltered find_node protect English use. Filtering stays in _query_terms, so the per-term seed guarantee in _pick_seeds composes unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/serve.py | 47 ++++++++++++++++++++++++++++++++++++++++----- tests/test_serve.py | 33 +++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/graphify/serve.py b/graphify/serve.py index 33db0108f..c94b048fb 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -107,14 +107,30 @@ def _is_searchable(term: str) -> bool: return True -# English question/filler words dropped from query terms so content words drive -# BFS seeding. Without this, "how does the frontier cache work" seeds on "how"/ +# Question/filler words dropped from query terms so content words drive BFS +# seeding. Without this, "how does the frontier cache work" seeds on "how"/ # "the"/"work" (which prefix-match prose labels like "Working Principles" at 100x) # instead of "frontier"/"cache", and lands in the wrong part of the graph. Applied # to query terms only — node text is never filtered, so a symbol literally named # `work` stays findable via explain/path. `work`/`works`/`working` are included # because "how does X work" / "how X works" is the most common question phrasing. +# +# Non-English question words are just as damaging (#1900): in a mostly-English +# code corpus, German "wie"/"funktioniert" are rare, so they get HIGH IDF weight +# and out-seed the actual content noun by orders of magnitude. So this also +# carries a curated German set plus a trimmed French/Spanish/Portuguese/Italian +# set of question/filler words. Diacritics are kept intact (the query tokenizer +# does not NFKD-strip). +# +# Collision tradeoff: a few foreign stopwords are also English content words. +# We include high-German-value ones like "die"/"hat" (the all-stopword fallback +# in _query_terms and the unfiltered find_node path keep an English "die"/"hat" +# query workable), but deliberately OMIT "war"/"bald" (German was/soon) so +# English queries about "war" or "bald" are not clobbered. On the Romance side +# we likewise omit "comment" (FR how), "come" (IT how), "son"/"sin"/"con" (ES), +# and "pour"/"des" (FR) — all too common as English/code terms. _QUERY_STOPWORDS = frozenset({ + # English "how", "what", "why", "when", "where", "which", "who", "whom", "whose", "does", "did", "is", "are", "was", "were", "be", "been", "being", "can", "could", "should", "would", "will", "shall", "may", "might", "must", @@ -122,14 +138,35 @@ def _is_searchable(term: str) -> bool: "without", "into", "onto", "off", "that", "this", "these", "those", "there", "here", "its", "their", "them", "they", "about", "any", "all", "some", "work", "works", "working", + # German (articles/conjunctions/question words/auxiliaries/prepositions) + "der", "die", "das", "den", "dem", "ein", "eine", "und", "oder", "nicht", + "wie", "wer", "wann", "wo", "warum", "wieso", + "welche", "welcher", "welches", + "ist", "sind", "wird", "wurde", "hat", "haben", + "kann", "koennen", "können", "soll", "muss", "sich", + "bei", "mit", "von", "fuer", "für", "ueber", "über", "nach", "aus", + "gibt", "es", + "funktioniert", "geaendert", "geändert", "aendert", "ändert", + # French + "pourquoi", "quand", "quel", "quelle", "quels", "quelles", "quoi", + "qui", "que", "est", "sont", "fonctionne", "cette", "dans", "avec", "où", + # Spanish + "cómo", "como", "qué", "cuál", "cuáles", "cuándo", "dónde", "donde", + "porque", "por", "para", "funciona", "está", "están", "hay", + # Portuguese + "qual", "quais", "quando", "onde", "são", "estão", "tem", "uma", "não", + # Italian + "perché", "cosa", "quale", "quali", "dove", "funziona", "sono", "che", + "della", }) def _query_terms(question: str) -> list[str]: """Split a query into searchable terms, segmenting Chinese text, then drop - English question/filler words (`_QUERY_STOPWORDS`) so content words drive - seeding. Falls back to the unfiltered terms if the query is all stopwords, so - a question like "how does it work" still seeds on something.""" + question/filler words (`_QUERY_STOPWORDS`, English plus common German/ + Romance-language fillers) so content words drive seeding. Falls back to the + unfiltered terms if the query is all stopwords, so a question like "how does + it work" or "wie funktioniert das" still seeds on something.""" terms: list[str] = [] for raw in question.split(): if _has_chinese(raw): diff --git a/tests/test_serve.py b/tests/test_serve.py index 859cde026..3a9656124 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -364,6 +364,39 @@ def test_query_terms_all_stopwords_falls_back_to_unfiltered(): assert _query_terms("how does it work") == ["how", "does", "work"] +def test_query_terms_drops_german_question_stopwords(): + # #1900: German full-sentence queries must reduce to the content noun. + # In a mostly-English corpus "wie"/"funktioniert" are rare, get high IDF + # weight, and out-seed the actual keyword unless dropped here. + assert _query_terms("Wie funktioniert die Authentifizierung?") == ["authentifizierung"] + + +def test_query_terms_all_german_stopwords_falls_back_to_unfiltered(): + # Existing all-stopword fallback applies to German fillers too: the query + # keeps its terms rather than seeding on nothing. + terms = _query_terms("wie funktioniert das") + assert terms == ["wie", "funktioniert", "das"] + + +def test_pick_seeds_german_query_seeds_content_node_not_heading_noise(): + """End-to-end for #1900: a German question over a graph with German + heading-noise nodes must seed on the content noun, not on nodes that + happen to contain 'die'/'wie'/'wird'.""" + G = nx.DiGraph() + G.add_node("cfg", label="Die Konfiguration", source_file="docs/konfiguration.md") + G.add_node("sec", label="Wie wird gesichert", source_file="docs/sicherheit.md") + G.add_node("auth", label="Authentifizierung", source_file="src/auth.py") + G.add_node("helper", label="login_helper", source_file="src/auth.py") + G.add_edge("helper", "auth") + + q = "Wie funktioniert die Authentifizierung?" + terms = _query_terms(q) + seeds = _pick_seeds(_score_nodes(G, terms), G=G, terms=terms) + assert "auth" in seeds + assert "cfg" not in seeds + assert "sec" not in seeds + + def test_query_terms_filters_only_short_english_terms(monkeypatch): import graphify.serve as serve_mod From 876c043c2fe70cb84462c114dff733a4970a56fe Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 00:33:14 +0100 Subject: [PATCH 25/42] fix(export): prune orphaned Obsidian notes on re-export (#1896) Re-exporting into an existing vault left notes for nodes that dropped out of the graph, and rewriting the manifest to only this run's files disowned those orphans - so a returning node's stale note became permanently unwritable. Before rewriting the manifest, delete stale = owned - written - skipped. This only ever touches files graphify itself wrote (foreign files go to _skipped, never the manifest), and each path is containment-checked against the vault dir to defuse a corrupt/hostile manifest with ../ entries. The manifest rewrite then correctly drops the pruned files. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/export.py | 22 +++++++++++++++++ tests/test_export.py | 58 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/graphify/export.py b/graphify/export.py index 0e95b467f..2490249e2 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -690,6 +690,28 @@ def _community_name(cid) -> str: } _owned_write(".obsidian/graph.json", json.dumps(graph_config, indent=2)) + # #1896: prune notes for nodes that dropped out of the graph. Only files the + # manifest says graphify owns are candidates, and anything written or skipped + # this run is excluded — so a user's own note is never touched (foreign files + # land in _skipped, never _owned). Guard each path to stay inside the vault in + # case a corrupt/hostile manifest contains `../` entries. + stale = _owned - set(_written) - set(_skipped) + pruned = 0 + for rel_name in sorted(stale): + target = (out / rel_name).resolve() + if out.resolve() not in target.parents: + continue + try: + target.unlink(missing_ok=True) + pruned += 1 + except OSError: + pass + if pruned: + print( + f"[graphify] pruned {pruned} note(s) for nodes no longer in the graph", + file=sys.stderr, + ) + # Persist the manifest of files graphify owns, so a re-run can safely update its # own notes while still refusing to touch the user's. Warn (once, aggregated) # about anything skipped to avoid clobbering a pre-existing file. diff --git a/tests/test_export.py b/tests/test_export.py index 645659ac5..05262b461 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -481,6 +481,64 @@ def test_to_obsidian_rerun_updates_own_notes_but_not_user_files(): assert (out / "UserNote.md").read_text().strip() == "mine" # user's untouched +def _four_node_two_community_graph(): + import networkx as nx + G = nx.Graph() + G.add_node("n1", label="Database", community=0, source_file="app/db.py", type="code") + G.add_node("n2", label="Server", community=0, source_file="app/srv.py", type="code") + G.add_node("n3", label="Cache", community=1, source_file="infra/cache.py", type="code") + G.add_node("n4", label="Queue", community=1, source_file="infra/queue.py", type="code") + G.add_edge("n1", "n2") + G.add_edge("n3", "n4") + return G, {0: ["n1", "n2"], 1: ["n3", "n4"]} + + +def test_to_obsidian_rerun_prunes_removed_nodes(): + """#1896: re-exporting into the same vault must delete graphify's own notes for + nodes (and communities) that dropped out of the graph, so the vault mirrors the + current graph rather than old-union-new. User files are never touched.""" + G4, comm4 = _four_node_two_community_graph() + G2, comm2 = _two_node_graph() + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "obsidian" + to_obsidian(G4, comm4, str(out), community_labels={0: "Backend", 1: "Infra"}) + assert (out / "Cache.md").exists() and (out / "_COMMUNITY_Infra.md").exists() + (out / "MyOwnNote.md").write_text("mine\n", encoding="utf-8") + to_obsidian(G2, comm2, str(out), community_labels={0: "Backend"}) + # notes for removed nodes and the stale community overview are pruned + assert not (out / "Cache.md").exists() + assert not (out / "Queue.md").exists() + assert not (out / "_COMMUNITY_Infra.md").exists() + # surviving graphify notes and the user's own note remain + assert (out / "Database.md").exists() and (out / "Server.md").exists() + assert (out / "_COMMUNITY_Backend.md").exists() + assert (out / "MyOwnNote.md").read_text().strip() == "mine" + + +def test_to_obsidian_removed_node_returning_is_writable_again(capsys): + """#1896 follow-on: a node that disappears and later returns must be writable + again. Before the fix, the manifest was rewritten to only this run's files, so + the orphaned note was disowned and the returning node's write was skipped as a + 'pre-existing user file' forever.""" + import networkx as nx + GA, commA = _two_node_graph() + GB = nx.Graph() + GB.add_node("n1", label="Database", community=0, source_file="app/db.py", type="code") + commB = {0: ["n1"]} + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "obsidian" + to_obsidian(GA, commA, str(out), community_labels={0: "Backend"}) + to_obsidian(GB, commB, str(out), community_labels={0: "Backend"}) + assert not (out / "Server.md").exists() # pruned while absent + capsys.readouterr() + to_obsidian(GA, commA, str(out), community_labels={0: "Backend"}) + # returned node's note exists with current content, written this run + assert (out / "Server.md").exists() + assert "# Server" in (out / "Server.md").read_text() + captured = capsys.readouterr() + assert "skipped" not in captured.err.lower() + + # ── Case-only-distinct labels must not collide on case-insensitive filesystems ── def _case_collision_graph(): From f2a6139667a0f53f639477a7a5ed19a75015b0c3 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 00:38:35 +0100 Subject: [PATCH 26/42] Fix spurious hooksPath warning and register graph.json merge driver Fix #1907: _hooks_dir parsed .git/config with a strict configparser, which rejects git-legal duplicate keys/sections (VS Code writes such configs) and printed a spurious "could not read core.hooksPath" warning on every hook command. Ask git itself (rev-parse --git-path hooks) as the primary resolution instead; it handles core.hooksPath, includeIf and linked worktrees, keeps the Windows-path and multiline/NUL guards, and surfaces git's own stderr when the config is genuinely corrupt. The .git/hooks final fallback is unchanged. Fix #1902: hook install never registered the graph.json union merge driver that README and CHANGELOG 0.7.0 document. install() now sets merge.graphify.name/driver via git config (pinning sys.executable with the same shell-safe allowlist the hook scripts use, falling back to the graphify launcher) and idempotently appends the graph.json merge=graphify line to .gitattributes without clobbering other entries. uninstall() mirrors the removal, and status()/install() report the merge-driver state. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/hooks.py | 238 +++++++++++++++++++++++++++++++++----------- tests/test_hooks.py | 101 +++++++++++++++++++ 2 files changed, 279 insertions(+), 60 deletions(-) diff --git a/graphify/hooks.py b/graphify/hooks.py index 14f8b8f81..77ef39cd3 100644 --- a/graphify/hooks.py +++ b/graphify/hooks.py @@ -1,6 +1,5 @@ # git hook integration - install/uninstall graphify post-commit and post-checkout hooks from __future__ import annotations -import configparser import os import re import sys @@ -398,38 +397,17 @@ def _reject_windows_path(value: str, source: str) -> None: def _hooks_dir(root: Path) -> Path: - """Return the git hooks directory, respecting core.hooksPath if set (e.g. Husky).""" - try: - cfg = configparser.RawConfigParser() - cfg.read(root / ".git" / "config", encoding="utf-8") - # configparser lowercases option names; git's hooksPath becomes hookspath - custom = cfg.get("core", "hookspath", fallback="").strip() - if custom: - _reject_windows_path(custom, "core.hooksPath") - p = Path(custom).expanduser() - if not p.is_absolute(): - p = root / p - # Validate the resolved path stays within the repository root - # to prevent supply-chain attacks via malicious core.hooksPath values - try: - p.resolve().relative_to(root.resolve()) - except ValueError: - pass # Path escapes repo root; fall through to default .git/hooks - else: - p.mkdir(parents=True, exist_ok=True) - return p - except (configparser.Error, OSError) as exc: - # Narrow the exception (PR747-NEW-2): a bare `except Exception: pass` - # was hiding tampering signals (corrupt .git/config, permission flips - # by another tool). Surface them on stderr instead of silently - # falling through to the default hooks directory. - print( - f"[graphify hooks] could not read core.hooksPath from " - f"{root / '.git' / 'config'}: {exc}", - file=sys.stderr, - ) - # In a linked worktree .git is a file not a directory, so constructing - # root/.git/hooks directly fails. Ask git for the real hooks path instead. + """Return the git hooks directory, respecting core.hooksPath if set (e.g. Husky). + + Asks git itself via ``rev-parse --git-path hooks`` rather than parsing + ``.git/config`` with configparser: git legally allows duplicate keys and + sections (VS Code writes such configs), which a strict configparser rejects + with DuplicateOptionError/DuplicateSectionError, so every hook command + printed a spurious "could not read core.hooksPath" warning (#1907). git + resolves core.hooksPath, includeIf, and linked worktrees (where .git is a + file, not a directory) correctly in one place. Genuinely corrupt configs + are still surfaced: git itself fails on them, and its stderr is printed. + """ # NOTE: do NOT pass --path-format=absolute — added in git 2.31; older git # echoes it back as a literal argument, contaminating stdout and causing a # phantom directory to be created (#907). git -C already returns an @@ -441,14 +419,25 @@ def _hooks_dir(root: Path) -> Path: ["git", "-C", str(root), "rev-parse", "--git-path", "hooks"], capture_output=True, text=True, ) - raw = res.stdout.strip() - # A valid hooks path can never contain newlines or NUL. Their presence - # means git echoed an unrecognised flag back (old git behaviour). - if res.returncode == 0 and raw and not any(c in raw for c in ("\n", "\r", "\x00")): - _reject_windows_path(raw, "git rev-parse --git-path hooks") - d = (root / raw).resolve() - d.mkdir(parents=True, exist_ok=True) - return d + if res.returncode != 0: + # git failing here is a real signal (corrupt .git/config, tampering, + # permission flips by another tool). Surface git's own stderr rather + # than silently falling through to the default hooks directory. + err = (res.stderr or "").strip() + print( + f"[graphify hooks] git could not resolve the hooks path for " + f"{root}: {err or f'git exited with code {res.returncode}'}", + file=sys.stderr, + ) + else: + raw = res.stdout.strip() + # A valid hooks path can never contain newlines or NUL. Their presence + # means git echoed an unrecognised flag back (old git behaviour). + if raw and not any(c in raw for c in ("\n", "\r", "\x00")): + _reject_windows_path(raw, "git rev-parse --git-path hooks") + d = (root / raw).resolve() + d.mkdir(parents=True, exist_ok=True) + return d except (OSError, FileNotFoundError): pass d = root / ".git" / "hooks" @@ -491,6 +480,143 @@ def _uninstall_hook(hooks_dir: Path, name: str, marker: str, marker_end: str) -> return f"graphify removed from {name} at {hook_path} (other hook content preserved)" +def _pinned_python() -> str: + """Return sys.executable if its path is shell-safe, else an empty string. + + Applies the same allowlist used in _PYTHON_DETECT: rejects any character + that is not a valid plain filesystem path character, preventing $(...), + backtick, double-quote, semicolon, etc. from being injected into generated + shell scripts or the merge-driver command line. The allowlist includes ':' + and '\\' so Windows paths (C:\\...) are accepted. An empty return means + callers must fall back to the `graphify` launcher on PATH — safe degradation. + """ + if re.search(r"[^a-zA-Z0-9/_.@:\\-]", sys.executable): + return "" + return sys.executable + + +def _merge_attr_line() -> str: + """The .gitattributes line assigning the graphify merge driver to graph.json. + + The graph lives under the configured output directory (graphify.paths, + GRAPHIFY_OUT env override). gitattributes patterns are repo-relative, so an + absolute output-dir override cannot be expressed there — fall back to the + default name in that case. + """ + from graphify.paths import GRAPHIFY_OUT + out = GRAPHIFY_OUT + if not out or Path(out).is_absolute() or "\\" in out: + out = "graphify-out" + return f"{out.rstrip('/')}/graph.json merge=graphify" + + +def _has_merge_attr(content: str) -> bool: + """True if a (non-comment) `<...>graph.json ... merge=graphify` line exists.""" + for raw in content.splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + fields = line.split() + if fields and fields[0].endswith("graph.json") and "merge=graphify" in fields[1:]: + return True + return False + + +def _register_merge_driver(root: Path) -> str: + """Register the graph.json union merge driver in git config + .gitattributes (#1902). + + README and CHANGELOG 0.7.0 document `graphify merge-driver` as being set up + by `hook install`, but install never actually registered it. Writes go + through `git config` (never hand-edit .git/config — in a linked worktree the + effective config is not at root/.git/config). The interpreter is pinned the + same way the hook scripts pin it, so the driver works even when the graphify + launcher is not on PATH at merge time. + """ + import subprocess as _sp + pinned = _pinned_python() + if pinned: + driver = f"{pinned} -m graphify merge-driver %O %A %B" + else: + driver = "graphify merge-driver %O %A %B" + try: + for key, value in ( + ("merge.graphify.name", "graphify graph.json union merge"), + ("merge.graphify.driver", driver), + ): + _sp.run( + ["git", "-C", str(root), "config", key, value], + check=True, capture_output=True, text=True, + ) + except (OSError, _sp.CalledProcessError) as exc: + return f"not registered (git config failed: {exc})" + + line = _merge_attr_line() + attrs = root / ".gitattributes" + if attrs.exists(): + content = attrs.read_text(encoding="utf-8") + if _has_merge_attr(content): + return f"already registered ({line})" + # Never clobber other entries; preserve a trailing newline. + if content and not content.endswith("\n"): + content += "\n" + attrs.write_text(content + line + "\n", encoding="utf-8", newline="\n") + else: + attrs.write_text(line + "\n", encoding="utf-8", newline="\n") + return f"registered ({line})" + + +def _unregister_merge_driver(root: Path) -> str: + """Remove the merge-driver git config keys and the .gitattributes line.""" + import subprocess as _sp + for key in ("merge.graphify.name", "merge.graphify.driver"): + try: + # --unset exits nonzero if the key is absent; that is fine. + _sp.run( + ["git", "-C", str(root), "config", "--unset", key], + capture_output=True, text=True, + ) + except OSError: + pass + attrs = root / ".gitattributes" + if not attrs.exists(): + return "not registered - nothing to remove." + content = attrs.read_text(encoding="utf-8") + kept = [ + raw for raw in content.splitlines() + if not _has_merge_attr(raw) + ] + if kept == content.splitlines(): + return "gitattributes entry not found - nothing to remove." + if kept: + # Other entries survive; the file stays. + attrs.write_text("\n".join(kept) + "\n", encoding="utf-8", newline="\n") + return "removed from .gitattributes (other entries preserved)" + attrs.unlink() + return "removed (.gitattributes deleted - no other entries)" + + +def _merge_driver_status(root: Path) -> str: + """Report whether the merge driver is registered (config + gitattributes).""" + import subprocess as _sp + try: + res = _sp.run( + ["git", "-C", str(root), "config", "--get", "merge.graphify.driver"], + capture_output=True, text=True, + ) + cfg_ok = res.returncode == 0 and bool(res.stdout.strip()) + except OSError: + cfg_ok = False + attrs = root / ".gitattributes" + attr_ok = attrs.exists() and _has_merge_attr(attrs.read_text(encoding="utf-8")) + if cfg_ok and attr_ok: + return "registered" + if cfg_ok: + return "partially registered (git config set, .gitattributes line missing)" + if attr_ok: + return "partially registered (.gitattributes line set, git config missing)" + return "not registered" + + def _user_hooks_dir(hooks_dir: Path) -> Path: """Return the user-editable hooks directory. @@ -516,29 +642,19 @@ def install(path: Path = Path(".")) -> str: # launcher is not on PATH at git-trigger time (uv tool / pipx isolation). # sys.executable is the Python running this very install command, so it is # always the correct isolated-venv interpreter. The placeholder is replaced - # in both scripts before writing; the allowlist in _PYTHON_DETECT strips any - # characters unsafe in a shell path, and import-verification catches a stale - # pinned path so it safely falls through to the dynamic detection. - # Apply the same allowlist used in _PYTHON_DETECT for all other probes. - # This rejects any character that is not a valid plain filesystem path - # character, preventing $(...), backtick, double-quote, semicolon, etc. - # from being injected into the generated shell scripts. The allowlist - # includes ':' and '\' so Windows paths (C:\...) are accepted. - import re as _re - _safe = sys.executable - if _re.search(r"[^a-zA-Z0-9/_.@:\\-]", _safe): - # Path contains characters outside the allowlist (spaces, quotes, etc.). - # Embed an empty string so the pinned probe is skipped and the hook - # falls through to the dynamic detection — safe degradation. - _safe = "" - pinned = _safe + # in both scripts before writing; the allowlist in _pinned_python() strips + # any characters unsafe in a shell path (empty result -> the pinned probe is + # skipped), and import-verification catches a stale pinned path so it safely + # falls through to the dynamic detection. + pinned = _pinned_python() hook = _HOOK_SCRIPT.replace("__PINNED_PYTHON__", pinned) checkout = _CHECKOUT_SCRIPT.replace("__PINNED_PYTHON__", pinned) commit_msg = _install_hook(hooks_dir, "post-commit", hook, _HOOK_MARKER) checkout_msg = _install_hook(hooks_dir, "post-checkout", checkout, _CHECKOUT_MARKER) + merge_msg = _register_merge_driver(root) - return f"post-commit: {commit_msg}\npost-checkout: {checkout_msg}" + return f"post-commit: {commit_msg}\npost-checkout: {checkout_msg}\nmerge driver: {merge_msg}" def uninstall(path: Path = Path(".")) -> str: @@ -550,8 +666,9 @@ def uninstall(path: Path = Path(".")) -> str: hooks_dir = _user_hooks_dir(_hooks_dir(root)) commit_msg = _uninstall_hook(hooks_dir, "post-commit", _HOOK_MARKER, _HOOK_MARKER_END) checkout_msg = _uninstall_hook(hooks_dir, "post-checkout", _CHECKOUT_MARKER, _CHECKOUT_MARKER_END) + merge_msg = _unregister_merge_driver(root) - return f"post-commit: {commit_msg}\npost-checkout: {checkout_msg}" + return f"post-commit: {commit_msg}\npost-checkout: {checkout_msg}\nmerge driver: {merge_msg}" def status(path: Path = Path(".")) -> str: @@ -569,4 +686,5 @@ def _check(name: str, marker: str) -> str: commit = _check("post-commit", _HOOK_MARKER) checkout = _check("post-checkout", _CHECKOUT_MARKER) - return f"post-commit: {commit}\npost-checkout: {checkout}" + merge = _merge_driver_status(root) + return f"post-commit: {commit}\npost-checkout: {checkout}\nmerge driver: {merge}" diff --git a/tests/test_hooks.py b/tests/test_hooks.py index 88199a5de..8e95aabbe 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -488,3 +488,104 @@ def _git(*args, cwd): capture_output=True, text=True) assert "RAN" in r_primary.stdout, "guard wrongly skipped the primary checkout" assert "RAN" not in r_linked.stdout, "guard failed to skip the linked worktree" + + +# ── #1907: duplicate keys in .git/config must not trigger spurious warnings ── + +def _append_duplicate_config_entries(repo: Path) -> None: + """Append git-legal duplicate keys/sections (as VS Code writes them).""" + cfg = repo / ".git" / "config" + cfg.write_text( + cfg.read_text(encoding="utf-8") + + '[remote "origin"]\n' + + "\tfetch = +refs/heads/*:refs/remotes/origin/*\n" + + "\tfetch = +refs/heads/*:refs/remotes/origin/*\n" + + "[core]\n" + + "\tignorecase = true\n", + encoding="utf-8", + ) + + +def test_hooks_dir_no_warning_on_duplicate_config_keys(tmp_path, capsys): + """git legally allows duplicate keys and repeated sections in .git/config; + a strict configparser raised DuplicateOptionError/DuplicateSectionError and + printed a spurious 'could not read core.hooksPath' warning on every hook + command (#1907). _hooks_dir must resolve cleanly with no stderr noise.""" + repo = _make_git_repo(tmp_path) + _append_duplicate_config_entries(repo) + d = _hooks_dir(repo) + err = capsys.readouterr().err + assert "could not read core.hooksPath" not in err + assert d == (repo / ".git" / "hooks").resolve() + + +def test_hooks_dir_duplicate_config_keys_honor_custom_hookspath(tmp_path, capsys): + """With duplicate keys present, a custom core.hooksPath must still be + honored (no fall-through to .git/hooks) and no warning printed (#1907).""" + repo = _make_git_repo(tmp_path) + _set_hookspath(repo, ".husky") + _append_duplicate_config_entries(repo) + d = _hooks_dir(repo) + err = capsys.readouterr().err + assert "could not read core.hooksPath" not in err + assert d == (repo / ".husky").resolve() + + +# ── #1902: hook install must register the graph.json union merge driver ───── + +def test_install_registers_merge_driver(tmp_path): + """install() must set merge.graphify.* via git config and add the + .gitattributes line that README/CHANGELOG 0.7.0 document (#1902).""" + repo = _make_git_repo(tmp_path) + result = install(repo) + res = subprocess.run( + ["git", "-C", str(repo), "config", "--get", "merge.graphify.driver"], + capture_output=True, text=True, + ) + assert res.returncode == 0 + driver = res.stdout.strip() + assert driver + assert "merge-driver %O %A %B" in driver + attrs = (repo / ".gitattributes").read_text(encoding="utf-8") + assert any( + "graph.json" in line and "merge=graphify" in line + for line in attrs.splitlines() + ) + assert "merge driver" in result + + +def test_install_merge_driver_idempotent(tmp_path): + """Running install twice must not duplicate the .gitattributes line.""" + repo = _make_git_repo(tmp_path) + install(repo) + install(repo) + lines = (repo / ".gitattributes").read_text(encoding="utf-8").splitlines() + matches = [l for l in lines if "merge=graphify" in l] + assert len(matches) == 1 + + +def test_install_preserves_existing_gitattributes(tmp_path): + """A pre-existing .gitattributes entry must survive install (no clobber).""" + repo = _make_git_repo(tmp_path) + (repo / ".gitattributes").write_text("*.png binary\n", encoding="utf-8") + install(repo) + content = (repo / ".gitattributes").read_text(encoding="utf-8") + assert "*.png binary" in content + assert "merge=graphify" in content + + +def test_uninstall_removes_merge_driver_keeps_other_attrs(tmp_path): + """uninstall() must unset merge.graphify.* and remove only the graphify + .gitattributes line, keeping the file when other entries exist.""" + repo = _make_git_repo(tmp_path) + (repo / ".gitattributes").write_text("*.png binary\n", encoding="utf-8") + install(repo) + uninstall(repo) + res = subprocess.run( + ["git", "-C", str(repo), "config", "--get", "merge.graphify.driver"], + capture_output=True, text=True, + ) + assert res.returncode != 0 + content = (repo / ".gitattributes").read_text(encoding="utf-8") + assert "*.png binary" in content + assert "merge=graphify" not in content From 0cacd708e9823baff8b77ac506e9292528b6de3d Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 00:39:59 +0100 Subject: [PATCH 27/42] fix(llm): drop out-of-scope nodes from the merged extraction result (#1895) The #1757 cache guard refuses to write a cache entry for a node whose source_file resolves to a real corpus file that was not dispatched, but the node itself still flowed into merged["nodes"] and landed in graph.json (plus any edges/hyperedges built on it). extract_corpus_ parallel now filters the merged result right before the #1890 dispatched-vs-returned reconciliation: a node is dropped when its source_file resolves (against root, same normalization as #1890) to an existing file (.is_file(), mirroring the #1757 condition) outside the dispatched set. Non-file source_files (concepts, model-invented anchors) pass through untouched. Edges whose endpoint and hyperedges whose member is a dropped node id go with it, plus any edge/hyperedge itself attributed to an undispatched real file. One summary warning names the offending files and merged["out_of_scope_dropped"] records the count. Running before the reconciliation keeps covered/uncovered reflecting the post-filter graph. Regression tests: a chunk over A.md+C.md returning a stray B.py node loses the stray (and its edge/hyperedge) while sibling and concept attributions survive; a clean run records a zero count and no warning. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/llm.py | 66 ++++++++++++++++++++++++++++++++ tests/test_chunking.py | 86 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) diff --git a/graphify/llm.py b/graphify/llm.py index d019b704c..52e2b48b5 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -1994,6 +1994,72 @@ def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: # they are silently re-dispatched (and re-omitted) forever. Diff the files we # dispatched against the source_files that actually came back and surface the gap. dispatched = {unit_path(f) for chunk in chunks for f in chunk} + + # Out-of-scope node filter (#1895). The #1757 cache guard already refuses + # to WRITE a cache entry for a node whose source_file is a real file that + # was not dispatched, but the node itself still flowed into the merged + # result and landed in graph.json. Mirror the #1757 condition here: resolve + # each source_file against root and drop the node only when it resolves to + # an existing file (.is_file()) outside the dispatched set — non-file + # source_files (concepts, model-invented anchors) pass through untouched. + # Runs BEFORE the #1890 covered/uncovered reconciliation so that diff + # reflects the post-filter graph. + def _resolve_against_root(value: "str | Path") -> Path: + p = Path(value) + if not p.is_absolute(): + p = root / p + try: + return p.resolve() + except (OSError, RuntimeError): + return p + + _dispatched_resolved = {_resolve_against_root(p) for p in dispatched} + + def _out_of_scope(item: dict) -> bool: + sf = item.get("source_file") + if not sf: + return False + p = _resolve_against_root(sf) + return p.is_file() and p not in _dispatched_resolved + + dropped_ids: set = set() + dropped_files: set[str] = set() + kept_nodes: list[dict] = [] + for n in merged.get("nodes", []): + if _out_of_scope(n): + if n.get("id") is not None: + dropped_ids.add(n.get("id")) + dropped_files.add(str(n.get("source_file"))) + continue + kept_nodes.append(n) + dropped_node_count = len(merged.get("nodes", [])) - len(kept_nodes) + merged["out_of_scope_dropped"] = dropped_node_count + if dropped_node_count: + merged["nodes"] = kept_nodes + # Keep the graph consistent: an edge or hyperedge referencing a + # dropped node's id (or itself attributed to an undispatched real + # file) must not survive its endpoint. + merged["edges"] = [ + e for e in merged.get("edges", []) + if not _out_of_scope(e) + and e.get("source") not in dropped_ids + and e.get("target") not in dropped_ids + ] + merged["hyperedges"] = [ + h for h in merged.get("hyperedges", []) + if not _out_of_scope(h) + and not (dropped_ids & set(h.get("nodes", []) or [])) + ] + shown = ", ".join(sorted(Path(f).name for f in dropped_files)[:5]) + more = f" (+{len(dropped_files) - 5} more)" if len(dropped_files) > 5 else "" + print( + f"[graphify] WARNING: dropped {dropped_node_count} out-of-scope node(s) " + f"attributed to file(s) not dispatched for extraction: {shown}{more}. " + "The model mis-attributed them to another corpus file; they were " + "excluded from the graph (#1895).", + file=sys.stderr, + ) + covered: set[Path] = set() for n in merged.get("nodes", []): sf = n.get("source_file") diff --git a/tests/test_chunking.py b/tests/test_chunking.py index f0ec4f52e..546503ac1 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -360,6 +360,92 @@ def omit_odd(chunk, **kwargs): assert "produced no nodes" in err and "doc1.md" in err +def test_out_of_scope_nodes_are_dropped_from_merged_result(tmp_path, capsys): + """#1895: the #1757 cache guard skips the CACHE write for a node attributed + to a real corpus file that was not dispatched, but the node itself still + flowed into merged["nodes"] and landed in graph.json. The merged result must + drop such nodes (and edges/hyperedges touching them), warn once, and record + the count — while keeping in-scope sibling attributions (a node attributed + to a different dispatched file in the same chunk) and non-file concept + source_files, mirroring the #1757 `.is_file()` condition.""" + from graphify.llm import extract_corpus_parallel + + a = tmp_path / "A.md"; a.write_text("# a\n") + c = tmp_path / "C.md"; c.write_text("# c\n") + # B.py exists on disk but is NOT dispatched — the #1895 out-of-scope case. + b = tmp_path / "B.py"; b.write_text("def b(): pass\n") + + def stray(chunk, **kwargs): + return { + "nodes": [ + {"id": "a_ok", "source_file": "A.md", "file_type": "document"}, + # sibling attribution: a different dispatched file in the same chunk + {"id": "c_sibling", "source_file": "C.md", "file_type": "document"}, + # out-of-scope: real file on disk, never dispatched + {"id": "b_stray", "source_file": "B.py", "file_type": "code"}, + # concept node: source_file is not a file — must survive + {"id": "auth_flow", "source_file": "auth flow", "file_type": "concept"}, + ], + "edges": [ + {"source": "a_ok", "target": "c_sibling", "source_file": "A.md"}, + {"source": "a_ok", "target": "b_stray", "source_file": "A.md"}, + ], + "hyperedges": [ + {"id": "h_bad", "nodes": ["a_ok", "c_sibling", "b_stray"], "source_file": "A.md"}, + {"id": "h_ok", "nodes": ["a_ok", "c_sibling", "auth_flow"], "source_file": "A.md"}, + ], + "input_tokens": 1, "output_tokens": 1, + } + + with patch("graphify.llm.extract_files_direct", side_effect=stray): + result = extract_corpus_parallel( + [a, c], backend="kimi", root=tmp_path, + token_budget=None, chunk_size=2, max_concurrency=1, + ) + + ids = {n["id"] for n in result["nodes"]} + assert "b_stray" not in ids, "out-of-scope node leaked into the merged graph (#1895)" + assert {"a_ok", "c_sibling", "auth_flow"} <= ids, ( + f"in-scope sibling/concept attributions must be kept: {ids}" + ) + assert result["out_of_scope_dropped"] == 1 + # Edges/hyperedges referencing the dropped node id are gone; in-scope ones stay. + assert all( + "b_stray" not in (e.get("source"), e.get("target")) for e in result["edges"] + ), f"edge to dropped node survived: {result['edges']}" + assert any( + e["source"] == "a_ok" and e["target"] == "c_sibling" for e in result["edges"] + ) + assert [h["id"] for h in result["hyperedges"]] == ["h_ok"] + err = capsys.readouterr().err + assert "out-of-scope" in err and "B.py" in err + # The dispatched files all produced nodes — reconciliation sees no gaps. + assert result["uncovered_files"] == [] + + +def test_out_of_scope_drop_count_is_zero_when_all_in_scope(tmp_path, capsys): + """Counter-test: a clean run records out_of_scope_dropped == 0 and no warning.""" + from graphify.llm import extract_corpus_parallel + + a = tmp_path / "A.md"; a.write_text("# a\n") + + def clean(chunk, **kwargs): + return { + "nodes": [{"id": "a_ok", "source_file": "A.md", "file_type": "document"}], + "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1, + } + + with patch("graphify.llm.extract_files_direct", side_effect=clean): + result = extract_corpus_parallel( + [a], backend="kimi", root=tmp_path, + token_budget=None, chunk_size=1, max_concurrency=1, + ) + + assert result["out_of_scope_dropped"] == 0 + assert [n["id"] for n in result["nodes"]] == ["a_ok"] + assert "out-of-scope" not in capsys.readouterr().err + + def test_checkpoint_caches_sliced_document_chunks(tmp_path, capsys): """#1870: the checkpoint's allowlist must resolve a FileSlice to its parent path (via unit_path), not read a non-existent `.rel`. An oversized doc is From b1e313cc5f2879b664765d4626cfb423baaa5512 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 00:40:21 +0100 Subject: [PATCH 28/42] fix(cli): stamp freshly-extracted semantic docs in the manifest (#1897) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #933 manifest filter built its extracted-set from node/edge source_file values, which are root-relative on a fresh extraction, and compared them against files_by_type entries, which are absolute (from detect()). The raw string membership test therefore never matched, so every freshly-extracted semantic doc was dropped from the manifest and re-queued as changed on the next run; only code files and cache-replayed docs got stamped. The filter now lives in _stamped_manifest_files(), which resolves BOTH sides against the scan root before the membership test — the same Path/is_absolute/resolve normalization the #1890 reconciliation uses in graphify.llm. Genuinely omitted zero-node docs still have no source_file entry and stay unstamped, preserving the intentional #933 re-queue behavior. Regression tests: a CLI extract with a mocked corpus extractor returning root-relative source_files lands the doc in manifest.json with a non-empty semantic_hash while a zero-node doc stays out; a unit test covers relative (fresh) and absolute (cache-hit) source_file shapes. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/cli.py | 59 +++++++++++++++++++++----- tests/test_extract_cli.py | 87 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 11 deletions(-) diff --git a/graphify/cli.py b/graphify/cli.py index 92f05875d..3c1ae0e51 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -52,6 +52,50 @@ def _default_graph_path() -> str: return str(Path(_GRAPHIFY_OUT) / "graph.json") + + +def _stamped_manifest_files( + files_by_type: dict[str, list[str]], + sem_result: dict, + root: Path, +) -> dict[str, list[str]]: + """Manifest-safe files dict: only stamp semantic files that actually + produced output (cache hit or fresh extraction). Files whose chunk failed + have no source_file entry in sem_result — leaving their semantic_hash + empty so detect_incremental re-queues them (#933). + + Both sides of the membership test are resolved against the scan ``root`` + before comparing (#1897): node/edge ``source_file`` values are + root-relative on a fresh extraction while ``files_by_type`` entries are + absolute (from detect()), so a raw string comparison never matched and + every freshly-extracted semantic doc was dropped from the manifest. + Mirrors the #1890 path normalization in graphify.llm. + """ + root = Path(root) + + def _resolve(value: str) -> Path: + p = Path(value) + if not p.is_absolute(): + p = root / p + try: + return p.resolve() + except (OSError, RuntimeError): + return p + + sem_extracted: set[Path] = set() + for coll in ("nodes", "edges"): + for item in sem_result.get(coll, []): + sf = item.get("source_file", "") + if sf: + sem_extracted.add(_resolve(sf)) + sem_types = {"document", "paper", "image"} + return { + ftype: [ + f for f in flist + if ftype not in sem_types or _resolve(f) in sem_extracted + ] + for ftype, flist in files_by_type.items() + } class _StageTimer: """Print per-stage wall-clock timings to stderr when --timing is set (#1490). @@ -2438,17 +2482,10 @@ def _progress(idx: int, total: int, _result: dict) -> None: # that actually produced output (cache hit or fresh extraction). Files # whose chunk failed have no source_file entry in sem_result — leaving # their semantic_hash empty so detect_incremental re-queues them (#933). - _sem_extracted: set[str] = { - n.get("source_file", "") for n in sem_result.get("nodes", []) - } | { - e.get("source_file", "") for e in sem_result.get("edges", []) - } - _sem_extracted.discard("") - _sem_types = {"document", "paper", "image"} - _manifest_files = { - ftype: [f for f in flist if ftype not in _sem_types or f in _sem_extracted] - for ftype, flist in files_by_type.items() - } + # Path normalization against the scan root happens inside the helper + # (#1897) so fresh root-relative source_files match detect()'s + # absolute file lists. + _manifest_files = _stamped_manifest_files(files_by_type, sem_result, target) if no_cluster: # --no-cluster: dump the raw merged extraction as graph.json. diff --git a/tests/test_extract_cli.py b/tests/test_extract_cli.py index cf5762a7e..f56de047a 100644 --- a/tests/test_extract_cli.py +++ b/tests/test_extract_cli.py @@ -135,6 +135,93 @@ def _capture_semantic_cache(*args, **kwargs): } == {str(corpus / "README.md")} +def test_manifest_stamps_freshly_extracted_semantic_docs(monkeypatch, tmp_path): + """#1897: fresh extraction returns nodes with ROOT-RELATIVE source_file, + while the #933 manifest filter compared them against detect()'s ABSOLUTE + paths — so `f in _sem_extracted` was always False and every freshly + extracted doc was dropped from the manifest (only code/zero-node files + survived). Both sides must be resolved against the scan root; a genuinely + omitted doc (zero nodes) must still stay unstamped (#933 is intentional).""" + import json + + corpus = _make_corpus(tmp_path) # main.go + README.md + (corpus / "OMITTED.md").write_text("# never extracted\n") + out_dir = tmp_path / "out" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + + def _fresh_relative(paths, **kwargs): + on_chunk = kwargs.get("on_chunk_done") + if on_chunk: + on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) + # Root-relative source_file, exactly what a fresh extraction produces. + # OMITTED.md gets no nodes/edges — the model skipped it. + return { + "nodes": [{"id": "readme", "source_file": "README.md", + "file_type": "document"}], + "edges": [], + "hyperedges": [], + "input_tokens": 10, + "output_tokens": 5, + } + + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _fresh_relative) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, "argv", + ["graphify", "extract", str(corpus), "--backend", "claude", + "--no-cluster", "--out", str(out_dir)], + ) + + try: + mainmod.main() + except SystemExit as exc: + assert exc.code in (None, 0), f"unexpected exit code {exc.code}" + + manifest_path = out_dir / "graphify-out" / "manifest.json" + assert manifest_path.exists() + manifest = json.loads(manifest_path.read_text()) + + assert "README.md" in manifest, ( + f"freshly-extracted doc missing from manifest (#1897): {sorted(manifest)}" + ) + assert manifest["README.md"].get("semantic_hash"), ( + "freshly-extracted doc must carry a non-empty semantic_hash" + ) + # Code files are always stamped. + assert manifest.get("main.go", {}).get("semantic_hash") + # The zero-node doc stays unstamped so detect_incremental re-queues it (#933). + assert "OMITTED.md" not in manifest, ( + "zero-node doc must not be stamped in the manifest" + ) + + +def test_stamped_manifest_files_normalizes_both_sides(tmp_path): + """Unit test for the #1897 helper: relative (fresh) and absolute (cache-hit) + source_file values must both match detect()'s absolute file lists; docs with + no output are filtered; code files pass through untouched.""" + from graphify.cli import _stamped_manifest_files + + fresh_doc = tmp_path / "fresh.md"; fresh_doc.write_text("# fresh") + cached_doc = tmp_path / "cached.md"; cached_doc.write_text("# cached") + omitted_doc = tmp_path / "omitted.md"; omitted_doc.write_text("# omitted") + code = tmp_path / "app.py"; code.write_text("x = 1") + + files_by_type = { + "code": [str(code)], + "document": [str(fresh_doc), str(cached_doc), str(omitted_doc)], + } + sem_result = { + # fresh extraction: root-relative source_file + "nodes": [{"id": "n1", "source_file": "fresh.md"}], + # cache replay: absolute source_file (edge-only coverage counts too) + "edges": [{"source": "a", "target": "b", "source_file": str(cached_doc)}], + } + + out = _stamped_manifest_files(files_by_type, sem_result, tmp_path) + assert out["code"] == [str(code)] + assert out["document"] == [str(fresh_doc), str(cached_doc)] + + def _code_only_corpus(tmp_path): """A corpus with only code — no docs/papers/images.""" (tmp_path / "auth.py").write_text( From 43b2affd5638df19cd0e7e0dee2b60c43bfa61ef Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 00:43:46 +0100 Subject: [PATCH 29/42] chore: open 0.9.17 with 8 batch fixes (#1895 #1897 #1902 #1907 #1896 #1900 #1901 #1906) Bumps to 0.9.17 (unreleased) and records the batch implemented via Fable subagents in isolated worktrees, integrated onto v8: out-of-scope node drop, manifest stamping, merge-driver registration, hooksPath configparser fix, obsidian prune, multilingual query stopwords, .skill classification, and the postgres package-name hint. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 11 +++++++++++ pyproject.toml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43e7b658e..b13ef51d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.9.17 (unreleased) + +- Security/privacy follow-up: nodes whose `source_file` was never dispatched are now dropped from the graph, not just skipped from the cache (#1895). The #1757 guard stopped a mis-attributed node from clobbering another file's cache entry, but the node itself still flowed into `graph.json`; it is now filtered out of the merged result (real-file, non-dispatched attributions only), consistent with the cache rejection. +- Fix: `manifest.json` now records every successfully-extracted file, not just the zero-node ones (#1897). The #933 stamping filter compared root-relative node `source_file`s against absolute `detect()` paths, so it dropped every freshly-extracted semantic document from the manifest and broke the incremental-update baseline. Both sides are now resolved before comparison; genuinely omitted/zero-node docs stay unstamped so they retry. +- Fix: `graphify hook install` now registers the `graph.json` union merge driver that the README and CHANGELOG have long documented (#1902). It writes the `merge.graphify` config via `git config` and an idempotent, append-only `graphify-out/graph.json merge=graphify` line in `.gitattributes`; `uninstall` removes them. +- Fix: `hook install`/`status` no longer print a spurious "could not read core.hooksPath" warning on repos whose `.git/config` contains git-legal duplicate keys (VS Code writes these) (#1907). Config is now resolved via `git rev-parse --git-path hooks` instead of a strict `configparser`, which rejected duplicate keys. +- Fix: `graphify export obsidian` prunes notes for nodes that left the graph instead of merging old and new on re-export (#1896). Only notes graphify itself wrote (tracked in its ownership manifest) are removed, with a vault-containment guard, so user-authored notes are never touched. +- Fix: non-English query sentences no longer pick wrong BFS seeds because their filler words were unfiltered (#1900). The query stopword set now covers German and the major Romance languages (curated to avoid clobbering English content words), so `Wie funktioniert die Authentifizierung?` seeds the keyword, not the stopwords. +- Fix: Python calls to an imported module now resolve (already shipped in 0.9.16); `.skill` files (Markdown-with-frontmatter agent files) are now classified as documents instead of being silently dropped as an unsupported extension (#1901). +- Fix: the `--postgres` missing-driver error now points at the correct PyPI package, `graphifyy[postgres]` (was the nonexistent `graphify[postgres]`) (#1906). + ## 0.9.16 (2026-07-14) - Fix: semantic extraction now reconciles dispatched files against returned results, so a document the model silently omits is no longer lost without a trace (#1890). A chunk can return a clean, non-empty response that simply leaves out some of the documents it was given; those docs previously produced no node, no warning, and no cache/manifest stamp, so they were re-dispatched and re-omitted on every run. `extract_corpus_parallel` now diffs the dispatched file set against the `source_file`s that came back, records the gap in `uncovered_files`, and prints a loud warning listing the omitted files. (This is the visibility guard; routing documents through the deterministic extractor so they always get at least a file node is tracked separately.) diff --git a/pyproject.toml b/pyproject.toml index 3e5662a7a..2e5340d39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.9.16" +version = "0.9.17" description = "AI coding assistant skill (Claude Code, CodeBuddy, Codex, OpenCode, Kilo Code, Cursor, Gemini CLI, Aider, OpenClaw, Factory Droid, Trae, Hermes, Kiro, Pi, Devin CLI, Google Antigravity) - turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } From 75cf56bfbc1d4c36dee152f12260556fd8a4e0bb Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 10:57:45 +0100 Subject: [PATCH 30/42] fix(detect,cli): excluded files are pruned from graph and manifest, not misreported as deleted (#1908 #1909) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two coupled excluded-vs-deleted fixes: #1909 — incremental extract's prune set was derived from the manifest alone (manifest - corpus), so a file that became excluded without ever being manifest-listed (every pre-#1897 graph) kept its stale nodes in graph.json forever. The prune set is now also derived from the existing graph's own node source_files reconciled against the post-exclude detect corpus (_stale_graph_sources), restricted to in-root paths; out-of-root --include/symlinked entries and remote (://) sources are never pruned. Relative source_files are anchored against both the scan root and the --out root (the #555/#1899 relativized form). The --no-cluster incremental early exit never runs build_merge, so an exclusion-only change now prunes the raw graph.json in place instead. #1908 — save_manifest retained any prior row whose file still existed on disk, so an excluded-but-alive file survived as a permanent phantom that detect_incremental reported as deleted on every run. Full-scan callers (extract's saves, watch._rebuild_code's saves) now pass the RAW detect corpus via a new scan_corpus parameter and in-root rows outside it are dropped; the corpus is deliberately not the #933 stamp-filtered files dict, so failed-chunk/omitted-doc rows and --code-only doc rows survive. Subset saves (changed_paths hooks, #917) keep the seeding default. detect_incremental now splits manifest rows that left the scan into deleted_files (gone from disk) and excluded_files (alive but out of scan), mirroring the watch-side #1795 distinction, and the extract summaries report the two separately. Ordering matters: extract's cleanup of newly-excluded nodes previously worked only through the #1908 conflation, so the graph-source prune lands together with the manifest split to avoid regressing #1909. --- graphify/cli.py | 210 +++++++++++++++++++++++++++++++++++++- graphify/detect.py | 76 +++++++++++++- graphify/watch.py | 21 +++- tests/test_detect.py | 159 +++++++++++++++++++++++++++++ tests/test_extract_cli.py | 167 ++++++++++++++++++++++++++++++ 5 files changed, 620 insertions(+), 13 deletions(-) diff --git a/graphify/cli.py b/graphify/cli.py index 3c1ae0e51..6d037332a 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -96,6 +96,153 @@ def _resolve(value: str) -> Path: ] for ftype, flist in files_by_type.items() } + + +def _stale_graph_sources( + graph_path: Path, + scan_root: Path, + seen_files: set[str], +) -> list[str]: + """Source files graph.json still references but the current scan no longer + contains (#1909). + + Incremental extract's prune set was historically derived from the manifest + alone (``manifest - corpus``), so a file that became EXCLUDED + (.graphifyignore/.gitignore/--exclude changed) without being listed in the + manifest kept its stale nodes in graph.json forever. Derive prune + candidates from the graph's own node ``source_file``s instead: anything + the graph references that the post-exclude detect corpus no longer + contains is stale, whether the file was deleted or newly excluded. + + Only IN-ROOT paths are candidates: out-of-root/absolute entries + (--include sources, symlinked external corpora) are never walked by + detect, so their absence from the corpus is not staleness evidence. + Relative entries are re-anchored against both the scan root and the + graph's own output root (``--out`` extracts store source_files relative + to the OUT root, e.g. ``../project/x.py``, #555/#1899); only anchors + that land inside the scan root count. + ``seen_files`` must be the FULL detect output including unclassified + files, so nodes from walked-but-unsupported sources (e.g. introspected + Cargo.toml manifests) are not misread as stale. + """ + try: + data = json.loads(graph_path.read_text(encoding="utf-8")) + except Exception: + return [] + if not isinstance(data, dict): + return [] + try: + root_res = scan_root.resolve() + except (OSError, RuntimeError): + root_res = scan_root + # /graphify-out/graph.json — relative source_files may be anchored here. + out_base = graph_path.parent.parent + try: + out_base = out_base.resolve() + except (OSError, RuntimeError): + pass + + def _within_root(p: Path) -> bool: + try: + p.relative_to(root_res) + return True + except ValueError: + pass + try: + p.resolve().relative_to(root_res) + return True + except (ValueError, OSError, RuntimeError): + return False + + def _in_seen(p: Path) -> bool: + if str(p) in seen_files: + return True + try: + return str(p.resolve()) in seen_files + except (OSError, RuntimeError): + return False + + stale: list[str] = [] + checked: set[str] = set() + for n in data.get("nodes", []): + if not isinstance(n, dict): + continue + sf = n.get("source_file") + if not sf or not isinstance(sf, str) or sf in checked: + continue + checked.add(sf) + if "://" in sf: + continue # remote/virtual source (e.g. Google Workspace), not a scanned path + p = Path(sf) + if p.is_absolute(): + candidates = [p] + else: + rel = sf.replace("\\", "/") + bases = [root_res] + if out_base != root_res: + bases.append(out_base) + candidates = [ + Path(os.path.normpath(str(base / rel))) for base in bases + ] + in_root = [c for c in candidates if _within_root(c)] + if not in_root: + continue # out-of-root under every anchor: never prune + if any(_in_seen(c) for c in in_root): + continue # still part of the scan corpus + stale.append(sf) + return stale + + +def _prune_graph_json_sources(graph_path: Path, stale_sources: list[str]) -> int: + """Drop nodes/edges/hyperedges owned by ``stale_sources`` from graph.json + in place. Returns the number of nodes removed. + + Used by the ``--no-cluster`` incremental early-exit: that path never runs + ``build_merge`` (it would raw-dump only the new chunks), so an + exclusion-only change must prune the existing raw graph directly or the + newly-excluded file's nodes survive forever (#1909). + ``stale_sources`` comes from :func:`_stale_graph_sources`, i.e. the + graph's own ``source_file`` spellings, so exact string matching is enough. + """ + try: + data = json.loads(graph_path.read_text(encoding="utf-8")) + except Exception: + return 0 + if not isinstance(data, dict): + return 0 + stale = set(stale_sources) + links_key = "links" if "links" in data else "edges" + nodes = [n for n in data.get("nodes", []) if isinstance(n, dict)] + kept_nodes = [n for n in nodes if n.get("source_file") not in stale] + removed_ids = { + n.get("id") for n in nodes if n.get("source_file") in stale + } + n_removed = len(nodes) - len(kept_nodes) + kept_edges = [ + e for e in data.get(links_key, []) + if isinstance(e, dict) + and e.get("source_file") not in stale + and e.get("source") not in removed_ids + and e.get("target") not in removed_ids + ] + kept_hyper = [ + h for h in data.get("hyperedges", []) + if isinstance(h, dict) and h.get("source_file") not in stale + ] + if n_removed == 0 and len(kept_edges) == len(data.get(links_key, [])) and ( + len(kept_hyper) == len(data.get("hyperedges", [])) + ): + return 0 + data["nodes"] = kept_nodes + data[links_key] = kept_edges + if "hyperedges" in data: + data["hyperedges"] = kept_hyper + from graphify.export import backup_if_protected as _backup + _backup(graph_path.parent) + graph_path.write_text(json.dumps(data, indent=2), encoding="utf-8") + return n_removed + + class _StageTimer: """Print per-stage wall-clock timings to stderr when --timing is set (#1490). @@ -2137,6 +2284,8 @@ def _parse_float(name: str, raw: str) -> float: paper_files = [] image_files = [] deleted_files = [] + excluded_files = [] + graph_stale_sources = [] unchanged_total = 0 files_by_type = {} elif incremental_mode: @@ -2154,7 +2303,18 @@ def _parse_float(name: str, raw: str) -> float: paper_files = [Path(p) for p in new_by_type.get("paper", [])] image_files = [Path(p) for p in new_by_type.get("image", [])] deleted_files = list(detection.get("deleted_files", [])) + excluded_files = list(detection.get("excluded_files", [])) unchanged_total = sum(len(v) for v in detection.get("unchanged_files", {}).values()) + # #1909: derive the prune set from the existing graph itself, not + # just the manifest. A file that became excluded without ever + # being manifest-listed (every pre-#1897 graph is in this state) + # still has stale nodes carried forward by build_merge unless the + # graph's own sources are reconciled against the current corpus. + _seen_files = {f for _fl in files_by_type.values() for f in _fl} + _seen_files.update(detection.get("unclassified", [])) + graph_stale_sources = _stale_graph_sources( + existing_graph_path, target, _seen_files + ) else: print(f"[graphify extract] scanning {target}") detection = _detect(target, google_workspace=google_workspace or None, extra_excludes=cli_excludes or None, cache_root=out_root) @@ -2164,6 +2324,8 @@ def _parse_float(name: str, raw: str) -> float: paper_files = [Path(p) for p in files_by_type.get("paper", [])] image_files = [Path(p) for p in files_by_type.get("image", [])] deleted_files = [] + excluded_files = [] + graph_stale_sources = [] unchanged_total = 0 semantic_files = doc_files + paper_files + image_files @@ -2182,10 +2344,15 @@ def _parse_float(name: str, raw: str) -> float: paper_files = [] image_files = [] if incremental_mode: + # Excluded-but-alive files are reported separately from deletions + # (#1908): they still exist on disk, the scan just stopped + # covering them (ignore rules / --exclude changed). + _excl_note = f"; {len(excluded_files)} excluded" if excluded_files else "" print( f"[graphify extract] {len(code_files)} code, {len(doc_files)} docs, " f"{len(paper_files)} papers, {len(image_files)} images changed; " f"{unchanged_total} unchanged; {len(deleted_files)} deleted" + f"{_excl_note}" ) else: print( @@ -2487,6 +2654,16 @@ def _progress(idx: int, total: int, _result: dict) -> None: # absolute file lists. _manifest_files = _stamped_manifest_files(files_by_type, sem_result, target) + # Full-scan manifest saves prune rows for in-root files that left the + # scan corpus but still exist on disk (#1908). The corpus must be the + # RAW detect output (files_by_type), NOT the #933-stamp-filtered + # _manifest_files above — pruning to the filtered set would erase + # failed-chunk/omitted-doc rows and every doc row on --code-only runs. + _scan_corpus = ( + {f for _fl in files_by_type.values() for f in _fl} + if has_path else None + ) + if no_cluster: # --no-cluster: dump the raw merged extraction as graph.json. # No NetworkX, no community detection, no analysis sidecar. @@ -2506,12 +2683,26 @@ def _progress(idx: int, total: int, _result: dict) -> None: and not cargo_result.get("nodes") and not cargo_result.get("edges") ): + # An exclusion-only change reaches this gate (excluded files + # are deliberately NOT in deleted_files, #1908) but must still + # scrub the newly-excluded sources from the raw graph (#1909). + # This path never runs build_merge, so prune in place. + if graph_stale_sources: + _n_pruned = _prune_graph_json_sources( + existing_graph_path, graph_stale_sources + ) + if _n_pruned: + print( + f"[graphify extract] pruned {_n_pruned} node(s) from " + f"{len(graph_stale_sources)} source file(s) no longer " + "in the scan (deleted or excluded)." + ) print( "[graphify extract] no incremental changes detected " "(--no-cluster); outputs left untouched." ) try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus) except Exception as exc: print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) stages.total() @@ -2548,7 +2739,7 @@ def _progress(idx: int, total: int, _result: dict) -> None: f"est. cost: ${cost:.4f}" ) try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus) except Exception as exc: print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) if global_merge: @@ -2577,10 +2768,18 @@ def _progress(idx: int, total: int, _result: dict) -> None: from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising dedup_backend = backend if dedup_llm else None if incremental_mode: + # Prune everything the current scan no longer covers: genuinely + # deleted manifest rows, excluded-but-alive manifest rows (#1908), + # and the graph's own stale sources — which catches files that + # became excluded without ever being manifest-listed (#1909). + _prune_sources: list[str] = list(deleted_files) + for _src in list(excluded_files) + graph_stale_sources: + if _src not in _prune_sources: + _prune_sources.append(_src) G = _build_merge( [merged], graph_path=existing_graph_path, - prune_sources=deleted_files or None, + prune_sources=_prune_sources or None, dedup=True, dedup_llm_backend=dedup_backend, root=target, @@ -2642,7 +2841,7 @@ def _progress(idx: int, total: int, _result: dict) -> None: } analysis_path.write_text(json.dumps(analysis, indent=2), encoding="utf-8") try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus) except Exception as exc: print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) @@ -2654,11 +2853,12 @@ def _progress(idx: int, total: int, _result: dict) -> None: ) print(f"[graphify extract] wrote {analysis_path}") if incremental_mode: + _excl_note = f", {len(excluded_files)} excluded" if excluded_files else "" print( f"[graphify extract] incremental summary: " f"{sem_cache_hits + unchanged_total} files cached/unchanged, " f"{len(code_files) + sem_cache_misses} re-extracted, " - f"{len(deleted_files)} deleted" + f"{len(deleted_files)} deleted{_excl_note}" ) elif sem_cache_hits: print(f"[graphify extract] semantic cache: {sem_cache_hits} cached, {sem_cache_misses} re-extracted") diff --git a/graphify/detect.py b/graphify/detect.py index bc73fbde7..195b5e543 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -1466,6 +1466,7 @@ def save_manifest( *, kind: str = "both", root: Path | None = None, + scan_corpus: set[str] | list[str] | None = None, ) -> None: """Save current file mtimes + content hashes for change detection. @@ -1481,9 +1482,51 @@ def save_manifest( machines and checkout locations (#777). Out-of-root entries are written as absolute so they continue to round-trip on the saving machine. When ``root`` is None the legacy absolute-keyed format is preserved. + + ``scan_corpus`` (#1908): full-scan callers pass the COMPLETE detect + corpus (absolute paths) so seeded rows for in-root files that are still + alive on disk but no longer part of the scan (newly excluded via + .graphifyignore/.gitignore/--exclude) are dropped instead of surviving + forever and masquerading as deletions in detect_incremental. It must be + the RAW detect output, not a stamp-filtered subset — pruning to a + filtered set would erase rows the filter merely omitted (failed chunks, + --code-only doc rows). Out-of-root entries are never pruned. Callers + saving a SUBSET of files (changed_paths hooks, skill runbooks, #917) + must leave this None so their untouched rows are preserved. """ existing = load_manifest(manifest_path, root=root) + scan_set: set[str] | None = set(scan_corpus) if scan_corpus is not None else None + try: + root_res: Path | None = Path(root).resolve() if root is not None else None + except (OSError, RuntimeError): + root_res = Path(root) if root is not None else None + + def _in_scan(path_str: str) -> bool: + if path_str in scan_set: + return True + try: + return str(Path(path_str).resolve()) in scan_set + except (OSError, RuntimeError): + return False + + def _in_root(path_str: str) -> bool: + # Without a root we cannot tell in-root from out-of-root; fail open + # (keep the row) so out-of-root corpora are never pruned by accident. + if root_res is None: + return False + p = Path(path_str) + try: + p.relative_to(root_res) + return True + except ValueError: + pass + try: + p.resolve().relative_to(root_res) + return True + except (ValueError, OSError, RuntimeError): + return False + def _normalise_entry(entry): if isinstance(entry, (int, float)): return {"mtime": entry, "ast_hash": "", "semantic_hash": ""} @@ -1496,17 +1539,23 @@ def _normalise_entry(entry): # Seed from the existing manifest so incremental callers passing a subset # of files don't silently erase entries for untouched files (#917). # Prune entries whose file no longer exists on disk — those are genuine - # deletions that detect_incremental() should treat as gone. + # deletions that detect_incremental() should treat as gone. When the + # caller supplied the full scan corpus, additionally prune in-root rows + # the scan no longer covers: those files were excluded, not deleted, and + # keeping the row makes them look deleted on every future run (#1908). manifest: dict[str, dict] = {} for f, entry in existing.items(): normalised = _normalise_entry(entry) if normalised is None: continue try: - if Path(f).exists(): - manifest[f] = normalised + if not Path(f).exists(): + continue except OSError: continue + if scan_set is not None and not _in_scan(f) and _in_root(f): + continue # excluded-but-alive: drop the stale row (#1908) + manifest[f] = normalised all_files = [f for file_list in files.values() for f in file_list] with ThreadPoolExecutor() as pool: @@ -1584,6 +1633,8 @@ def detect_incremental( full["new_files"] = full["files"] full["unchanged_files"] = {k: [] for k in full["files"]} full["new_total"] = full["total_files"] + full["deleted_files"] = [] + full["excluded_files"] = [] return full new_files: dict[str, list[str]] = {k: [] for k in full["files"]} @@ -1636,9 +1687,23 @@ def detect_incremental( else: unchanged_files[ftype].append(f) - # Files in manifest that no longer exist - their cached nodes are now ghost nodes + # Manifest rows that left the corpus, split by disk existence (#1908): + # a row whose file is gone from DISK is a genuine deletion (its cached + # nodes are ghosts); a row whose file still exists but is out of the + # current scan was EXCLUDED (ignore rules / --exclude changed) and must + # not be reported as deleted. Mirrors the watch-side excluded-vs-deleted + # distinction (#1795). current_files = {f for flist in full["files"].values() for f in flist} - deleted_files = [f for f in manifest if f not in current_files] + deleted_files: list[str] = [] + excluded_files: list[str] = [] + for f in manifest: + if f in current_files: + continue + try: + alive = Path(f).exists() + except OSError: + alive = False + (excluded_files if alive else deleted_files).append(f) new_total = sum(len(v) for v in new_files.values()) full["incremental"] = True @@ -1646,4 +1711,5 @@ def detect_incremental( full["unchanged_files"] = unchanged_files full["new_total"] = new_total full["deleted_files"] = deleted_files + full["excluded_files"] = excluded_files return full diff --git a/graphify/watch.py b/graphify/watch.py index df911ef8f..57364228a 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -1021,7 +1021,14 @@ def _add_deleted_source(path: Path) -> None: try: from graphify.detect import save_manifest - save_manifest(detected["files"], kind="ast", root=project_root) + # detected["files"] is a FULL detect of the watched root, so + # pass it as the scan corpus too: rows for files that left the + # scan but still exist on disk (newly excluded) are pruned + # instead of surviving as phantom "deleted" entries (#1908). + save_manifest( + detected["files"], kind="ast", root=project_root, + scan_corpus={f for _fl in detected["files"].values() for f in _fl}, + ) except Exception: pass @@ -1060,7 +1067,11 @@ def _add_deleted_source(path: Path) -> None: if same_topology: try: from graphify.detect import save_manifest - save_manifest(detected["files"], kind="ast", root=project_root) + # Full-scan save: prune excluded-but-alive rows (#1908). + save_manifest( + detected["files"], kind="ast", root=project_root, + scan_corpus={f for _fl in detected["files"].values() for f in _fl}, + ) except Exception: pass flag = out / "needs_update" @@ -1138,7 +1149,11 @@ def _add_deleted_source(path: Path) -> None: try: from graphify.detect import save_manifest - save_manifest(detected["files"], kind="ast", root=project_root) + # Full-scan save: prune excluded-but-alive rows (#1908). + save_manifest( + detected["files"], kind="ast", root=project_root, + scan_corpus={f for _fl in detected["files"].values() for f in _fl}, + ) except Exception: pass diff --git a/tests/test_detect.py b/tests/test_detect.py index 374b2552f..a783eaf6a 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -1851,3 +1851,162 @@ def test_nested_gitignore_patterns_still_apply_inside_their_dir(tmp_path): result = detect(tmp_path) assert result["total_files"] == 2 # main.py + sub/keep.py; sub/noise.log ignored + + +# --------------------------------------------------------------------------- +# #1908: manifest must not retain scan-excluded files as permanent +# "deleted" entries. Full-scan saves prune excluded-but-alive rows; subset +# saves keep preserving untouched rows (#917); out-of-root rows never prune. +# --------------------------------------------------------------------------- + +def test_save_manifest_full_scan_prunes_excluded_but_alive_row(tmp_path): + """A row for a file that still exists on disk but left the scan corpus + (newly excluded) is dropped when the caller passes the full corpus.""" + import json + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("x = 1\n") + b.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + + save_manifest({"code": [str(a), str(b)]}, manifest_path, root=tmp_path) + raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert set(raw) == {"a.py", "b.py"} + + # Second full scan no longer covers b.py (excluded), yet b.py is alive. + save_manifest( + {"code": [str(a)]}, manifest_path, root=tmp_path, + scan_corpus={str(a)}, + ) + raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert set(raw) == {"a.py"}, ( + f"excluded-but-alive row must be pruned on a full-scan save, got {set(raw)}" + ) + + +def test_save_manifest_full_scan_still_prunes_missing_file(tmp_path): + """Genuine deletions keep being pruned when scan_corpus is passed.""" + import json + a = tmp_path / "a.py" + gone = tmp_path / "gone.py" + a.write_text("x = 1\n") + gone.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + save_manifest({"code": [str(a), str(gone)]}, manifest_path, root=tmp_path) + + gone.unlink() + save_manifest( + {"code": [str(a)]}, manifest_path, root=tmp_path, + scan_corpus={str(a)}, + ) + raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert set(raw) == {"a.py"} + + +def test_save_manifest_subset_save_preserves_untouched_rows(tmp_path): + """Without scan_corpus (changed_paths hooks, skill runbooks, #917) a + subset save must keep seeding rows for files it wasn't given.""" + import json + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("x = 1\n") + b.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + save_manifest({"code": [str(a), str(b)]}, manifest_path, root=tmp_path) + + # Incremental hook re-stamps only a.py; b.py's row must survive. + save_manifest({"code": [str(a)]}, manifest_path, root=tmp_path) + raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert set(raw) == {"a.py", "b.py"}, ( + f"subset saves must preserve untouched rows (#917), got {set(raw)}" + ) + + +def test_save_manifest_full_scan_keeps_out_of_root_rows(tmp_path): + """Out-of-root entries (--include sources, symlinked corpora) are never + walked by detect, so their absence from the corpus is not exclusion + evidence — a full-scan save must keep them.""" + import json + a = tmp_path / "a.py" + a.write_text("x = 1\n") + outside = tmp_path.parent / f"{tmp_path.name}-extern.py" + outside.write_text("z = 3\n") + try: + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + save_manifest( + {"code": [str(a), str(outside)]}, manifest_path, root=tmp_path + ) + save_manifest( + {"code": [str(a)]}, manifest_path, root=tmp_path, + scan_corpus={str(a)}, + ) + raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert "a.py" in raw + assert str(outside.resolve()) in raw, ( + f"out-of-root rows must never be pruned to the scan, got {set(raw)}" + ) + finally: + outside.unlink(missing_ok=True) + + +def test_detect_incremental_reports_excluded_not_deleted(tmp_path): + """A previously-indexed file that becomes excluded (still on disk) must + land in excluded_files, not deleted_files (#1908).""" + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("x = 1\n") + b.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + full = detect(tmp_path) + save_manifest(full["files"], manifest_path, root=tmp_path) + + inc = detect_incremental( + tmp_path, manifest_path, extra_excludes=["b.py"] + ) + assert inc["deleted_files"] == [], ( + f"excluded-but-alive file misreported as deleted: {inc['deleted_files']}" + ) + assert [Path(f).name for f in inc["excluded_files"]] == ["b.py"] + + +def test_detect_incremental_still_reports_real_deletions(tmp_path): + """Counterpart: a manifest row whose file is gone from disk stays in + deleted_files.""" + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("x = 1\n") + b.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + full = detect(tmp_path) + save_manifest(full["files"], manifest_path, root=tmp_path) + + b.unlink() + inc = detect_incremental(tmp_path, manifest_path) + assert [Path(f).name for f in inc["deleted_files"]] == ["b.py"] + assert inc["excluded_files"] == [] + + +def test_detect_incremental_exclusion_stable_across_runs(tmp_path): + """After a full-scan save prunes the excluded row, later incremental runs + report the file neither as deleted nor as excluded — the exclusion has + fully settled instead of resurfacing forever.""" + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("x = 1\n") + b.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + full = detect(tmp_path) + save_manifest(full["files"], manifest_path, root=tmp_path) + + # Run 1: b.py newly excluded — reported as excluded, then the full-scan + # save (what extract does at the end of the run) prunes its row. + inc1 = detect_incremental(tmp_path, manifest_path, extra_excludes=["b.py"]) + assert [Path(f).name for f in inc1["excluded_files"]] == ["b.py"] + assert inc1["deleted_files"] == [] + corpus = {f for flist in inc1["files"].values() for f in flist} + save_manifest(inc1["files"], manifest_path, root=tmp_path, scan_corpus=corpus) + + # Run 2 (and beyond): steady state — nothing deleted, nothing excluded. + inc2 = detect_incremental(tmp_path, manifest_path, extra_excludes=["b.py"]) + assert inc2["deleted_files"] == [] + assert inc2["excluded_files"] == [] diff --git a/tests/test_extract_cli.py b/tests/test_extract_cli.py index f56de047a..3442a4877 100644 --- a/tests/test_extract_cli.py +++ b/tests/test_extract_cli.py @@ -364,3 +364,170 @@ def test_extract_timing_flag_emits_stage_timings(monkeypatch, tmp_path, capsys): mainmod.main() assert exc2.value.code == 0 assert "graphify timing" not in capsys.readouterr().err + + +# --------------------------------------------------------------------------- +# #1909: a newly-excluded file's nodes must be pruned from graph.json on the +# next incremental extract even when the manifest never listed the file (the +# pre-#1897 state every 0.9.16 graph is in), so the manifest-diff prune set +# (`manifest - corpus`) can never see it. +# --------------------------------------------------------------------------- + +def _two_file_corpus(tmp_path): + project = tmp_path / "project" + project.mkdir() + (project / "x.py").write_text( + "def secret_helper():\n return 42\n\n" + "def secret_caller():\n return secret_helper()\n" + ) + (project / "keep.py").write_text( + "def kept():\n return still_here()\n\n" + "def still_here():\n return 1\n" + ) + return project + + +def _node_sources(graph_path): + import json + data = json.loads(graph_path.read_text(encoding="utf-8")) + return {n.get("source_file", "") for n in data.get("nodes", [])} + + +def _run_extract(monkeypatch, argv): + monkeypatch.setattr(mainmod.sys, "argv", argv) + try: + mainmod.main() + except SystemExit as exc: + assert exc.code in (None, 0), f"unexpected exit code {exc.code}" + + +def test_incremental_extract_prunes_newly_excluded_file_not_in_manifest( + monkeypatch, tmp_path +): + """Seed a graph with nodes for x.py, drop x.py from the manifest (pre-#1897 + manifests never listed excluded/omitted files), exclude x.py via + .graphifyignore, re-run extract: x.py's nodes must be gone even though it + was never on the deleted list.""" + import json + project = _two_file_corpus(tmp_path) + out_dir = tmp_path / "out" + _clear_backend_keys(monkeypatch) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + _run_extract( + monkeypatch, + ["graphify", "extract", str(project), "--out", str(out_dir)], + ) + graph_path = out_dir / "graphify-out" / "graph.json" + manifest_path = out_dir / "graphify-out" / "manifest.json" + assert any("x.py" in s for s in _node_sources(graph_path)), ( + "seed extract must produce nodes for x.py" + ) + + # Simulate the pre-#1897 manifest state: x.py was never manifest-listed, + # so `manifest - corpus` can never flag it. + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest = {k: v for k, v in manifest.items() if "x.py" not in k} + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + (project / ".graphifyignore").write_text("x.py\n") + _run_extract( + monkeypatch, + ["graphify", "extract", str(project), "--out", str(out_dir)], + ) + + sources = _node_sources(graph_path) + assert not any("x.py" in s for s in sources), ( + f"newly-excluded x.py must be pruned from graph.json, still see {sources}" + ) + assert any("keep.py" in s for s in sources), ( + "unchanged keep.py nodes must survive the incremental merge" + ) + # x.py exists on disk, is excluded, and must not creep into the manifest. + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert not any("x.py" in k for k in manifest), ( + f"excluded x.py must not be (re)listed in the manifest: {set(manifest)}" + ) + + +def test_incremental_extract_prunes_excluded_file_listed_in_manifest( + monkeypatch, tmp_path +): + """Post-#1897 state: the excluded file IS manifest-listed. It must be + pruned from graph.json AND dropped from the manifest (#1908), and stay + settled on a further run.""" + import json + project = _two_file_corpus(tmp_path) + out_dir = tmp_path / "out" + _clear_backend_keys(monkeypatch) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + _run_extract( + monkeypatch, + ["graphify", "extract", str(project), "--out", str(out_dir)], + ) + graph_path = out_dir / "graphify-out" / "graph.json" + manifest_path = out_dir / "graphify-out" / "manifest.json" + assert any("x.py" in k for k in json.loads(manifest_path.read_text())) + + (project / ".graphifyignore").write_text("x.py\n") + _run_extract( + monkeypatch, + ["graphify", "extract", str(project), "--out", str(out_dir)], + ) + + sources = _node_sources(graph_path) + assert not any("x.py" in s for s in sources) + assert any("keep.py" in s for s in sources) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert not any("x.py" in k for k in manifest), ( + "excluded-but-alive manifest row must be pruned (#1908)" + ) + + # Steady state: a third run neither resurrects x.py nor loses keep.py. + _run_extract( + monkeypatch, + ["graphify", "extract", str(project), "--out", str(out_dir)], + ) + sources = _node_sources(graph_path) + assert not any("x.py" in s for s in sources) + assert any("keep.py" in s for s in sources) + + +def test_no_cluster_incremental_prunes_newly_excluded_file( + monkeypatch, tmp_path, capsys +): + """--no-cluster's exclusion-only early exit must still scrub the excluded + file's nodes from the raw graph.json (that path never runs build_merge), + and must not report the alive file as deleted.""" + import json + project = _two_file_corpus(tmp_path) + out_dir = tmp_path / "out" + _clear_backend_keys(monkeypatch) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + monkeypatch.setattr( + mainmod.sys, "argv", + ["graphify", "extract", str(project), "--no-cluster", "--out", str(out_dir)], + ) + with pytest.raises(SystemExit) as exc: + mainmod.main() + assert exc.value.code == 0 + graph_path = out_dir / "graphify-out" / "graph.json" + assert any("x.py" in s for s in _node_sources(graph_path)) + capsys.readouterr() + + (project / ".graphifyignore").write_text("x.py\n") + with pytest.raises(SystemExit) as exc: + mainmod.main() + assert exc.value.code == 0 + out_text = capsys.readouterr().out + assert "1 deleted" not in out_text, ( + "excluded-but-alive file must not be reported as deleted" + ) + + sources = _node_sources(graph_path) + assert not any("x.py" in s for s in sources), ( + f"--no-cluster early exit must prune excluded sources, still see {sources}" + ) + assert any("keep.py" in s for s in sources) From 49df4663856311dcaab909d052d55d1200aad470 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 10:31:59 +0100 Subject: [PATCH 31/42] fix(sql): recover CREATE FUNCTION/PROCEDURE from tree-sitter ERROR nodes (#1910) tree-sitter-sql parses PL/pgSQL CREATE FUNCTION statements (OUT/INOUT params, tagged dollar quotes, PERFORM/:= body statements) as ERROR nodes, and the dispatch loop had no branch for them, so the functions were silently dropped from the graph. Handle ERROR nodes inside walk() (they can nest inside a merged create_function during multi-statement error recovery) and dispatch top-level ERROR statements to it. The branch regex-scans the raw node text for every CREATE [OR REPLACE] FUNCTION/PROCEDURE, mirroring the existing fb_proc_or_trigger and has_error fallbacks, with a name class that keeps schema-qualified names (exposed.important_function) whole. The PL/pgSQL body is not scanned for FROM/JOIN references to avoid junk reads_from targets, and node ids match the clean create_function branch so seen_ids dedups consistently. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/extractors/sql.py | 21 ++++++++++++++++++++- tests/fixtures/sample_plpgsql.sql | 27 +++++++++++++++++++++++++++ tests/test_multilang.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/sample_plpgsql.sql diff --git a/graphify/extractors/sql.py b/graphify/extractors/sql.py index fa87982f2..b9ce6fd8c 100644 --- a/graphify/extractors/sql.py +++ b/graphify/extractors/sql.py @@ -191,6 +191,25 @@ def walk(node) -> None: tbl_nid = table_nids.get(tbl_name.lower()) or _make_id(stem, tbl_name) _add_edge(trig_nid, tbl_nid, "triggers", line) + elif t == "ERROR": + # tree-sitter-sql cannot parse PL/pgSQL CREATE FUNCTION/PROCEDURE + # bodies (OUT/INOUT params, tagged dollar quotes, PERFORM, :=) and + # emits an ERROR node instead, silently dropping the object. + # Regex-scan the raw text as fallback, mirroring the + # fb_proc_or_trigger recovery below. One ERROR blob can swallow + # several statements, so scan for every CREATE in it. We deliberately + # do not scan the body for FROM/JOIN references: PL/pgSQL loop + # variables and locals would produce junk reads_from targets. + text = _read(node) + for m in re.finditer( + r"CREATE\s+(?:OR\s+REPLACE\s+)?(?:FUNCTION|PROCEDURE)\s+([\w$.]+)", + text, re.IGNORECASE, + ): + name = m.group(1) + m_line = line + text[: m.start()].count("\n") + nid = _make_id(stem, name) + _add_node(nid, f"{name}()", m_line) + elif t == "fb_proc_or_trigger": text = _read(node) m = re.match( @@ -249,7 +268,7 @@ def _walk_from_refs(node, caller_nid: str, line: int) -> None: if stmt.type == "statement": for child in stmt.children: walk(child) - elif stmt.type in ("fb_proc_or_trigger", "set_term", "declare_external_function"): + elif stmt.type in ("fb_proc_or_trigger", "set_term", "declare_external_function", "ERROR"): walk(stmt) # Global regex fallback: catch any REFERENCES missed due to ERROR nodes in the parse tree diff --git a/tests/fixtures/sample_plpgsql.sql b/tests/fixtures/sample_plpgsql.sql new file mode 100644 index 000000000..de2ef1617 --- /dev/null +++ b/tests/fixtures/sample_plpgsql.sql @@ -0,0 +1,27 @@ +CREATE TABLE accounts ( + id INT PRIMARY KEY, + name TEXT +); + +CREATE FUNCTION exposed.important_function(a int, OUT fuzz text) LANGUAGE plpgsql AS $$ +BEGIN + PERFORM 1; + fuzz := 'x'; +END +$$; + +CREATE OR REPLACE FUNCTION tagged_quote_fn(n int) RETURNS int LANGUAGE plpgsql AS $fn$ +DECLARE + total int := 0; +BEGIN + total := total + n; + RETURN total; +END +$fn$; + +CREATE FUNCTION plain_sql_fn() RETURNS int LANGUAGE sql AS 'SELECT 1'; + +CREATE TABLE audit_log ( + id INT PRIMARY KEY, + account_id INT REFERENCES accounts +); diff --git a/tests/test_multilang.py b/tests/test_multilang.py index 5ad78f724..0cc26539f 100644 --- a/tests/test_multilang.py +++ b/tests/test_multilang.py @@ -513,3 +513,32 @@ def test_sql_schema_qualified_alter_fk(): for e in fk_edges: assert e["source"] in node_ids, f"dangling source: {e['source']}" assert e["target"] in node_ids, f"dangling target: {e['target']}" + +def test_sql_plpgsql_functions_survive_parse_errors(): + """PL/pgSQL bodies make tree-sitter-sql emit ERROR nodes; the functions + must still be extracted (#1910), without cascading into later statements.""" + r = _extract_sql_or_skip("sample_plpgsql.sql") + labels = [n["label"] for n in r["nodes"]] + # Both PL/pgSQL functions extracted, schema-qualified name kept whole + assert "exposed.important_function()" in labels + assert "tagged_quote_fn()" in labels + # Tables before and after the broken functions still extract + assert any("accounts" in l for l in labels) + assert any("audit_log" in l for l in labels) + # No spurious or empty nodes from the error recovery + for l in labels: + assert l, "empty node label" + assert l != "ERROR" + # Every function got a contains edge from the file node + contains_targets = {e["target"] for e in r["edges"] if e["relation"] == "contains"} + fn_ids = {n["id"] for n in r["nodes"] if n["label"].endswith("()")} + assert fn_ids <= contains_targets + +def test_sql_plpgsql_clean_function_not_double_emitted(): + """A cleanly-parsed LANGUAGE sql function in the same file is emitted once.""" + r = _extract_sql_or_skip("sample_plpgsql.sql") + labels = [n["label"] for n in r["nodes"]] + assert labels.count("plain_sql_fn()") == 1 + # And nothing else is duplicated either + ids = [n["id"] for n in r["nodes"]] + assert len(ids) == len(set(ids)) From 0d783915a654b4d386276af18153b94c6f70d732 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 12:18:55 +0100 Subject: [PATCH 32/42] docs(changelog): record #1908/#1909 and #1910 in unreleased 0.9.17 Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b13ef51d0..00058d8f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.17 (unreleased) +- Fix: files that become excluded (`.graphifyignore`/`.gitignore`/`--exclude`) are now pruned from both the graph and the manifest instead of lingering (#1908 / #1909). Two coupled gaps: `save_manifest` retained any prior row that still existed on disk (disk-existence, not scan-membership), so an excluded-but-present file was reported as `deleted` on every run; and the incremental prune set was derived from the manifest alone, so a newly-excluded file's stale nodes carried forward from the existing `graph.json` were never removed (the state every 0.9.16 graph is in). Now the incremental prune set also derives from the existing graph's own `source_file`s minus the post-exclude corpus (in-root only), `save_manifest` prunes rows outside the full scan corpus, and `detect_incremental` distinguishes truly-deleted files from excluded-but-present ones (which are no longer misreported as deleted). +- Fix: PostgreSQL PL/pgSQL functions are no longer silently dropped (#1910). `CREATE FUNCTION ... LANGUAGE plpgsql AS $$...$$` with `OUT` params, tagged dollar-quotes, or procedural body statements parses as a tree-sitter `ERROR` node, which the SQL extractor skipped entirely. It now recovers the function/procedure name from an ERROR node via the same regex-fallback pattern the extractor already uses elsewhere, so the function node and its `contains` edge are kept (the unparseable body is left opaque) and surrounding statements are unaffected. + + - Security/privacy follow-up: nodes whose `source_file` was never dispatched are now dropped from the graph, not just skipped from the cache (#1895). The #1757 guard stopped a mis-attributed node from clobbering another file's cache entry, but the node itself still flowed into `graph.json`; it is now filtered out of the merged result (real-file, non-dispatched attributions only), consistent with the cache rejection. - Fix: `manifest.json` now records every successfully-extracted file, not just the zero-node ones (#1897). The #933 stamping filter compared root-relative node `source_file`s against absolute `detect()` paths, so it dropped every freshly-extracted semantic document from the manifest and broke the incremental-update baseline. Both sides are now resolved before comparison; genuinely omitted/zero-node docs stay unstamped so they retry. - Fix: `graphify hook install` now registers the `graph.json` union merge driver that the README and CHANGELOG have long documented (#1902). It writes the `merge.graphify` config via `git config` and an idempotent, append-only `graphify-out/graph.json merge=graphify` line in `.gitattributes`; `uninstall` removes them. From 2851f31a07def99a7e4f8380d2d74f9839f8b4fd Mon Sep 17 00:00:00 2001 From: Kookwater Date: Wed, 15 Jul 2026 10:42:35 +0200 Subject: [PATCH 33/42] fix: treat .cjs (explicit CommonJS) as a code extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit .cjs was half-registered: the language maps in build.py and extract.py already routed it to the JS grammar, but it was missing from CODE_EXTENSIONS, the extractor _DISPATCH, _LANG_FAMILY, and the JS resolution/cache suffix sets — so .cjs sources (Electron main/preload scripts, CommonJS escape hatches in "type": "module" packages) were silently skipped during a build: classified as non-code and never handed to the JS extractor. Add .cjs alongside .mjs in the six lists that gate/route JS files (detect.py, extract.py _DISPATCH, analyze.py _LANG_FAMILY, extractors/models.py cache-bypass, extractors/resolution.py resolve exts, cli.py _HOOK_SOURCE_EXTS), with regression locks mirroring the .mts/.cts fix (#1607). Real-world impact: an Electron app whose ~2000-line main.cjs backend was invisible went from 201 to 294 nodes and 256 to 441 edges on rebuild. Co-Authored-By: Claude Opus 4.8 --- graphify/analyze.py | 2 +- graphify/cli.py | 2 +- graphify/detect.py | 2 +- graphify/extract.py | 1 + graphify/extractors/models.py | 2 +- graphify/extractors/resolution.py | 2 +- tests/test_cjs_module_extension.py | 90 ++++++++++++++++++++++++++++++ 7 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 tests/test_cjs_module_extension.py diff --git a/graphify/analyze.py b/graphify/analyze.py index 7f3eb72ff..cb6c76484 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -23,7 +23,7 @@ # Language families — extensions sharing a runtime can legitimately call each other _LANG_FAMILY: dict[str, str] = { **{e: "python" for e in (".py", ".pyw")}, - **{e: "js" for e in (".js", ".jsx", ".mjs", ".ejs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte")}, + **{e: "js" for e in (".js", ".jsx", ".mjs", ".cjs", ".ejs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte")}, **{e: "go" for e in (".go",)}, **{e: "rust" for e in (".rs",)}, **{e: "jvm" for e in (".java", ".kt", ".kts", ".scala")}, diff --git a/graphify/cli.py b/graphify/cli.py index 6d037332a..5061d85fa 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -38,7 +38,7 @@ } }, ensure_ascii=False, separators=(",", ":")) + "\n" _HOOK_SOURCE_EXTS = ( - '.py', '.js', '.ts', '.tsx', '.jsx', '.astro', '.vue', '.svelte', '.go', + '.py', '.js', '.cjs', '.ts', '.tsx', '.jsx', '.astro', '.vue', '.svelte', '.go', '.rs', '.java', '.rb', '.c', '.h', '.cpp', '.hpp', '.cc', '.cs', '.kt', '.swift', '.php', '.scala', '.lua', '.sh', '.md', '.rst', '.txt', '.mdx', ) diff --git a/graphify/detect.py b/graphify/detect.py index 195b5e543..1c1c27c3e 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -27,7 +27,7 @@ class FileType(str, Enum): _MANIFEST_PATH = str(out_path("manifest.json")) -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index 0343a5673..12a4f108e 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -3838,6 +3838,7 @@ def add_existing_edge(edge: dict) -> None: ".js": extract_js, ".jsx": extract_js, ".mjs": extract_js, + ".cjs": extract_js, ".ts": extract_js, ".tsx": extract_js, ".mts": extract_js, diff --git a/graphify/extractors/models.py b/graphify/extractors/models.py index 56dbbc0dd..e6da5173e 100644 --- a/graphify/extractors/models.py +++ b/graphify/extractors/models.py @@ -8,7 +8,7 @@ _WORKSPACE_PACKAGE_CACHE: dict[str, dict[str, Path]] = {} -_JS_CACHE_BYPASS_SUFFIXES = {".js", ".jsx", ".mjs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte"} +_JS_CACHE_BYPASS_SUFFIXES = {".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte"} @dataclass class LanguageConfig: diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index d46b4cbcc..9736cccef 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -21,7 +21,7 @@ _WORKSPACE_MANIFEST_NAMES = ("pnpm-workspace.yaml", "package.json") -_JS_RESOLVE_EXTS = (".ts", ".tsx", ".mts", ".cts", ".svelte", ".js", ".jsx", ".mjs") +_JS_RESOLVE_EXTS = (".ts", ".tsx", ".mts", ".cts", ".svelte", ".js", ".jsx", ".mjs", ".cjs") _JS_INDEX_FILES = ("index.ts", "index.tsx", "index.svelte", "index.js", "index.jsx", "index.mjs") diff --git a/tests/test_cjs_module_extension.py b/tests/test_cjs_module_extension.py new file mode 100644 index 000000000..f94f962ff --- /dev/null +++ b/tests/test_cjs_module_extension.py @@ -0,0 +1,90 @@ +"""CommonJS module extension (`.cjs`) is treated as code. + +`.cjs` is the explicit-CommonJS counterpart of `.mjs` (used pervasively in +Electron main/preload scripts and in `"type": "module"` packages that need a +CommonJS escape hatch). The language maps in `build.py` and `extract.py` +already routed `.cjs` to the JS grammar, but the extension was missing from +`CODE_EXTENSIONS`, the extractor `_DISPATCH`, `_LANG_FAMILY`, and the JS +resolution/cache sets — so `.cjs` sources were silently skipped during a build +(detected as non-code and never handed to the JS extractor). These are +regression locks for the extension sets plus an end-to-end extraction proving +`.cjs` parses identically to `.js`. + +Same shape of gap (and fix) as `.mts`/`.cts` in +tests/test_typescript_module_extensions.py. +""" +from __future__ import annotations + +from pathlib import Path + + +def _labels(r): + return [n["label"] for n in r["nodes"]] + + +def test_cjs_registered_as_code(): + from graphify.detect import CODE_EXTENSIONS + assert ".cjs" in CODE_EXTENSIONS + + +def test_cjs_in_extractor_dispatch(): + from graphify.extract import _DISPATCH, extract_js + assert _DISPATCH.get(".cjs") is extract_js + + +def test_cjs_in_js_language_family(): + from graphify.analyze import _LANG_FAMILY + assert _LANG_FAMILY.get(".cjs") == "js" + + +def test_cjs_in_js_resolution_sets(): + from graphify.extract import _JS_CACHE_BYPASS_SUFFIXES, _JS_RESOLVE_EXTS + assert ".cjs" in _JS_RESOLVE_EXTS + assert ".cjs" in _JS_CACHE_BYPASS_SUFFIXES + + +def test_cjs_in_hook_source_exts(): + from graphify.cli import _HOOK_SOURCE_EXTS + assert ".cjs" in _HOOK_SOURCE_EXTS + + +# A representative CommonJS source: require() imports, a class, a function, and +# module.exports — the shape of an Electron main-process script. +_CJS_SOURCE = ( + "const path = require('path');\n" + "const { app, BrowserWindow } = require('electron');\n" + "class WindowManager {\n" + " open() { return new BrowserWindow(); }\n" + "}\n" + "function createWindow() {\n" + " const manager = new WindowManager();\n" + " return manager.open();\n" + "}\n" + "module.exports = { createWindow };\n" +) + + +def _extract(tmp_path: Path, ext: str): + from graphify.extract import extract_js + f = tmp_path / f"main{ext}" + f.write_text(_CJS_SOURCE, encoding="utf-8") + return extract_js(f) + + +def test_cjs_extracts_like_js(tmp_path): + # `.cjs` must parse identically to the same source saved as `.js` — same + # node set (require() imports, class, function) modulo the file-node label. + cjs = _extract(tmp_path, ".cjs") + js = _extract(tmp_path, ".js") + assert "error" not in cjs + cjs_labels = set(_labels(cjs)) + js_labels = set(_labels(js)) + assert any("WindowManager" in label for label in cjs_labels), ( + ".cjs class declaration missing — file was not parsed as JS" + ) + assert any("createWindow" in label for label in cjs_labels), ( + ".cjs function declaration missing — file was not parsed as JS" + ) + assert {label for label in cjs_labels if not label.endswith(".cjs")} == { + label for label in js_labels if not label.endswith(".js") + } From 76f5bc98fe7a49c8b6aba38c95e1609017d9d771 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 13:26:09 +0100 Subject: [PATCH 34/42] fix(watch): stop AST-quick-scanning docs that already have semantic nodes (#1915) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _rebuild_code fed Markdown/doc files (.md/.mdx/.qmd/.skill) into the AST quick-scan while _reconcile_existing_graph preserved the same docs' semantic (LLM) nodes, so every semantic-backed doc was represented twice (heading nodes + concept nodes) and the graph bloated ~4x vs the CLI `graphify . --update` path. Semantic now supersedes AST per doc source: before choosing extract_targets, read the existing graph's source identities that carry semantic doc nodes (non-_origin=="ast", gated on file_type=="document" so pre-#1865 marker-less graphs aren't misread) and exclude those docs from the quick-scan in both the full-rebuild and changed_paths branches. They stay in code_files, so the #1795 fail-closed corpus check and the shrink accounting still cover them — a previously-bloated graph self-heals on the next full rebuild (the AST ownership rule drops the stale heading nodes) without the shrink guard refusing the smaller write. On incremental rebuilds the excluded docs never enter rebuilt/node-evicted identities, mirroring #1865's tier-scoped edge rule at the node level, so a doc's semantic nodes and edges survive. Docs with no semantic layer (and brand-new docs) keep the no-LLM quick-scan structure from #09b33b7. --- graphify/watch.py | 63 ++++++++++++++- tests/test_watch.py | 186 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 247 insertions(+), 2 deletions(-) diff --git a/graphify/watch.py b/graphify/watch.py index 57364228a..a4eee5883 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -876,16 +876,64 @@ def _rebuild_code( code_files = [Path(f) for f in detected['files']['code']] # Include document files that have AST extractors (e.g. .md, .mdx, .qmd) + ast_doc_files: list[Path] = [] for doc_file in detected['files'].get('document', []): p = Path(doc_file) if _get_extractor(p) is not None: code_files.append(p) + ast_doc_files.append(p) existing_graph = out / "graph.json" if not code_files and not existing_graph.exists(): print("[graphify watch] No code files found - nothing to rebuild.") return False + # #1915: a document that already carries SEMANTIC (LLM) nodes in the + # existing graph must not ALSO be AST-quick-scanned — otherwise every + # rebuild mints heading nodes on top of the preserved semantic nodes + # and the doc is represented twice (~4x graph bloat vs the CLI update + # path, which AST-extracts only code). Semantic supersedes AST per doc + # source: the quick-scan stays as a fallback for docs with no semantic + # layer (the no-LLM doc-structure feature, #09b33b7) and for brand-new + # docs the graph has never seen. These docs stay in ``code_files`` so + # corpus membership (#1795 fail-closed deletion evidence) and the + # shrink accounting below still cover them — a previously-bloated + # graph must be allowed to self-heal on a full rebuild without the + # shrink-guard refusing the smaller write. + semantic_doc_files: set[Path] = set() + if ast_doc_files and existing_graph.exists(): + try: + check_graph_file_size_cap(existing_graph) + prior = json.loads(existing_graph.read_text(encoding="utf-8")) + prior_paths = _StoredSourcePaths( + prior, + out=out, + project_root=project_root, + watch_root=watch_root, + normalize_source=_nsf, + ) + # Semantic doc nodes lack the AST origin marker. Gate on + # file_type=="document" as well so a pre-#1865 graph whose + # AST nodes lack the ``_origin`` marker isn't misread as + # semantic-backed via some other marker-less node kind. + semantic_doc_identities: set[str] = set() + for node in prior.get("nodes", []): + if node.get("_origin") == "ast": + continue + if node.get("file_type") != "document": + continue + identity = prior_paths.identity(node.get("source_file")) + if identity: + semantic_doc_identities.add(identity) + if semantic_doc_identities: + semantic_doc_files = { + p for p in ast_doc_files + if prior_paths.absolute_identity(str(p), project_root) + in semantic_doc_identities + } + except Exception: + semantic_doc_files = set() + # Incremental path: when the caller passed an explicit change list, # extract only changed-and-still-existing files. Deleted paths are # tracked separately so their stale nodes can be evicted below. @@ -898,6 +946,12 @@ def _add_deleted_source(path: Path) -> None: if changed_paths is not None: code_set = {Path(os.path.abspath(p)) for p in code_files} + # #1915: semantic-backed docs are never AST-quick-scanned; their + # semantic nodes are the sole representation. Mirroring #1865's + # tier-scoped edge rule at the node level, they also must NOT + # enter extract_targets (hence rebuilt/node-evicted identities) on + # an incremental rebuild, or their semantic nodes would be wiped. + semantic_doc_set = {Path(os.path.abspath(p)) for p in semantic_doc_files} wanted: list[Path] = [] change_root = Path.cwd().resolve() for raw in changed_paths: @@ -908,7 +962,7 @@ def _add_deleted_source(path: Path) -> None: ) tracked = next((cand for cand in candidates if cand.exists() and cand in code_set), None) if tracked is not None: - if tracked not in wanted: + if tracked not in wanted and tracked not in semantic_doc_set: wanted.append(tracked) continue @@ -938,7 +992,12 @@ def _add_deleted_source(path: Path) -> None: return True extract_targets = wanted else: - extract_targets = code_files + # Full rebuild: skip the AST quick-scan for semantic-backed docs + # (#1915). They remain in code_files, so stale _origin=="ast" + # heading nodes from a previously-bloated graph are dropped by the + # full-rebuild AST ownership rule while the shrink accounting + # below still counts the doc as a rebuilt source. + extract_targets = [p for p in code_files if p not in semantic_doc_files] commit = _git_head() result = extract(extract_targets, cache_root=watch_root) if extract_targets else { diff --git a/tests/test_watch.py b/tests/test_watch.py index 8df9c1b84..fc91cf181 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -1649,3 +1649,189 @@ def test_rebuild_code_still_evicts_when_excluded_file_is_also_deleted(tmp_path): labels = {n["label"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} assert "brainstorm.md" not in labels, "deleted file's nodes must still be evicted" assert "login()" in labels + + +# --- #1915: semantic-backed docs must not be double-represented by the AST quick-scan --- + + +_SEMANTIC_GUIDE_IDS = {"guide_doc", "auth_flow", "session_model"} +_AST_GUIDE_IDS = {"guide", "guide_overview", "guide_setup", "guide_usage"} + + +def _seed_semantic_doc_graph(corpus): + """Build a code-only graph, then add guide.md represented ONLY semantically. + + Mimics a graph produced by the CLI ``graphify . --update`` path: code AST + nodes plus a semantic (LLM) layer for the document — a ``_doc`` node + and concept nodes, none carrying the ``_origin`` marker — and NO AST + heading nodes for the doc. + """ + from graphify.watch import _rebuild_code + + corpus.mkdir() + (corpus / "app.py").write_text( + "def handle_login():\n return 1\n", encoding="utf-8" + ) + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + (corpus / "guide.md").write_text( + "# Overview\n\nIntro.\n\n## Setup\n\nSteps.\n\n## Usage\n\nMore.\n", + encoding="utf-8", + ) + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + code_node_id = next( + n["id"] for n in data["nodes"] if n.get("source_file") == "app.py" + ) + data["nodes"].extend([ + {"id": "guide_doc", "label": "Guide", "file_type": "document", + "source_file": "guide.md"}, + {"id": "auth_flow", "label": "Auth Flow", "file_type": "concept", + "source_file": "guide.md"}, + {"id": "session_model", "label": "Session Model", "file_type": "concept", + "source_file": "guide.md"}, + ]) + data["links"].extend([ + {"source": "guide_doc", "target": "auth_flow", "relation": "explains", + "confidence": "INFERRED", "source_file": "guide.md"}, + {"source": "auth_flow", "target": code_node_id, + "relation": "implemented_by", "confidence": "INFERRED", + "source_file": "guide.md"}, + ]) + graph_path.write_text(json.dumps(data), encoding="utf-8") + return graph_path + + +def test_rebuild_code_semantic_doc_not_double_represented_on_full_rebuild(tmp_path): + """#1915: a full _rebuild_code must not AST-quick-scan a doc whose semantic + (LLM) nodes already represent it. Before the fix the quick-scan minted + heading nodes ON TOP of the preserved semantic nodes, representing every + doc twice (~4x bloated graph vs the CLI update path).""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + graph_path = _seed_semantic_doc_graph(corpus) + before = json.loads(graph_path.read_text(encoding="utf-8")) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + after_ids = {n["id"] for n in after["nodes"]} + assert _SEMANTIC_GUIDE_IDS <= after_ids, "semantic doc nodes must be preserved" + assert not (_AST_GUIDE_IDS & after_ids), ( + "AST heading nodes minted for a semantic-backed doc (#1915)" + ) + assert len(after["nodes"]) == len(before["nodes"]), ( + f"node count inflated {len(before['nodes'])} -> {len(after['nodes'])} (#1915)" + ) + + +@pytest.mark.parametrize( + "changed", + [[Path("guide.md")], [Path("guide.md"), Path("app.py")]], + ids=["doc-only", "doc-plus-code"], +) +def test_rebuild_code_incremental_preserves_semantic_doc_nodes_and_edges( + tmp_path, changed +): + """#1915: an incremental rebuild whose change set includes a semantic-backed + doc must not wipe the doc's semantic nodes or their edges — re-extraction + owns only a source's AST tier (node-level mirror of #1865's edge rule).""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + graph_path = _seed_semantic_doc_graph(corpus) + + assert _rebuild_code( + corpus, changed_paths=changed, no_cluster=True, acquire_lock=False + ) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + after_ids = {n["id"] for n in after["nodes"]} + assert _SEMANTIC_GUIDE_IDS <= after_ids, ( + "semantic doc nodes wiped by an incremental rebuild" + ) + relations = { + (e.get("source"), e.get("target"), e.get("relation")) + for e in after["links"] + } + assert ("guide_doc", "auth_flow", "explains") in relations, ( + "semantic doc edge dropped by an incremental rebuild" + ) + assert any( + src == "auth_flow" and rel == "implemented_by" + for src, _tgt, rel in relations + ), "doc-to-code semantic edge dropped by an incremental rebuild" + assert not (_AST_GUIDE_IDS & after_ids), ( + "incremental rebuild AST-quick-scanned a semantic-backed doc (#1915)" + ) + + +def test_rebuild_code_quick_scans_doc_without_semantic_nodes(tmp_path): + """#09b33b7 guard: a doc with NO semantic layer still gets the AST + quick-scan so no-LLM corpora keep their heading structure — #1915's + semantic-supersedes-AST rule must not regress the fallback.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text("def f():\n return 1\n", encoding="utf-8") + (corpus / "notes.md").write_text("# Alpha\n\n## Beta\n", encoding="utf-8") + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert {"notes", "notes_alpha", "notes_beta"} <= ids + + # A rebuild over the existing graph (still no semantic nodes for the doc) + # keeps quick-scanning it rather than dropping its structure. + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert {"notes", "notes_alpha", "notes_beta"} <= ids + + +def test_rebuild_code_polluted_graph_self_heals_on_full_rebuild(tmp_path): + """#1915: a graph already bloated by the bug (semantic doc nodes PLUS stale + _origin=="ast" heading nodes for the same doc) sheds the heading nodes on + the next full rebuild via the AST ownership rule — and the shrink guard + accepts the smaller write without --force.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text( + "def handle_login():\n return 1\n", encoding="utf-8" + ) + (corpus / "guide.md").write_text( + "# Overview\n\n## Setup\n\n## Usage\n", encoding="utf-8" + ) + # Initial build quick-scans guide.md (no semantic layer yet): AST nodes. + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + assert _AST_GUIDE_IDS <= {n["id"] for n in data["nodes"]} + + # Layer the semantic representation on top -> the double-represented state. + data["nodes"].extend([ + {"id": "guide_doc", "label": "Guide", "file_type": "document", + "source_file": "guide.md"}, + {"id": "auth_flow", "label": "Auth Flow", "file_type": "concept", + "source_file": "guide.md"}, + ]) + data["links"].append({ + "source": "guide_doc", "target": "auth_flow", "relation": "explains", + "confidence": "INFERRED", "source_file": "guide.md", + }) + graph_path.write_text(json.dumps(data), encoding="utf-8") + nodes_before = len(data["nodes"]) + + # No force=True: the self-heal shrink must be accepted by the guard. + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + after_ids = {n["id"] for n in after["nodes"]} + assert {"guide_doc", "auth_flow"} <= after_ids + assert not (_AST_GUIDE_IDS & after_ids), ( + "stale AST heading nodes for a semantic-backed doc must self-heal away" + ) + assert len(after["nodes"]) < nodes_before, "polluted graph should shrink" From d916c61559e2e4f6885e3eb09816ce51bd611544 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 13:29:24 +0100 Subject: [PATCH 35/42] fix(cache): stop persisting dangling edges/hyperedges in the semantic cache (#1916) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit save_semantic_cache groups nodes/edges/hyperedges by their own source_file and the write loop skips a group whose path is a ghost (not .is_file(), silently) or out-of-scope per the #1757 guard — but an edge or hyperedge in an ALLOWED group that references a node id from a skipped group was still written verbatim, so on replay (check_semantic_cache) it dangled forever. The #1895 filter does not cover this: it cleans the in-memory merged result, while the checkpoint writes the cache BEFORE it runs and replay bypasses it entirely. Fix at the cache layer (the authoritative write path): before the write loop, compute the node ids belonging to groups that will be skipped — mirroring BOTH skip branches — minus ids also defined in a group that will be written (duplicate-attribution nodes must not be over-pruned). Each written group then drops edges whose source/target is a skipped id and hyperedges whose member list intersects them (whole-hyperedge drop, mirroring #1895). Pruning runs on the incoming result only, so with merge_existing=True (the llm.py checkpoint path) the prior cached entry's valid edges survive the union untouched. Everything is gated on allowed_source_files being provided, so unscoped callers stay byte-identical. Complementary hardening in build_from_json: hyperedges were copied into G.graph["hyperedges"] verbatim without member validation, so a dangling hyperedge reached graph.json even from a live (non-cache) extraction. Members are now remapped via the same normalization the pairwise-edge loop uses and pruned when they still don't resolve; a hyperedge with no surviving member is dropped whole with a stderr warning. (Single-member hyperedges are legal in this codebase — per-file flows in the #1574 tests — so the drop threshold is zero survivors, not two.) Tests: scoped save with an edge to an out-of-scope real file, the same with a ghost source_file, whole-hyperedge drop, unscoped save preserved byte-identically (raw cache entry compared), merge_existing keeping the prior slice's valid edges, and build_from_json pruning a dangling hyperedge member / dropping an all-dangling hyperedge. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/build.py | 35 ++++++++- graphify/cache.py | 56 ++++++++++++++ tests/test_build.py | 24 ++++++ tests/test_cache.py | 174 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 288 insertions(+), 1 deletion(-) diff --git a/graphify/build.py b/graphify/build.py index 428fa1cd4..b05c83978 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -755,10 +755,43 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # Relativize hyperedge source_file the same way nodes and edges are # (above), so to_json — which has no root and writes G.graph["hyperedges"] # verbatim — never leaks an absolute path from a semantic subagent (#1418). + kept_hyperedges = [] for he in hyperedges: if isinstance(he, dict) and he.get("source_file"): he["source_file"] = _norm_source_file(he["source_file"], _root) - G.graph["hyperedges"] = hyperedges + # Validate members against the built node set (#1916): a hyperedge + # member absent from the graph used to be copied into + # G.graph["hyperedges"] verbatim and reach graph.json dangling, + # even from a live (non-cache) extraction. Mirror the pairwise-edge + # handling above: remap mismatched ids via normalization first, + # then drop members that still don't resolve; drop the hyperedge + # itself when no valid member remains (single-member hyperedges + # are legal in this codebase, e.g. a per-file flow, so we prune + # rather than require two survivors). + if isinstance(he, dict) and isinstance(he.get("nodes"), list): + valid_members = [] + for m in he["nodes"]: + try: + hash(m) + except TypeError: + continue + if m not in node_set and isinstance(m, str): + m = norm_to_id.get(_normalize_id(m), m) + if m in node_set: + valid_members.append(m) + if not valid_members: + print( + f"[graphify] WARNING: dropping hyperedge " + f"{he.get('id', '?')!r} — none of its members " + f"{he.get('nodes')!r} match built nodes.", + file=sys.stderr, + ) + continue + if valid_members != he["nodes"]: + he["nodes"] = valid_members + kept_hyperedges.append(he) + if kept_hyperedges: + G.graph["hyperedges"] = kept_hyperedges return G diff --git a/graphify/cache.py b/graphify/cache.py index 1a7d42d06..3055db7b0 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -615,6 +615,62 @@ def resolved_source_path(value: str | Path) -> Path: if allowed_source_files is not None: allowed_paths = {resolved_source_path(path) for path in allowed_source_files} + def group_skipped(fpath: str) -> bool: + """Mirror the write-loop skip condition for one source_file group.""" + p = resolved_source_path(fpath) + return not p.is_file() or (allowed_paths is not None and p not in allowed_paths) + + # Dangling-reference pruning (#1916). A node group is skipped by the write + # loop below when its source_file is not a real file (ghost path) or is + # out-of-scope per the #1757 guard — but an edge/hyperedge in an ALLOWED + # group that references a node id from a skipped group used to be written + # verbatim, so on replay (check_semantic_cache) it dangled forever (the + # #1895 merged-result filter runs AFTER this checkpoint write and is + # bypassed entirely on replay). Compute the node ids that will be skipped + # and drop any to-be-written edge whose endpoint — or hyperedge whose + # member (whole-hyperedge drop, mirroring #1895) — references one. Gated + # on allowed_source_files so unscoped callers stay byte-identical. + if allowed_paths is not None: + skipped_ids: set = set() + written_ids: set = set() + for fpath, result in by_file.items(): + target = skipped_ids if group_skipped(fpath) else written_ids + for n in result["nodes"]: + nid = n.get("id") + if nid is None: + continue + try: + hash(nid) + except TypeError: + continue + target.add(nid) + # A duplicate-attribution node (defined in a skipped AND a written + # group) still reaches the cache — don't over-prune references to it. + skipped_ids -= written_ids + if skipped_ids: + + def edge_dangles(e: dict) -> bool: + try: + return e.get("source") in skipped_ids or e.get("target") in skipped_ids + except TypeError: + # Non-hashable endpoint from an untrusted result; leave it + # to build-time validation rather than fail the save. + return False + + def hyperedge_dangles(h: dict) -> bool: + try: + return bool(skipped_ids & set(h.get("nodes") or [])) + except TypeError: + return False + + for fpath, result in by_file.items(): + if group_skipped(fpath): + continue + result["edges"] = [e for e in result["edges"] if not edge_dangles(e)] + result["hyperedges"] = [ + h for h in result["hyperedges"] if not hyperedge_dangles(h) + ] + saved = 0 for fpath, result in by_file.items(): p = resolved_source_path(fpath) diff --git a/tests/test_build.py b/tests/test_build.py index 872a2e12b..ed87e6b11 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -970,3 +970,27 @@ def test_doc_twin_merge_does_not_touch_code_symbols(): } G = build_from_json(ext, directed=False) assert {"m_foo", "m_foo_doc"} <= set(G.nodes()) + + +def test_build_from_json_prunes_dangling_hyperedge_members(capsys): + """#1916: build_from_json used to copy hyperedges into G.graph["hyperedges"] + verbatim without validating members, so a dangling member reached graph.json + even from a live (non-cache) extraction. Members absent from the built node + set are pruned — matching how dangling pairwise edges are skipped — and a + hyperedge with no surviving member is dropped whole.""" + ext = { + "nodes": [ + {"id": "alpha", "label": "alpha", "file_type": "code", "source_file": "a.py"}, + {"id": "beta", "label": "beta", "file_type": "code", "source_file": "a.py"}, + ], + "edges": [], + "hyperedges": [ + {"id": "he_partial", "nodes": ["alpha", "beta", "ghost_member"], "source_file": "a.py"}, + {"id": "he_all_ghost", "nodes": ["ghost1", "ghost2"], "source_file": "a.py"}, + ], + } + G = build_from_json(ext) + hes = {h["id"]: h for h in G.graph.get("hyperedges", [])} + assert set(hes) == {"he_partial"}, "an all-dangling hyperedge must be dropped" + assert hes["he_partial"]["nodes"] == ["alpha", "beta"] + assert "he_all_ghost" in capsys.readouterr().err diff --git a/tests/test_cache.py b/tests/test_cache.py index d772d1ec4..8588df996 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -586,3 +586,177 @@ def test_save_semantic_cache_merge_existing_unions(tmp_path): ids = {n["id"] for n in cached["nodes"]} assert ids == {"a", "b"}, "merge_existing must union both chunk slices" assert len(cached["edges"]) == 1 + + +def test_save_semantic_cache_drops_edges_to_out_of_scope_nodes(tmp_path): + """#1916: an edge in an ALLOWED file's group referencing a node grouped + under an out-of-scope REAL file used to be written verbatim, so on replay + (check_semantic_cache) it dangled forever — the #1895 merged-result filter + runs after this checkpoint write and is bypassed entirely on replay. The + written entry must carry no reference to the skipped id, while a + duplicate-attribution node (also defined in a written group) must not be + over-pruned.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + allowed = tmp_path / "allowed.md" + allowed.write_text("# Allowed\n") + outside = tmp_path / "outside.md" + outside.write_text("# Outside\n") + + nodes = [ + {"id": "kept", "source_file": "allowed.md"}, + {"id": "stray", "source_file": "outside.md"}, + # duplicate attribution: same id defined in a written AND a skipped group + {"id": "dup", "source_file": "allowed.md"}, + {"id": "dup", "source_file": "outside.md"}, + ] + edges = [ + {"source": "kept", "target": "stray", "source_file": "allowed.md"}, + {"source": "stray", "target": "kept", "source_file": "allowed.md"}, + {"source": "kept", "target": "dup", "source_file": "allowed.md"}, + ] + with pytest.warns(RuntimeWarning, match="out-of-scope source_file"): + saved = save_semantic_cache( + nodes, edges, root=tmp_path, allowed_source_files=["allowed.md"] + ) + assert saved == 1 + + cached_nodes, cached_edges, _, uncached = check_semantic_cache( + [str(allowed)], root=tmp_path + ) + assert uncached == [] + assert {n["id"] for n in cached_nodes} == {"kept", "dup"} + pairs = [(e["source"], e["target"]) for e in cached_edges] + assert pairs == [("kept", "dup")], "edges touching the skipped id must be dropped" + + +def test_save_semantic_cache_drops_edges_to_ghost_file_nodes(tmp_path): + """#1916 (ghost variant): a node group whose source_file does not exist is + silently skipped by the write loop; edges in a written group referencing + its node ids must not survive into the cache.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + real = tmp_path / "real.md" + real.write_text("# Real\n") + + nodes = [ + {"id": "kept", "source_file": "real.md"}, + {"id": "phantom", "source_file": "ghost.md"}, # no such file on disk + ] + edges = [ + {"source": "kept", "target": "phantom", "source_file": "real.md"}, + {"source": "kept", "target": "kept", "relation": "self", "source_file": "real.md"}, + ] + saved = save_semantic_cache( + nodes, edges, root=tmp_path, allowed_source_files=["real.md"] + ) + assert saved == 1 + + cached_nodes, cached_edges, _, uncached = check_semantic_cache( + [str(real)], root=tmp_path + ) + assert uncached == [] + assert {n["id"] for n in cached_nodes} == {"kept"} + pairs = [(e["source"], e["target"]) for e in cached_edges] + assert pairs == [("kept", "kept")] + + +def test_save_semantic_cache_drops_hyperedges_touching_skipped_nodes(tmp_path): + """#1916: a hyperedge whose member list intersects the skipped ids is + dropped whole (mirroring the #1895 semantics), while hyperedges over + surviving nodes are kept.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + allowed = tmp_path / "allowed.md" + allowed.write_text("# Allowed\n") + outside = tmp_path / "outside.md" + outside.write_text("# Outside\n") + + nodes = [ + {"id": "kept", "source_file": "allowed.md"}, + {"id": "kept2", "source_file": "allowed.md"}, + {"id": "stray", "source_file": "outside.md"}, + ] + hyperedges = [ + {"id": "he_bad", "nodes": ["kept", "stray"], "source_file": "allowed.md"}, + {"id": "he_ok", "nodes": ["kept", "kept2"], "source_file": "allowed.md"}, + ] + with pytest.warns(RuntimeWarning, match="out-of-scope source_file"): + save_semantic_cache( + nodes, [], hyperedges, root=tmp_path, allowed_source_files=["allowed.md"] + ) + + _, _, cached_hyperedges, uncached = check_semantic_cache( + [str(allowed)], root=tmp_path + ) + assert uncached == [] + assert {h["id"] for h in cached_hyperedges} == {"he_ok"} + + +def test_save_semantic_cache_unscoped_preserves_dangling_refs_verbatim(tmp_path): + """#1916 guard-rail: unscoped callers (allowed_source_files=None) must stay + byte-identical — no pruning happens even when an edge or hyperedge + references a node grouped under a ghost file.""" + from graphify.cache import save_semantic_cache + + doc = tmp_path / "doc.md" + doc.write_text("# Doc\n") + + nodes = [ + {"id": "a", "source_file": "doc.md"}, + {"id": "ghost_n", "source_file": "ghost.md"}, # skipped group (no file) + ] + edges = [{"source": "a", "target": "ghost_n", "source_file": "doc.md"}] + hyperedges = [{"id": "he", "nodes": ["a", "ghost_n"], "source_file": "doc.md"}] + + saved = save_semantic_cache(nodes, edges, hyperedges, root=tmp_path) + assert saved == 1 + + import json + raw = json.loads( + (cache_dir(tmp_path, "semantic") / f"{file_hash(doc, tmp_path)}.json").read_text() + ) + assert raw["edges"] == edges + assert raw["hyperedges"] == hyperedges + + +def test_save_semantic_cache_merge_existing_prunes_only_incoming(tmp_path): + """#1916 + #1715: with merge_existing=True (the llm.py checkpoint path), + only the INCOMING slice is pruned before the union — the prior cached + entry's valid edges must survive untouched.""" + from graphify.cache import save_semantic_cache + + big = tmp_path / "big.md" + big.write_text("# Big\n") + other = tmp_path / "other.md" + other.write_text("# Other\n") + + # checkpoint 1: a clean slice + save_semantic_cache( + [{"id": "a", "source_file": "big.md"}], + [{"source": "a", "target": "a", "relation": "self", "source_file": "big.md"}], + root=tmp_path, + merge_existing=True, + allowed_source_files=["big.md"], + ) + # checkpoint 2: incoming slice with a dangling edge to an out-of-scope node + nodes2 = [ + {"id": "b", "source_file": "big.md"}, + {"id": "stray", "source_file": "other.md"}, + ] + edges2 = [ + {"source": "b", "target": "stray", "source_file": "big.md"}, + {"source": "a", "target": "b", "source_file": "big.md"}, + ] + with pytest.warns(RuntimeWarning, match="out-of-scope source_file"): + save_semantic_cache( + nodes2, edges2, root=tmp_path, merge_existing=True, + allowed_source_files=["big.md"], + ) + + cached = load_cached(big, root=tmp_path, kind="semantic") + assert {n["id"] for n in cached["nodes"]} == {"a", "b"} + pairs = [(e["source"], e["target"]) for e in cached["edges"]] + assert ("a", "a") in pairs, "prior entry's valid edge must survive the union" + assert ("a", "b") in pairs, "incoming valid edge must be kept" + assert not any("stray" in p for p in pairs) From b7ddee3c28dc2c5df806931898ecf8b702730868 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 13:55:32 +0100 Subject: [PATCH 36/42] fix(extract): make --mode deep effective over a warm cache; add --force (#1894) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `graphify extract --mode deep` over a warm tree was a silent no-op, for three stacked reasons: 1. The semantic cache ignored mode: deep runs were served standard-mode entries (and vice versa). check_semantic_cache/save_semantic_cache now take `mode` (default None, byte-identical when omitted so older installed callers keep working) and map it to a namespaced kind — cache/semantic/ for None, cache/semantic-{mode}/ otherwise. The per-chunk checkpoint in llm.extract_corpus_parallel and the extract / cache-check consumers thread the run's mode through (cache-check grows --mode/--deep). cached_files, clear_cache, and prune_semantic_cache sweep BOTH namespaces; prune uses the same live-hash set for both (liveness is content-based, mode-independent) so semantic-deep/ can't regrow the #1527 unbounded-orphan problem and inherits the files_by_type-derived exclusion gating for free. 2. extract had no --force — the flag was silently swallowed by the parser's unknown-arg fallthrough. It is now real (plus GRAPHIFY_FORCE env parity with `update`): force disables the incremental gate so detection is a full scan and skips the semantic cache READ so every semantic file re-dispatches, while the post-run save and manifest stamping still happen. 3. The incremental gate dispatched zero files on a warm unchanged tree before the cache was ever consulted, so namespacing alone couldn't fix the repro. In deep+incremental runs the semantic pass now widens to the full live doc/paper/image set from detect_incremental's files_by_type (already exclusion-filtered, #1908/#1909) and lets the mode-namespaced cache decide hits/misses, with a loud count line so the first deep run's full re-dispatch is visible. Skill-side threading of mode is deliberately deferred to PR-2; mode defaults keep generated skills byte-compatible. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/__main__.py | 2 + graphify/cache.py | 84 +++++++++++++------- graphify/cli.py | 71 +++++++++++++++-- graphify/llm.py | 4 + tests/test_cache.py | 143 ++++++++++++++++++++++++++++++++++ tests/test_chunking.py | 32 ++++++++ tests/test_extract_cli.py | 158 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 462 insertions(+), 32 deletions(-) diff --git a/graphify/__main__.py b/graphify/__main__.py index c5151b8e3..b4243275a 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -597,6 +597,8 @@ def _run_cli() -> None: print(" proxy, gateways): set ANTHROPIC_BASE_URL and ANTHROPIC_MODEL") print(" --model M override backend default model") print(" --mode deep aggressive INFERRED-edge semantic extraction") + print(" --force full re-scan and re-dispatch: skip the incremental") + print(" manifest gate and semantic cache reads (env: GRAPHIFY_FORCE=1)") print(" --max-workers N AST extraction subprocess count (default: cpu_count)") print(" --token-budget N per-chunk token cap for semantic extraction (default: 60000)") print(" --max-concurrency N parallel semantic chunks in flight (default: 4; set 1 for local LLMs)") diff --git a/graphify/cache.py b/graphify/cache.py index 3055db7b0..15795d441 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -90,7 +90,8 @@ def _body_content(content: bytes) -> bytes: # size+mtime_ns are unchanged — same trade-off as make(1). # Correctness risks: `touch` causes a harmless extra re-hash; same-size edits # within NFS second-resolution mtime have a 1-second window (same as make). -# Use `graphify extract --force` to bypass when needed. +# `graphify extract --force` / `graphify update --force` (or GRAPHIFY_FORCE=1) +# skip the cache reads and re-dispatch everything when needed (#1894). _stat_index: dict[str, dict] = {} _stat_index_root: Path | None = None _stat_index_dirty: bool = False @@ -340,13 +341,15 @@ def _absolutize_source_files_in(payload: dict, root: Path) -> None: def cache_dir(root: Path = Path("."), kind: str = "ast") -> Path: """Returns the cache directory for ``kind`` - creates it if needed. - kind is "ast" or "semantic". Separate subdirectories prevent semantic cache + kind is "ast", "semantic", or a mode-namespaced semantic kind such as + "semantic-deep" (#1894). Separate subdirectories prevent semantic cache entries from overwriting AST cache entries for the same source_file (#582). AST entries live in graphify-out/cache/ast/v{version}/ — namespaced by graphify version because they depend on extractor code, not just file contents. Semantic entries live unversioned in graphify-out/cache/semantic/ - (re-extraction costs LLM calls). + (re-extraction costs LLM calls); deep-mode entries live beside them in + graphify-out/cache/semantic-deep/. """ _out = Path(_GRAPHIFY_OUT) base = _out if _out.is_absolute() else Path(root).resolve() / _out @@ -467,8 +470,10 @@ def cached_files(root: Path = Path(".")) -> set[str]: # Legacy flat entries if base.is_dir(): hashes.update(p.stem for p in base.glob("*.json")) - # Namespaced entries (ast/ recursively, covering per-version subdirs) - for kind, pattern in (("ast", "**/*.json"), ("semantic", "*.json")): + # Namespaced entries (ast/ recursively, covering per-version subdirs; + # semantic-deep/ holds --mode deep entries, #1894) + for kind, pattern in (("ast", "**/*.json"), ("semantic", "*.json"), + ("semantic-deep", "*.json")): d = base / kind if d.is_dir(): hashes.update(p.stem for p in d.glob(pattern)) @@ -476,14 +481,17 @@ def cached_files(root: Path = Path(".")) -> set[str]: def clear_cache(root: Path = Path(".")) -> None: - """Delete all cache entries (ast/, semantic/, and legacy flat entries).""" + """Delete all cache entries (ast/, semantic/, semantic-deep/, and legacy + flat entries).""" base = Path(root).resolve() / _GRAPHIFY_OUT / "cache" # Legacy flat entries if base.is_dir(): for f in base.glob("*.json"): f.unlink() - # Namespaced entries (ast/ recursively, covering per-version subdirs) - for kind, pattern in (("ast", "**/*.json"), ("semantic", "*.json")): + # Namespaced entries (ast/ recursively, covering per-version subdirs; + # semantic-deep/ holds --mode deep entries, #1894) + for kind, pattern in (("ast", "**/*.json"), ("semantic", "*.json"), + ("semantic-deep", "*.json")): d = base / kind if d.is_dir(): for f in d.glob(pattern): @@ -500,11 +508,16 @@ def prune_semantic_cache(root: Path, live_hashes: set[str]) -> int: AST version-cleanup, so every content change or file deletion leaves a permanent orphan entry that accumulates unbounded. - This sweeps ``cache/semantic/*.json`` and deletes any entry whose stem (the - content hash) is not in ``live_hashes`` — the hashes of the current live - document set. ``*.tmp`` atomic-write temporaries are skipped, and only this - directory is touched (never ``cache/ast/**`` or anything else). The - unversioned design is preserved: we prune by liveness, not by version. + This sweeps ``cache/semantic/*.json`` AND ``cache/semantic-deep/*.json`` + (the ``--mode deep`` namespace, #1894) and deletes any entry whose stem + (the content hash) is not in ``live_hashes`` — the hashes of the current + live document set. Both namespaces are pruned against the SAME live set: + liveness is content-based and mode-independent, so a hash that is live for + one namespace is live for both. Skipping the deep namespace would re-grow + the unbounded-orphan problem this function fixed (#1527). ``*.tmp`` + atomic-write temporaries are skipped, and only these directories are + touched (never ``cache/ast/**`` or anything else). The unversioned design + is preserved: we prune by liveness, not by version. Best-effort, mirroring :func:`_cleanup_stale_ast_entries`: each unlink is wrapped in ``try/except OSError`` and a failure is ignored. The worst-case @@ -513,30 +526,40 @@ def prune_semantic_cache(root: Path, live_hashes: set[str]) -> int: """ _out = Path(_GRAPHIFY_OUT) base = _out if _out.is_absolute() else Path(root).resolve() / _out - semantic_dir = base / "cache" / "semantic" - if not semantic_dir.is_dir(): - return 0 pruned = 0 - for entry in semantic_dir.glob("*.json"): - if entry.stem in live_hashes: + for kind in ("semantic", "semantic-deep"): + semantic_dir = base / "cache" / kind + if not semantic_dir.is_dir(): continue - try: - entry.unlink() - pruned += 1 - except OSError: - pass + for entry in semantic_dir.glob("*.json"): + if entry.stem in live_hashes: + continue + try: + entry.unlink() + pruned += 1 + except OSError: + pass return pruned def check_semantic_cache( files: list[str], root: Path = Path("."), + mode: str | None = None, ) -> tuple[list[dict], list[dict], list[dict], list[str]]: """Check semantic extraction cache for a list of absolute file paths. Returns (cached_nodes, cached_edges, cached_hyperedges, uncached_files). Uncached files need Claude extraction; cached files are merged directly. + + ``mode`` selects the cache namespace: ``None`` (the default) reads + ``cache/semantic/`` — byte-identical to the historical behavior, so + existing callers that omit it (including older installed skill flows) + are unaffected. A non-None mode (e.g. ``"deep"``) reads + ``cache/semantic-{mode}/`` instead, so deep-mode results never shadow + (or get shadowed by) standard-mode entries for the same content (#1894). """ + kind = "semantic" if mode is None else f"semantic-{mode}" cached_nodes: list[dict] = [] cached_edges: list[dict] = [] cached_hyperedges: list[dict] = [] @@ -546,7 +569,7 @@ def check_semantic_cache( p = Path(fpath) if not p.is_absolute(): p = Path(root) / p - result = load_cached(p, root, kind="semantic") + result = load_cached(p, root, kind=kind) if result is not None: cached_nodes.extend(result.get("nodes", [])) cached_edges.extend(result.get("edges", [])) @@ -564,6 +587,7 @@ def save_semantic_cache( root: Path = Path("."), merge_existing: bool = False, allowed_source_files: Iterable[str | Path] | None = None, + mode: str | None = None, ) -> int: """Save semantic extraction results to cache, keyed by source_file. @@ -571,6 +595,13 @@ def save_semantic_cache( under cache/semantic/ (separate from AST entries in cache/ast/) to prevent hash-key collisions (#582). + ``mode`` selects the cache namespace, mirroring + :func:`check_semantic_cache`: ``None`` (the default) writes + ``cache/semantic/`` — byte-identical to the historical behavior for + existing callers that omit it — while a non-None mode (e.g. ``"deep"``) + writes ``cache/semantic-{mode}/`` so richer deep-mode results never + overwrite standard-mode entries and vice versa (#1894). + When ``merge_existing`` is True, any already-cached entry for a file is unioned with the new results before saving instead of being overwritten. This lets callers checkpoint incrementally (e.g. once per chunk) without @@ -584,6 +615,7 @@ def save_semantic_cache( """ from collections import defaultdict + kind = "semantic" if mode is None else f"semantic-{mode}" by_file: dict[str, dict] = defaultdict(lambda: {"nodes": [], "edges": [], "hyperedges": []}) for n in nodes: src = n.get("source_file", "") @@ -684,13 +716,13 @@ def hyperedge_dangles(h: dict) -> bool: ) continue if merge_existing: - prev = load_cached(p, root, kind="semantic") + prev = load_cached(p, root, kind=kind) if prev: result = { "nodes": (prev.get("nodes", []) or []) + result["nodes"], "edges": (prev.get("edges", []) or []) + result["edges"], "hyperedges": (prev.get("hyperedges", []) or []) + result["hyperedges"], } - save_cached(p, result, root, kind="semantic") + save_cached(p, result, root, kind=kind) saved += 1 return saved diff --git a/graphify/cli.py b/graphify/cli.py index 5061d85fa..f2ae1da1b 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -2138,6 +2138,9 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": cli_exclude_hubs: float | None = None cli_excludes: list[str] = [] cli_timing: bool = False + # --force parity with `graphify update`: the flag or GRAPHIFY_FORCE=1 + # disables the incremental gate and skips semantic-cache reads (#1894). + force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") def _parse_int(name: str, raw: str) -> int: try: @@ -2228,6 +2231,8 @@ def _parse_float(name: str, raw: str) -> float: elif a == "--cargo": cli_cargo = True i += 1 + elif a == "--force": + force = True; i += 1 elif a == "--timing": cli_timing = True; i += 1 else: @@ -2277,6 +2282,11 @@ def _parse_float(name: str, raw: str) -> float: manifest_path = graphify_out / "manifest.json" existing_graph_path = graphify_out / "graph.json" incremental_mode = manifest_path.exists() and existing_graph_path.exists() if has_path else False + # --force: full scan, not the manifest-gated incremental diff — a warm + # unchanged tree would otherwise dispatch zero files (#1894). + incremental_mode = incremental_mode and not force + if force: + print("[graphify extract] --force: full re-scan, semantic cache reads skipped") if not has_path: code_files = [] @@ -2343,6 +2353,30 @@ def _parse_float(name: str, raw: str) -> float: doc_files = [] paper_files = [] image_files = [] + if deep_mode and incremental_mode and not code_only: + # Deep mode reads/writes its own cache namespace + # (cache/semantic-deep/), so the manifest's changed-file gate is + # not a valid proxy for deep coverage: over a warm unchanged tree + # it dispatches zero files and `--mode deep` silently no-ops + # (#1894). Widen the semantic pass to the FULL live + # doc/paper/image set (``files_by_type`` from detect_incremental, + # which already excludes excluded files) and let the + # mode-namespaced cache decide hits/misses — the first deep run + # re-dispatches everything (deep namespace cold), later deep runs + # hit the deep cache. + _deep_all = [ + Path(p) + for _ftype in ("document", "paper", "image") + for p in files_by_type.get(_ftype, []) + ] + if len(_deep_all) != len(semantic_files): + print( + f"[graphify extract] deep mode: widening semantic pass from " + f"{len(semantic_files)} changed to {len(_deep_all)} live " + f"doc/paper/image file(s); the deep semantic cache decides " + f"what is re-extracted" + ) + semantic_files = _deep_all if incremental_mode: # Excluded-but-alive files are reported separately from deletions # (#1908): they still exist on disk, the scan just stopped @@ -2495,11 +2529,21 @@ def _parse_float(name: str, raw: str) -> float: } sem_cache_hits = 0 sem_cache_misses = 0 + # Deep mode uses its own namespace (cache/semantic-deep/) so deep and + # standard results for the same content never shadow each other (#1894). + sem_cache_mode = "deep" if deep_mode else None if semantic_files: sem_paths_str = [str(p) for p in semantic_files] - cached_nodes, cached_edges, cached_hyperedges, uncached_paths = ( - _check_semantic_cache(sem_paths_str, root=out_root) - ) + if force: + # --force: skip the cache READ so every semantic file is + # re-dispatched; the save below still runs so the fresh + # results replace the stale entries. + cached_nodes, cached_edges, cached_hyperedges = [], [], [] + uncached_paths = list(sem_paths_str) + else: + cached_nodes, cached_edges, cached_hyperedges, uncached_paths = ( + _check_semantic_cache(sem_paths_str, root=out_root, mode=sem_cache_mode) + ) sem_cache_hits = len(semantic_files) - len(uncached_paths) sem_cache_misses = len(uncached_paths) sem_result["nodes"].extend(cached_nodes) @@ -2570,6 +2614,7 @@ def _progress(idx: int, total: int, _result: dict) -> None: fresh.get("hyperedges", []), root=out_root, allowed_source_files=uncached_paths, + mode=sem_cache_mode, ) except Exception as exc: print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr) @@ -2880,27 +2925,41 @@ def _progress(idx: int, total: int, _result: dict) -> None: stages.total() elif cmd == "cache-check": - # graphify cache-check [--root ] + # graphify cache-check [--root ] [--mode | --deep] # Reads file paths (one per line) from , checks semantic cache. + # --mode deep (or --deep) checks the cache/semantic-deep/ namespace + # written by `extract --mode deep` instead of cache/semantic/ (#1894). # Writes: # graphify-out/.graphify_cached.json — already-cached nodes/edges/hyperedges # graphify-out/.graphify_uncached.txt — paths that need extraction # Stdout: "Cache: N hit, M miss" from graphify.cache import check_semantic_cache if len(sys.argv) < 3: - print("Usage: graphify cache-check [--root ]", file=sys.stderr) + print("Usage: graphify cache-check [--root ] [--mode | --deep]", file=sys.stderr) sys.exit(1) files_from = Path(sys.argv[2]) root = Path(".") + cache_mode: str | None = None i = 3 while i < len(sys.argv): if sys.argv[i] == "--root" and i + 1 < len(sys.argv): root = Path(sys.argv[i + 1]) i += 2 + elif sys.argv[i] == "--mode" and i + 1 < len(sys.argv): + cache_mode = sys.argv[i + 1] + i += 2 + elif sys.argv[i].startswith("--mode="): + cache_mode = sys.argv[i].split("=", 1)[1] + i += 1 + elif sys.argv[i] == "--deep": + cache_mode = "deep" + i += 1 else: i += 1 files = [f for f in files_from.read_text(encoding="utf-8").splitlines() if f.strip()] - cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(files, root) + cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache( + files, root, mode=cache_mode + ) out = root / _GRAPHIFY_OUT out.mkdir(parents=True, exist_ok=True) if cached_nodes or cached_edges or cached_hyperedges: diff --git a/graphify/llm.py b/graphify/llm.py index 52e2b48b5..e3dbf681b 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -1924,6 +1924,9 @@ def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: # chunk leaked the FileSlice object into the allowlist and the write # raised TypeError, silently defeating the checkpoint.) allowed = [unit_path(item) for item in chunk] + # Deep-mode results checkpoint into their own namespace + # (cache/semantic-deep/) so a deep run never overwrites standard + # entries — and a later standard run never serves deep ones (#1894). _scs( result.get("nodes", []), result.get("edges", []), @@ -1931,6 +1934,7 @@ def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: root=root, merge_existing=True, allowed_source_files=allowed, + mode="deep" if deep_mode else None, ) except Exception as _exc: # noqa: BLE001 — checkpoint is best-effort print(f"[graphify] incremental cache checkpoint failed: {_exc}", file=sys.stderr) diff --git a/tests/test_cache.py b/tests/test_cache.py index 8588df996..ff89a3261 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -570,6 +570,149 @@ def test_save_semantic_cache_rejects_out_of_scope_source_file(tmp_path): assert protected_cache["hyperedges"] == [] +# --- #1894: mode-namespaced semantic cache ----------------------------------- +# `extract --mode deep` produces richer results than standard extraction, so +# deep entries live in their own namespace (cache/semantic-deep/). mode=None +# must stay byte-identical to the historical behavior: older installed skill +# flows call check/save without the parameter and must be unaffected. + +def test_semantic_cache_deep_mode_roundtrip_under_deep_namespace(tmp_path): + """mode='deep' saves under cache/semantic-deep/ and reads back from it.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# Doc\n\nBody.\n") + saved = save_semantic_cache( + [{"id": "deep_n", "source_file": "doc.md"}], [], root=tmp_path, mode="deep" + ) + assert saved == 1 + + deep_dir = tmp_path / "graphify-out" / "cache" / "semantic-deep" + h = file_hash(f, tmp_path) + assert (deep_dir / f"{h}.json").exists(), ( + "deep entry must land under cache/semantic-deep/" + ) + # And NOT in the plain namespace. + plain_dir = tmp_path / "graphify-out" / "cache" / "semantic" + assert not (plain_dir / f"{h}.json").exists() + + nodes, edges, hyper, uncached = check_semantic_cache( + [str(f)], root=tmp_path, mode="deep" + ) + assert [n["id"] for n in nodes] == ["deep_n"] + assert uncached == [] + + +def test_semantic_cache_deep_invisible_to_plain_reads_and_vice_versa(tmp_path): + """Deep entries must not satisfy mode=None reads (and plain entries must + not satisfy deep reads) — the namespaces are fully isolated.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + deep_doc = tmp_path / "deep.md" + deep_doc.write_text("# Deep\n") + plain_doc = tmp_path / "plain.md" + plain_doc.write_text("# Plain\n") + + save_semantic_cache([{"id": "d", "source_file": "deep.md"}], [], + root=tmp_path, mode="deep") + save_semantic_cache([{"id": "p", "source_file": "plain.md"}], [], + root=tmp_path) # mode omitted: historical call shape + + # Plain read: deep entry is a miss, plain entry is a hit. + nodes, _, _, uncached = check_semantic_cache( + [str(deep_doc), str(plain_doc)], root=tmp_path + ) + assert [n["id"] for n in nodes] == ["p"] + assert uncached == [str(deep_doc)] + + # Deep read: mirror image. + nodes, _, _, uncached = check_semantic_cache( + [str(deep_doc), str(plain_doc)], root=tmp_path, mode="deep" + ) + assert [n["id"] for n in nodes] == ["d"] + assert uncached == [str(plain_doc)] + + +def test_semantic_cache_mode_none_layout_unchanged(tmp_path): + """Omitting mode writes exactly the historical cache/semantic/ layout — + forward-compat for older installed callers that never pass mode.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# Doc\n") + save_semantic_cache([{"id": "n", "source_file": "doc.md"}], [], root=tmp_path) + h = file_hash(f, tmp_path) + assert (tmp_path / "graphify-out" / "cache" / "semantic" / f"{h}.json").exists() + assert not (tmp_path / "graphify-out" / "cache" / "semantic-deep").exists(), ( + "mode=None must never create the deep namespace" + ) + nodes, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path) + assert [n["id"] for n in nodes] == ["n"] and uncached == [] + + +def test_clear_cache_removes_deep_namespace(tmp_path): + """clear_cache sweeps cache/semantic-deep/ alongside semantic/ and ast/.""" + from graphify.cache import save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# Doc\n") + save_semantic_cache([{"id": "p", "source_file": "doc.md"}], [], root=tmp_path) + save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], + root=tmp_path, mode="deep") + base = tmp_path / "graphify-out" / "cache" + assert list((base / "semantic").glob("*.json")) + assert list((base / "semantic-deep").glob("*.json")) + + clear_cache(tmp_path) + assert not list(base.rglob("*.json")), ( + "clear_cache must remove entries in BOTH semantic namespaces" + ) + + +def test_cached_files_includes_deep_namespace(tmp_path): + """cached_files reports deep-namespace entries too.""" + from graphify.cache import save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# Doc\n") + save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], + root=tmp_path, mode="deep") + assert file_hash(f, tmp_path) in cached_files(tmp_path) + + +def test_semantic_prune_sweeps_both_namespaces_against_same_live_set(tmp_path): + """#1894 follow-up to #1527: prune must sweep cache/semantic/ AND + cache/semantic-deep/ against the SAME live-hash set (liveness is + content-based, mode-independent). Orphans go in both namespaces; live + entries survive in both.""" + from graphify.cache import prune_semantic_cache, save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# A\n\nContent A.\n") + h_old = file_hash(f, tmp_path) + save_semantic_cache([{"id": "pa", "source_file": "doc.md"}], [], root=tmp_path) + save_semantic_cache([{"id": "da", "source_file": "doc.md"}], [], + root=tmp_path, mode="deep") + + f.write_text("# B\n\nContent B.\n") + h_live = file_hash(f, tmp_path) + save_semantic_cache([{"id": "pb", "source_file": "doc.md"}], [], root=tmp_path) + save_semantic_cache([{"id": "db", "source_file": "doc.md"}], [], + root=tmp_path, mode="deep") + + plain_dir = tmp_path / "graphify-out" / "cache" / "semantic" + deep_dir = tmp_path / "graphify-out" / "cache" / "semantic-deep" + for d in (plain_dir, deep_dir): + assert (d / f"{h_old}.json").exists() + assert (d / f"{h_live}.json").exists() + + pruned = prune_semantic_cache(tmp_path, {h_live}) + assert pruned == 2, "one orphan in EACH namespace must be pruned" + for d in (plain_dir, deep_dir): + assert not (d / f"{h_old}.json").exists(), f"orphan survived in {d.name}" + assert (d / f"{h_live}.json").exists(), f"live entry pruned from {d.name}" + + def test_save_semantic_cache_merge_existing_unions(tmp_path): """#1715: merge_existing=True unions with the prior entry so a file split across chunks (checkpointed per chunk) keeps every slice.""" diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 546503ac1..a978a5daa 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -326,6 +326,38 @@ def stray(chunk, **kwargs): assert a_cache and any(n["id"] == "a_ok" for n in a_cache["nodes"]) +def test_checkpoint_writes_deep_namespace_in_deep_mode(tmp_path): + """#1894: the per-chunk checkpoint must follow the run's mode — a + deep_mode=True run checkpoints into cache/semantic-deep/, leaving the + standard cache/semantic/ namespace untouched (and vice versa).""" + from graphify.llm import extract_corpus_parallel + from graphify.cache import load_cached + + doc = tmp_path / "doc.md" + doc.write_text("# Doc\n\nsome content\n") + + def ok(chunk, **kwargs): + return { + "nodes": [{"id": "d1", "source_file": "doc.md", "file_type": "document"}], + "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1, + } + + with patch("graphify.llm.extract_files_direct", side_effect=ok): + extract_corpus_parallel( + [doc], backend="kimi", root=tmp_path, + token_budget=None, chunk_size=1, max_concurrency=1, + deep_mode=True, + ) + + deep = load_cached(doc, tmp_path, kind="semantic-deep") + assert deep and [n["id"] for n in deep["nodes"]] == ["d1"], ( + "deep-mode checkpoint must land in cache/semantic-deep/" + ) + assert load_cached(doc, tmp_path, kind="semantic") is None, ( + "deep-mode checkpoint must not write the standard semantic namespace" + ) + + def test_omitted_documents_are_reconciled_and_warned(tmp_path, capsys): """#1890: a chunk can return a clean, non-empty response that omits some of the documents it was given. Those docs must not vanish silently — the run reports diff --git a/tests/test_extract_cli.py b/tests/test_extract_cli.py index 3442a4877..6ef1b2c22 100644 --- a/tests/test_extract_cli.py +++ b/tests/test_extract_cli.py @@ -222,6 +222,164 @@ def test_stamped_manifest_files_normalizes_both_sides(tmp_path): assert out["document"] == [str(fresh_doc), str(cached_doc)] +# --- #1894: --force and deep-mode dispatch over a warm cache ----------------- + +def _recording_extractor(calls): + """extract_corpus_parallel stand-in that records each dispatch.""" + def _extract(paths, **kwargs): + calls.append({"paths": [str(p) for p in paths], "kwargs": kwargs}) + on_chunk = kwargs.get("on_chunk_done") + if on_chunk: + on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) + return { + "nodes": [{"id": "readme", "source_file": "README.md", + "file_type": "document"}], + "edges": [], + "hyperedges": [], + "input_tokens": 10, + "output_tokens": 5, + } + return _extract + + +def _run_extract(monkeypatch, argv): + monkeypatch.setattr(mainmod.sys, "argv", argv) + try: + mainmod.main() + except SystemExit as exc: + assert exc.code in (None, 0), f"unexpected exit code {exc.code}" + + +def test_extract_mode_deep_dispatches_over_warm_cache(monkeypatch, tmp_path): + """#1894 repro: over a warm manifest + warm standard semantic cache, + `extract --mode deep` was a silent no-op — the incremental gate dispatched + zero files before the cache was ever consulted, and the cache key ignored + mode anyway. Deep must re-dispatch on the first deep run (deep namespace + cold) and be served from cache/semantic-deep/ on the second.""" + corpus = _make_corpus(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + monkeypatch.delenv("GRAPHIFY_FORCE", raising=False) + calls: list[dict] = [] + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", + _recording_extractor(calls)) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + # No --out: the default layout (graphify-out/ beside the sources) keeps the + # CLI-level cache write's root anchored at the corpus, so the stub's + # root-relative source_file resolves (real runs also checkpoint per chunk + # inside llm.extract_corpus_parallel, which this stub replaces). + base = ["graphify", "extract", str(corpus), "--backend", "claude", + "--no-cluster"] + + # Run 1: cold standard extraction — warms manifest + plain semantic cache. + _run_extract(monkeypatch, base) + assert len(calls) == 1 + + # Sanity: a warm standard re-run dispatches nothing (expected behavior). + _run_extract(monkeypatch, base) + assert len(calls) == 1 + + # The repro: warm tree + --mode deep MUST dispatch. + _run_extract(monkeypatch, base + ["--mode", "deep"]) + assert len(calls) == 2, ( + "--mode deep over a warm cache must re-dispatch (#1894)" + ) + assert calls[1]["paths"] == [str(corpus / "README.md")] + assert calls[1]["kwargs"].get("deep_mode") is True + + # Second deep run: served from the (now warm) deep namespace, no dispatch. + _run_extract(monkeypatch, base + ["--mode", "deep"]) + assert len(calls) == 2, ( + "second deep run must be served from cache/semantic-deep/" + ) + # The deep entry landed in its own namespace, not cache/semantic/. + assert any((corpus / "graphify-out" / "cache" / "semantic-deep").glob("*.json")) + + +def test_extract_force_flag_redispatches_and_stamps_manifest(monkeypatch, tmp_path): + """extract accepts --force: a warm tree re-dispatches every semantic file + (cache read skipped, incremental gate off) and the manifest is still + stamped afterward (#1897-compatible full coverage).""" + import json + + corpus = _make_corpus(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + monkeypatch.delenv("GRAPHIFY_FORCE", raising=False) + calls: list[dict] = [] + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", + _recording_extractor(calls)) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + base = ["graphify", "extract", str(corpus), "--backend", "claude", + "--no-cluster"] + + _run_extract(monkeypatch, base) + assert len(calls) == 1 + _run_extract(monkeypatch, base) # warm: no dispatch + assert len(calls) == 1 + + _run_extract(monkeypatch, base + ["--force"]) + assert len(calls) == 2, ( + "--force over a warm tree must re-dispatch every semantic file" + ) + assert calls[1]["paths"] == [str(corpus / "README.md")] + + # The forced run still wrote the semantic cache and stamped the manifest. + assert any((corpus / "graphify-out" / "cache" / "semantic").glob("*.json")) + manifest = json.loads( + (corpus / "graphify-out" / "manifest.json").read_text() + ) + assert manifest.get("README.md", {}).get("semantic_hash"), ( + "forced re-dispatch must still stamp the manifest" + ) + assert manifest.get("main.go", {}).get("semantic_hash") + + +def test_extract_graphify_force_env_redispatches(monkeypatch, tmp_path): + """GRAPHIFY_FORCE=1 behaves like --force (env parity with `update`).""" + corpus = _make_corpus(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + monkeypatch.delenv("GRAPHIFY_FORCE", raising=False) + calls: list[dict] = [] + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", + _recording_extractor(calls)) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + base = ["graphify", "extract", str(corpus), "--backend", "claude", + "--no-cluster"] + + _run_extract(monkeypatch, base) + assert len(calls) == 1 + _run_extract(monkeypatch, base) # warm: no dispatch + assert len(calls) == 1 + + monkeypatch.setenv("GRAPHIFY_FORCE", "1") + _run_extract(monkeypatch, base) + assert len(calls) == 2, "GRAPHIFY_FORCE=1 must force a re-dispatch" + + +def test_cache_check_mode_deep_reads_deep_namespace(monkeypatch, tmp_path, capsys): + """cache-check --mode deep consults cache/semantic-deep/; without the flag + it keeps reading cache/semantic/ (deep entries are invisible to it).""" + from graphify.cache import save_semantic_cache + + doc = tmp_path / "doc.md" + doc.write_text("# Doc\n") + save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], + root=tmp_path, mode="deep") + files_from = tmp_path / "files.txt" + files_from.write_text(str(doc) + "\n") + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + _run_extract(monkeypatch, ["graphify", "cache-check", str(files_from), + "--root", str(tmp_path)]) + assert "Cache: 0 hit, 1 miss" in capsys.readouterr().out + + _run_extract(monkeypatch, ["graphify", "cache-check", str(files_from), + "--root", str(tmp_path), "--mode", "deep"]) + assert "Cache: 1 hit, 0 miss" in capsys.readouterr().out + + def _code_only_corpus(tmp_path): """A corpus with only code — no docs/papers/images.""" (tmp_path / "auth.py").write_text( From 1c280e4576596a3d026fbf92e56fd31da2aa1536 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 14:16:14 +0100 Subject: [PATCH 37/42] docs(changelog): record #1894(PR-1) #1916 #1915 #1912 in unreleased 0.9.17 Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00058d8f1..b0cd98d3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.17 (unreleased) +- Fix: `--mode deep` is now effective over a warm cache instead of a silent no-op (#1894). The semantic cache is namespaced by mode (`semantic` vs `semantic-deep`) so a shallow-cached file no longer satisfies a deep run; `graphify extract` gains `--force` (and honors `GRAPHIFY_FORCE`) to bypass the incremental gate and the cache read; and a deep incremental run widens its dispatch to the full live doc set so the deep namespace actually gets populated. Cache prune/clear now sweep both namespaces so the deep cache can't accumulate orphans. (The skill-side flow that passes the mode through is a follow-up; the CLI is complete and backward compatible — the new `mode` argument defaults to the existing behavior.) +- Fix: the semantic cache no longer persists dangling edges/hyperedges (#1916). When a node group was skipped on write (out-of-scope per the batch guard, or a ghost `source_file`), edges/hyperedges in the kept groups that referenced those never-written nodes were saved anyway and re-surfaced on every cache replay. Those references are now pruned at write time (gated on the scoping allowlist, so unscoped callers are unchanged), and `build_from_json` validates hyperedge members against the node set so a dangling hyperedge can't reach `graph.json` even from a live extraction. +- Fix: `graphify update`/watch no longer produces a bloated graph by double-representing documents (#1915). `_rebuild_code` AST-quick-scanned Markdown/doc files and then preserved their existing semantic (LLM) nodes on top, so each doc appeared twice (a real corpus came out ~4x). A doc that already has semantic nodes in the graph is no longer AST-quick-scanned (its semantic nodes are the sole representation), while a doc with no semantic layer still gets the structural quick-scan; incremental rebuilds now preserve a doc's semantic nodes instead of evicting them, and previously-bloated graphs self-heal on the next full rebuild. +- Fix: `.cjs` (explicit CommonJS) files are now recognized as code and parsed with the JavaScript grammar (#1912, thanks @Kookwater). The extension was half-registered (present in some internal maps but missing from the classification and dispatch sets), so `.cjs` files were silently dropped. + + - Fix: files that become excluded (`.graphifyignore`/`.gitignore`/`--exclude`) are now pruned from both the graph and the manifest instead of lingering (#1908 / #1909). Two coupled gaps: `save_manifest` retained any prior row that still existed on disk (disk-existence, not scan-membership), so an excluded-but-present file was reported as `deleted` on every run; and the incremental prune set was derived from the manifest alone, so a newly-excluded file's stale nodes carried forward from the existing `graph.json` were never removed (the state every 0.9.16 graph is in). Now the incremental prune set also derives from the existing graph's own `source_file`s minus the post-exclude corpus (in-root only), `save_manifest` prunes rows outside the full scan corpus, and `detect_incremental` distinguishes truly-deleted files from excluded-but-present ones (which are no longer misreported as deleted). - Fix: PostgreSQL PL/pgSQL functions are no longer silently dropped (#1910). `CREATE FUNCTION ... LANGUAGE plpgsql AS $$...$$` with `OUT` params, tagged dollar-quotes, or procedural body statements parses as a tree-sitter `ERROR` node, which the SQL extractor skipped entirely. It now recovers the function/procedure name from an ERROR node via the same regex-fallback pattern the extractor already uses elsewhere, so the function node and its `contains` edge are kept (the unparseable body is left opaque) and surrounding statements are unaffected. From ff999a909ab9df4ad9b8244e28002c138c932dc3 Mon Sep 17 00:00:00 2001 From: Sirhan1 Date: Tue, 14 Jul 2026 16:07:42 +0100 Subject: [PATCH 38/42] perf(serve): collapse T+1 per-term scoring passes into one traversal Add _score_query() producing combined ranking + per-term singleton winners in a single graph traversal. _pick_seeds consumes precomputed winners instead of rescoring each term via _score_nodes. Behavior byte-identical to the legacy T+1 path. Supersedes the rescoring loop from #1596 while preserving its per-term coverage guarantee; folds in coverage scaling from #1724 and multiplicity penalty from #1832. Orthogonal to the trigram prefilter from #1431. Benchmark (full sweep in #1889): 1k-100k nodes, 1-10 terms: 1.43x-2.43x median speedup Single-term: no regression (1.70x-1.89x improvement) Traversals: T+1 -> 1 regardless of term count Tests: 3221 passed, 3 skipped. Ruff clean. Graphify graph updated. Refs #1889 --- graphify/serve.py | 173 ++++++++++++++++++---- tests/bench_query_scoring.py | 280 +++++++++++++++++++++++++++++++++++ tests/test_serve.py | 251 +++++++++++++++++++++++++++++-- 3 files changed, 669 insertions(+), 35 deletions(-) create mode 100644 tests/bench_query_scoring.py diff --git a/graphify/serve.py b/graphify/serve.py index c94b048fb..0a0e7bfe3 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -6,6 +6,7 @@ import sys from array import array from pathlib import Path +from typing import NamedTuple import networkx as nx from networkx.readwrite import json_graph from graphify.security import sanitize_label, check_graph_file_size_cap @@ -320,8 +321,63 @@ def _trigram_candidates(G: nx.Graph, needles: list[str], *, guard_frac: float = return [ids[i] for i in sorted(cand)] +class _QueryScores(NamedTuple): + """Per-query scoring result, returned by the private `_score_query` helper. + + `ranked` is the existing ordered `(score, node_id)` ranking produced by the + combined query scorer (the value `_score_nodes` always returned). When the + caller asks for it via `collect_per_term_seeds=True`, `best_seed_by_term` + additionally carries the winning node id for each normalized search token — + the seed `_pick_seeds` would have picked for that token via the now-retired + per-token `_score_nodes([token])` rescoring pass — computed in the *same* + per-node traversal so the query path makes exactly one graph scoring pass + regardless of query length. Empty when `collect_per_term_seeds=False`. + """ + ranked: list[tuple[float, str]] + best_seed_by_term: dict[str, str] + + def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: - scored = [] + """Combined query scorer returning the existing ranked `(score, node_id)` list. + + Backwards-compatible thin wrapper around `_score_query` for path, explain, + tests, and every other caller that only needs the combined ranking. The + per-term seed metadata computed by `_score_query` (when requested) is + discarded here so existing callers see no API or runtime-cost change. + """ + return _score_query(G, terms, collect_per_term_seeds=False).ranked + + +def _score_query( + G: nx.Graph, terms: list[str], *, collect_per_term_seeds: bool +) -> _QueryScores: + """Single-pass combined scorer that optionally also records the best seed + for each normalized query token. + + The combined ranking is byte-identical to what `_score_nodes` produced + before the refactor; `_score_nodes` is now a thin wrapper that asks for + `collect_per_term_seeds=False` and returns only `.ranked`. + + When `collect_per_term_seeds=True`, the per-token singleton winner is + computed alongside the combined score in the *same* per-node visit (it + reuses the same `norm_label` / `label_tokens` / `source` already evaluated + for the combined tier), so `_query_graph_text` can feed `best_seed_by_term` + straight into `_pick_seeds` and skip the T additional whole-graph rescoring + passes the old per-token `_score_nodes([token])` loop ran. + + Singleton-winner semantics match the legacy per-token path exactly. The + score itself mirrors `_score_nodes([token])` with `n_terms == 1` (so the + coverage term is 1 and the per-token tier is unscaled) plus the broader + joined-singlet tier (which also checks `label_tokens` and `nid_lower`). + Tie-break order is (1) highest singleton score, (2) highest graph degree, + (3) shortest displayed label, (4) lexicographically smallest node id — + exactly what `max(tied, key=degree)` over a sort by `(-score, label_len, + nid)` produced in the legacy `_pick_seeds` per-token loop. The combined + trigram candidate set (needles `norm_terms + [joined]`) is a superset of + each per-token `[t]` candidate set, so iterating combined candidates + discovers every non-zero singleton-score node for every term. + """ + scored: list[tuple[float, str]] = [] # Dedupe tokens, order-preserving (as _pick_seeds already does): a repeated # query word must not double-count every tier, and with coverage scaling # below it would also inflate the matched-term ratio (#1602). @@ -342,6 +398,16 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: G.nodes(data=True) if candidate_ids is None else ((nid, G.nodes[nid]) for nid in candidate_ids) ) + # Per-token best tracking, only when the caller (the query path) wants the + # seed metadata. The key tuple is the full multi-key tie-break + # (`(-singleton_score, -degree, label_len, nid)`), so `min` over the + # stored key mirrors the legacy `max(tied, key=degree)` over a + # (-score, label_len, nid)-sorted term_scored list. `None` is comparable + # as "smaller" than every tuple, so the first non-zero candidate seeds the + # entry without a separate `if t not in best_by_term` branch. + best_by_term: dict[str, tuple[tuple, str]] | None = ( + {} if collect_per_term_seeds else None + ) for nid, data in node_iter: norm_label = data.get("norm_label") or _strip_diacritics(data.get("label") or "").lower() bare_label = norm_label.rstrip("()") @@ -352,6 +418,11 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: # driver". label_tokens = " ".join(_search_tokens(data.get("label") or "")) source = (data.get("source_file") or "").lower() + # `nid_lower` is needed both by the full-query tier (`if joined`) and by + # the per-token singleton tier (joined-singlet exact-match check). When + # neither runs (`joined` empty AND not collecting seeds) skip the call; + # this preserves the single-query-time perf where nid_lower was lazy. + nid_lower = nid.lower() if (joined or collect_per_term_seeds) else "" score = 0.0 # Full-query tier: a multi-word query that equals (or prefixes) the whole # label must dominate the per-token bag-of-words sums below, so `path`/ @@ -360,7 +431,6 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: # tier never fires, and every node sharing the token set ties -> arbitrary # node-id sort -> wrong/disconnected endpoint -> false "No path found". if joined: - nid_lower = nid.lower() if joined in (norm_label, bare_label, label_tokens, nid_lower): score += _EXACT_MATCH_BONUS * 10 * joined_w elif ( @@ -386,19 +456,55 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: tiered = 0.0 for t in norm_terms: w = idf.get(t, 1.0) - # Three-tier precedence: exact > prefix > substring (take the - # strongest tier per term so a single term cannot double-count). + # Per-tier contributions for this token, kept separate so the + # singleton tracking below can reuse them without re-evaluating + # the same predicates. Three-tier precedence: exact > prefix > + # substring (take the strongest tier per term so a single term + # cannot double-count). + tier_value = 0.0 + substr_value = 0.0 + source_value = 0.0 if t == norm_label or t == bare_label: - tiered += _EXACT_MATCH_BONUS * w + tier_value = _EXACT_MATCH_BONUS * w matched += 1 elif norm_label.startswith(t) or bare_label.startswith(t): - tiered += _PREFIX_MATCH_BONUS * w + tier_value = _PREFIX_MATCH_BONUS * w matched += 1 elif t in norm_label: - score += _SUBSTRING_MATCH_BONUS * w + substr_value = _SUBSTRING_MATCH_BONUS * w + score += substr_value matched += 1 if t in source: - score += _SOURCE_MATCH_BONUS * w + source_value = _SOURCE_MATCH_BONUS * w + score += source_value + tiered += tier_value + if collect_per_term_seeds and best_by_term is not None: + # Singleton score for [t] on this node, mirroring + # `_score_nodes(G, [t])` exactly (n_terms == 1, no coverage + # scaling). The joined-singlet tier is broader than the per- + # token tier: it also checks `label_tokens` and `nid_lower`, + # matching the legacy single-token `_score_nodes([t])` call + # (where `joined == t`). + if t in (norm_label, bare_label, label_tokens, nid_lower): + singleton = _EXACT_MATCH_BONUS * 10 * w + elif ( + norm_label.startswith(t) + or bare_label.startswith(t) + or label_tokens.startswith(t) + ): + singleton = _PREFIX_MATCH_BONUS * 10 * w + else: + singleton = 0.0 + singleton += tier_value + substr_value + source_value + if singleton > 0: + # Tie-break key mirrors the legacy sort+max(degree): + # (-singleton, -degree, label_len, nid) — the minimum + # tuple wins, exactly matching max(tied, key=degree) + # over (label_len asc, nid asc)-sorted ties. + key = (-singleton, -G.degree(nid), len(data.get("label") or nid), nid) + cur = best_by_term.get(t) + if cur is None or key < cur[0]: + best_by_term[t] = (key, nid) if tiered: score += tiered * (matched / n_terms) ** 2 if score > 0: @@ -406,7 +512,10 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: # Sort by score desc; break ties toward the shorter label so a concise exact # match beats a longer superset that happens to share the same score. scored.sort(key=lambda s: (-s[0], len(G.nodes[s[1]].get("label") or s[1]), s[1])) - return scored + best_seed_by_term: dict[str, str] = {} + if collect_per_term_seeds and best_by_term: + best_seed_by_term = {t: nid for t, (_key, nid) in best_by_term.items()} + return _QueryScores(ranked=scored, best_seed_by_term=best_seed_by_term) def _pick_scored_endpoint(G: nx.Graph, scored: list[tuple[float, str]], query: str) -> str: @@ -439,7 +548,7 @@ def _pick_seeds( gap_ratio: float = 0.2, *, G: "nx.Graph | None" = None, - terms: list[str] | None = None, + best_seed_by_term: dict[str, str] | None = None, ) -> list[str]: """Select BFS seed nodes, stopping when score drops too far below the top. @@ -458,11 +567,12 @@ def _pick_seeds( seeds, so the BFS traversal only ever explores the neighborhood of the one unrelated exact match — see #1445. - When `G` and `terms` are supplied, this guarantees at least one seed per - distinct query term that has any match at all, so one term's incidental - collision cannot starve out the others. Ties within a term are broken by - graph degree (structural centrality), so an isolated incidental match - doesn't out-rank a real, well-connected hub for that term. + When `G` and `best_seed_by_term` are supplied, this guarantees at least one + seed per distinct query term that has any match at all, so one term's + incidental collision cannot starve out the others. The per-token winners + in `best_seed_by_term` are precomputed by `_score_query` (during the same + traversal that produced `scored`) so this function no longer rescores the + graph per term — see #1445 and the `_score_query` docstring. Coverage scaling in _score_nodes (#1602) now dampens a lone collision's exact tier on multi-term queries, which brings label-matching relevant @@ -501,15 +611,20 @@ def _seed_label_key(nid: str) -> str: seen_labels.add(key) seeds.append(nid) - if G is not None and terms: - norm_terms = sorted({tok for t in terms for tok in _search_tokens(t)}) - for term in norm_terms: - term_scored = _score_nodes(G, [term]) - if not term_scored: - continue - best_score = term_scored[0][0] - tied = [nid for s, nid in term_scored if s == best_score] - best_nid = max(tied, key=lambda n: G.degree(n)) if len(tied) > 1 else term_scored[0][1] + if G is not None and best_seed_by_term: + # Guarantee one seed per distinct query term that has any match at all, + # so an incidental exact match on one term cannot starve matches on + # other terms (#1445). Iterate tokens in a deterministic sorted order + # so seeds added by this loop have a stable order independent of dict + # iteration — preserving the legacy `_pick_seeds(terms=...)` behavior + # which iterated `sorted({tok ...})`. Per-token winners arrive + # precomputed in `best_seed_by_term` from `_score_query`'s single + # traversal, so `_pick_seeds` no longer rescoring the graph per term. + # The per-label dedup cap also gates these additions, so the guarantee + # cannot reintroduce a second copy of an already-seeded generic label + # (#1766). + for term in sorted(best_seed_by_term): + best_nid = best_seed_by_term[term] # Honor the same per-label cap so the per-term guarantee can't # reintroduce a second copy of an already-seeded generic label. key = _seed_label_key(best_nid) @@ -753,8 +868,14 @@ def _query_graph_text( context_filters: list[str] | None = None, ) -> str: terms = _query_terms(question) - scored = _score_nodes(G, terms) - start_nodes = _pick_seeds(scored, G=G, terms=terms) + # One graph scoring pass produces both the combined ranking (used to drive + # the gap-based seed selection below) and the per-token singleton winners + # (used by _pick_seeds' per-term guarantee). Previously this was T+1 passes + # — one combined + one per query token — re-walking the whole graph each + # time; on a 100k-node, three-term benchmark ~71% of scoring time was + # spent in those redundant per-term passes. + qs = _score_query(G, terms, collect_per_term_seeds=True) + start_nodes = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) if not start_nodes: return "No matching nodes found." resolved_filters, filter_source = _resolve_context_filters(question, context_filters) diff --git a/tests/bench_query_scoring.py b/tests/bench_query_scoring.py new file mode 100644 index 000000000..375c9449b --- /dev/null +++ b/tests/bench_query_scoring.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +"""Non-CI microbenchmark: single-pass scoring vs legacy per-term rescoring. + +Verifies the single-pass refactor eliminates the T+1 graph-scoring passes a +T-term query used to run, without changing the result. Reports median/min +latency for: + + * legacy — one combined `_score_nodes(G, terms)` call PLUS one + `_score_nodes(G, [token])` call per distinct query token + (the old `_pick_seeds(terms=...)` per-term guarantee loop). + * optimized — one `_score_query(G, terms, collect_per_term_seeds=True)` + call, with the per-token best computed inline in the same + traversal. `_pick_seeds` then consumes `best_seed_by_term` + directly — no rescoring. + +Equality is asserted (ranked, best_seed_by_term, seeds) before timing. The +optimized path's traversal count is invariant in the number of query terms. + +Run it manually; do NOT wire this into CI (wall-clock assertions are flaky): + + uv run python tests/bench_query_scoring.py \\ + --nodes 100000 --term-counts 3,10 --repeats 5 + + uv run python tests/bench_query_scoring.py \\ + --graph graphify-out/graph.json \\ + --query "what calls extract" --query "symbol resolution" \\ + --repeats 10 +""" +from __future__ import annotations + +import argparse +import json +import random +import statistics +import sys +import time +from pathlib import Path + +import networkx as nx + +from graphify.serve import ( + _get_trigram_index, + _pick_seeds, + _query_terms, + _score_nodes, + _score_query, + _search_tokens, +) + + +SYLLABLES = [ + "foo", "bar", "baz", "get", "set", "run", "user", "name", "path", + "build", "report", "extract", "router", "config", "service", + "handler", "token", "auth", "rate", "limit", "widget", "model", +] + +QUERIES_BY_TERM_COUNT: dict[int, list[str]] = { + 1: ["foo"], + 2: ["foo", "bar"], + 3: ["router", "service", "handler"], + 5: ["get", "user", "run", "name", "path"], + 10: ["extract", "build", "report", "router", "config", + "service", "token", "rate", "limit", "widget"], +} + + +def _build_random_graph(n: int, *, seed: int) -> nx.DiGraph: + """Reproducible broad-match DiGraph: short constructed labels + edge noise. + + Labels draw from a small syllable pool so tokens collide across nodes, + forcing the trigram prefilter to be selective and exercising score ties + on common tokens. Edge noise provides degree variance so the legacy + tie-break (`max(tied, key=degree)`) is actually exercised against the + new `(-singleton, -degree, label_len, nid)` key tuple.""" + rng = random.Random(seed) + G: nx.DiGraph = nx.DiGraph() + for i in range(n): + label = "_".join(rng.sample(SYLLABLES, rng.randint(1, 3))) + G.add_node(f"n{i}", label=label, source_file=f"src/{label[:8]}.py") + for _ in range(n * 2): + a, b = rng.randrange(n), rng.randrange(n) + if a != b: + G.add_edge(f"n{a}", f"n{b}", relation="calls", confidence="EXTRACTED") + return G + + +def _load_real_graph(path: str) -> nx.Graph: + """Light wrapper that just builds a NetworkX graph from a real + `graphify-out/graph.json`, skipping the size cap and work-memory overlay + that `serve._load_graph` enforces (the bench is read-only).""" + data = json.loads(Path(path).read_text(encoding="utf-8")) + if "links" not in data and "edges" in data: + data = dict(data, links=data["edges"]) + data = {**data, "directed": True} + try: + return nx.readwrite.json_graph.node_link_graph(data, edges="links") + except TypeError: + return nx.readwrite.json_graph.node_link_graph(data) + + +def _legacy_score_and_pick(G, terms): + """Recreates the pre-refactor flow: combined scoring plus one + `_score_nodes([token])` call per distinct query token, with the legacy + tie-break (`max(tied, key=degree)` over a (-score, label_len, nid)-sorted + list) to derive `best_seed_by_term`. Returns the same triple as the + optimized path so they can be compared for equality.""" + ranked = _score_nodes(G, terms) + norm_terms = sorted({tok for t in terms for tok in _search_tokens(t)}) + best_seed_by_term: dict[str, str] = {} + for term in norm_terms: + term_scored = _score_nodes(G, [term]) + if not term_scored: + continue + best_score = term_scored[0][0] + tied = [nid for s, nid in term_scored if s == best_score] + best_nid = max(tied, key=lambda n: G.degree(n)) if len(tied) > 1 else term_scored[0][1] + best_seed_by_term[term] = best_nid + seeds = _pick_seeds(ranked, G=G, best_seed_by_term=best_seed_by_term) + return ranked, best_seed_by_term, seeds + + +def _optimized_score_and_pick(G, terms): + qs = _score_query(G, terms, collect_per_term_seeds=True) + seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) + return qs.ranked, qs.best_seed_by_term, seeds + + +def _warm_caches(G, terms): + """Pre-populate the trigram index and IDF cache so the first timed + iteration doesn't pay the amortized build cost on either path. Both + legacy and optimized share these caches via the graph object, so warming + once is fair to both.""" + _get_trigram_index(G) + # Touch the idf cache for the combined terms and every per-token singleton, + # matching exactly the calls the legacy path will make. + _score_nodes(G, terms) + for term in {tok for t in terms for tok in _search_tokens(t)}: + _score_nodes(G, [term]) + + +def _bench(fn, *, repeats: int) -> list[float]: + # One uncounted warm-up — `_warm_caches` already populated caches, but + # this also amortizes any per-call code-path setup unique to `fn`. + fn() + times: list[float] = [] + for _ in range(repeats): + t0 = time.perf_counter() + fn() + times.append(time.perf_counter() - t0) + return times + + +def _verify_equality(G, terms) -> tuple[int, int]: + leg_rank, leg_best, leg_seeds = _legacy_score_and_pick(G, terms) + opt_rank, opt_best, opt_seeds = _optimized_score_and_pick(G, terms) + assert leg_rank == opt_rank, ( + f"ranked diverged for terms={terms}: legacy[:5]={leg_rank[:5]} opt[:5]={opt_rank[:5]}" + ) + assert leg_best == opt_best, ( + f"best_seed_by_term diverged for terms={terms}: legacy={leg_best} opt={opt_best}" + ) + assert leg_seeds == opt_seeds, ( + f"seeds diverged for terms={terms}: legacy={leg_seeds} opt={opt_seeds}" + ) + return len(leg_rank), len(leg_seeds) + + +def _row(label: str, n_nodes: int, n_terms: int, times: list[float], + traversal_count: int, n_ranked: int, n_seeds: int) -> str: + med = statistics.median(times) * 1000 + mn = min(times) * 1000 + return (f"{label:<10} | n={n_nodes:<7} | terms={n_terms:<3} | " + f"median={med:7.2f}ms | min={mn:7.2f}ms | " + f"passes={traversal_count:<3} | ranked={n_ranked:<6} seeds={n_seeds}") + + +def _legacy_traversal_count(terms) -> int: + # 1 combined pass + one per-token singleton pass. + return 1 + len({tok for t in terms for tok in _search_tokens(t)}) + + +def _run_scenario(G, terms, *, repeats: int) -> tuple[float, float]: + _warm_caches(G, terms) + n_ranked, n_seeds = _verify_equality(G, terms) + + legacy_times = _bench(lambda: _legacy_score_and_pick(G, terms), repeats=repeats) + opt_times = _bench(lambda: _optimized_score_and_pick(G, terms), repeats=repeats) + + n_nodes = G.number_of_nodes() + n_terms = len(set(tok for t in terms for tok in _search_tokens(t))) + print(_row("legacy", n_nodes, n_terms, legacy_times, + _legacy_traversal_count(terms), n_ranked, n_seeds)) + print(_row("optimized", n_nodes, n_terms, opt_times, + 1, n_ranked, n_seeds)) + + med_legacy = statistics.median(legacy_times) + med_opt = statistics.median(opt_times) + speedup = med_legacy / med_opt if med_opt > 0 else float("inf") + print(f"speedup | median: {speedup:.2f}x | " + f"min: {min(legacy_times) / min(opt_times):.2f}x") + return med_legacy, med_opt + + +def _resolve_scenarios(args) -> list[list[str]]: + if args.graph: + # Real-graph mode: each --query is a natural-language sentence, tokenized + # using the same helper the production path uses. + sentences = args.query or ["what calls extract"] + scenarios = [_query_terms(s) for s in sentences] + # Dedupe identical token sets (multiple --query args may tokenize the same). + seen: list[list[str]] = [] + for q in scenarios: + if q not in seen: + seen.append(q) + return seen + term_counts = [int(s.strip()) for s in args.term_counts.split(",") if s.strip()] + scenarios: list[list[str]] = [] + for tc in term_counts: + if tc in QUERIES_BY_TERM_COUNT: + scenarios.append(QUERIES_BY_TERM_COUNT[tc]) + elif tc > 0: + scenarios.append([SYLLABLES[i % len(SYLLABLES)] for i in range(tc)]) + return scenarios + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser( + description="Microbenchmark single-pass query scoring vs legacy per-term rescoring.", + ) + p.add_argument("--nodes", type=int, default=100_000, + help="node count for the synthetic benchmark graph (default: 100000)") + p.add_argument("--seed", type=int, default=20260714, help="RNG seed for the synthetic graph") + p.add_argument("--term-counts", default="3,10", + help="comma-separated list of term counts to benchmark (synthetic mode)") + p.add_argument("--repeats", type=int, default=5, + help="timed iterations per scenario (after one warm-up)") + p.add_argument("--graph", default=None, + help="optional path to a real graphify-out/graph.json; overrides --nodes") + p.add_argument("--query", action="append", default=None, + help="natural-language query sentence (real-graph mode; repeat for multiple)") + args = p.parse_args(argv) + + if args.graph: + print(f"loading real graph: {args.graph} ...", file=sys.stderr) + t0 = time.perf_counter() + G = _load_real_graph(args.graph) + print(f" loaded in {time.perf_counter() - t0:.2f}s", file=sys.stderr) + else: + print(f"building synthetic graph: n={args.nodes} seed={args.seed} ...", + file=sys.stderr) + t0 = time.perf_counter() + G = _build_random_graph(args.nodes, seed=args.seed) + print(f" built in {time.perf_counter() - t0:.2f}s", file=sys.stderr) + + print() + print(f"graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") + scenarios = _resolve_scenarios(args) + print(f"scenarios: {len(scenarios)} | repeats per scenario: {args.repeats}") + print("-" * 110) + + summaries = [] + for terms in scenarios: + print() + med_legacy, med_opt = _run_scenario(G, terms, repeats=args.repeats) + summaries.append((terms, med_legacy, med_opt)) + + print() + print("-" * 110) + print("summary:") + for terms, med_legacy, med_opt in summaries: + speedup = med_legacy / med_opt if med_opt > 0 else float("inf") + print(f" terms={len(set(tok for t in terms for tok in _search_tokens(t))):>3} | " + f"median legacy={med_legacy*1000:>8.2f}ms | " + f"median optimized={med_opt*1000:>8.2f}ms | " + f"speedup={speedup:>5.2f}x") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tests/test_serve.py b/tests/test_serve.py index 3a9656124..855cdc414 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -7,6 +7,7 @@ from graphify.serve import ( _communities_from_graph, _score_nodes, + _score_query, _compute_idf, _EXACT_MATCH_BONUS, _SOURCE_MATCH_BONUS, @@ -26,6 +27,7 @@ _subgraph_to_text, _load_graph, _community_header, + _search_tokens, ) @@ -767,8 +769,8 @@ def test_pick_seeds_respects_max_k(): def test_pick_seeds_without_diversity_args_is_unchanged(): - """G/terms are optional and default to None: existing callers see identical - behavior to before this change.""" + """G/best_seed_by_term are optional and default to None: existing callers + see identical behavior to before this change.""" scored = [(1000.0, "fbs"), (1.0, "err1"), (0.9, "err2")] assert _pick_seeds(scored) == ["fbs"] @@ -777,9 +779,9 @@ def test_pick_seeds_diversity_recovers_starved_term(monkeypatch): """Reproduces #1445: a vague natural-language query where one term's incidental EXACT match on an unrelated node (e.g. a common word also used as an unrelated field/identifier) outscores every SUBSTRING match on the - query's other, actually-relevant terms by ~1000x. Without G/terms, the - 20%-gap cutoff discards the relevant candidate entirely; with them, it is - recovered as a guaranteed per-term seed. + query's other, actually-relevant terms by ~1000x. Without + G/best_seed_by_term, the 20%-gap cutoff discards the relevant candidate + entirely; with them, it is recovered as a guaranteed per-term seed. """ G = nx.DiGraph() # "unrelated" is an exact label match for the query term "unrelated" and @@ -791,13 +793,17 @@ def test_pick_seeds_diversity_recovers_starved_term(monkeypatch): G.add_edge("other", "target") terms = ["unrelated", "widget"] - scored = _score_nodes(G, terms) + # `_score_query` does the combined scoring and the per-term singleton + # winner tracking in one traversal; `_pick_seeds` consumes its + # `best_seed_by_term` to satisfy the per-term guarantee without rescoring. + qs = _score_query(G, terms, collect_per_term_seeds=True) + scored = qs.ranked # Sanity check the premise: without diversity, only the exact match survives. seeds_before = _pick_seeds(scored) assert seeds_before == ["noise"] - seeds_after = _pick_seeds(scored, G=G, terms=terms) + seeds_after = _pick_seeds(scored, G=G, best_seed_by_term=qs.best_seed_by_term) assert "noise" in seeds_after assert "target" in seeds_after @@ -840,8 +846,9 @@ def test_pick_seeds_per_term_guarantee_does_not_reintroduce_generic_dupe(monkeyp G.add_node(f"get{i}", label="GET", source_file=f"r{i}.py") G.add_node("um", label="users_model", source_file="users.py") G.add_edge("um", "get0") - scored = _score_nodes(G, ["get", "users"]) - seeds = _pick_seeds(scored, G=G, terms=["get", "users"]) + terms = ["get", "users"] + qs = _score_query(G, terms, collect_per_term_seeds=True) + seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) get_seeds = [s for s in seeds if s.startswith("get")] assert len(get_seeds) == 1, f"per-term guarantee reintroduced a GET dupe: {seeds}" @@ -1007,3 +1014,229 @@ def test_community_header_sanitizes_name(): out = _community_header(3, "Pay\x00ments\x1b[31m") assert out.startswith("Community 3 — ") assert "\x00" not in out and "\x1b" not in out + + +# --- single-pass scoring refactor: reference-impl equality + one-traversal --- + + +def _reference_best_seed_by_term(G: nx.Graph, terms: list[str]) -> dict[str, str]: + """Test-only oracle for the legacy per-term `_pick_seeds(terms=...)` loop. + + Re-creates what `_pick_seeds` did before the single-pass refactor: rescore + the whole graph per token via `_score_nodes(G, [token])`, take the top- + scoring ties, and break them by `max(tied, key=degree)` (which, over a + list sorted by `(-score, label_len, nid)`, returns the highest-degree node + with ties broken toward the shortest label then the smallest node id). + This is the semantics `_score_query(..., collect_per_term_seeds=True)` now + produces inline during its single traversal. + """ + norm_terms = sorted({tok for t in terms for tok in _search_tokens(t)}) + best: dict[str, str] = {} + for term in norm_terms: + term_scored = _score_nodes(G, [term]) + if not term_scored: + continue + best_score = term_scored[0][0] + tied = [nid for s, nid in term_scored if s == best_score] + best_nid = max(tied, key=lambda n: G.degree(n)) if len(tied) > 1 else term_scored[0][1] + best[term] = best_nid + return best + + +def _make_random_scoring_graph(n: int, *, seed: int) -> nx.DiGraph: + """Reproducible broad-match DiGraph: short constructed labels + edge noise. + + Labels draw from a small syllable pool so tokens collide across nodes, + forcing the trigram prefilter to be selective and exercising score ties + on common tokens. Edge noise provides degree variance so the legacy + tie-break (`max(tied, key=degree)`) is exercised against the new + `(-singleton, -degree, label_len, nid)` key tuple. + """ + import random + + rng = random.Random(seed) + syllables = [ + "foo", "bar", "baz", "get", "set", "run", "user", "name", "path", + "build", "report", "extract", "router", "config", "service", + "handler", "token", "auth", "rate", "limit", "widget", "model", + ] + G: nx.DiGraph = nx.DiGraph() + for i in range(n): + label = "_".join(rng.sample(syllables, rng.randint(1, 3))) + G.add_node(f"n{i}", label=label, source_file=f"src/{label[:8]}.py") + for _ in range(n * 2): + a, b = rng.randrange(n), rng.randrange(n) + if a != b: + G.add_edge(f"n{a}", f"n{b}", relation="calls", confidence="EXTRACTED") + return G + + +SYLLABLE_QUERIES = [ + ["get"], # single token, exact-match + ["get", "user"], # two distinct tokens + ["router", "service", "handler"], # multi-token identifier + ["extract", "build", "report", "path"], # broad term + ["nonexistent"], # no matches + ["nonexistent", "get"], # one missing term + match + ["bar", "bar"], # repeated token (must dedupe) + ["baz", "run", "set", "auth", "rate", "limit"], # many tokens +] + + +@pytest.mark.parametrize("terms", SYLLABLE_QUERIES) +def test_score_query_ranked_matches_score_nodes_byte_identical(terms): + """`_score_query(..., collect_per_term_seeds=False).ranked` is the byte-for- + byte match of `_score_nodes(G, terms)` — guaranteeing path/explain/tests see + no behavior change from the refactor.""" + G = _make_random_scoring_graph(80, seed=7) + assert _score_query(G, terms, collect_per_term_seeds=False).ranked == _score_nodes(G, terms) + + +@pytest.mark.parametrize("terms", SYLLABLE_QUERIES) +def test_score_query_best_seed_by_term_matches_legacy_singleton_scoring(terms): + """Per-token winner the single-pass scorer records matches the legacy + `_score_nodes([token])` + `max(tied, key=degree)` oracle exactly.""" + G = _make_random_scoring_graph(80, seed=7) + ref = _reference_best_seed_by_term(G, terms) + opt = _score_query(G, terms, collect_per_term_seeds=True).best_seed_by_term + assert ref == opt, f"terms={terms}: legacy={ref} optimized={opt}" + + +@pytest.mark.parametrize("terms", SYLLABLE_QUERIES) +def test_pick_seeds_with_optimized_best_seed_matches_legacy_semantics(terms): + """The seeds produced by `_pick_seeds(qs.ranked, G=G, best_seed_by_term= + qs.best_seed_by_term)` exactly match what the legacy `_pick_seeds(terms=...)` + loop would have produced (recreated via the reference oracle).""" + G = _make_random_scoring_graph(80, seed=7) + qs = _score_query(G, terms, collect_per_term_seeds=True) + ref_best = _reference_best_seed_by_term(G, terms) + # Legacy `_pick_seeds(terms=...)` ran `_score_nodes(G, [term])` per token + # to build ref_best, then deduped by label key. The new `_pick_seeds( + # best_seed_by_term=...)` only swaps the source of the per-token winners, + # so it must produce the same seeds given equivalent inputs. + opt_seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) + ref_seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=ref_best) + assert opt_seeds == ref_seeds, f"terms={terms}: ref={ref_seeds} opt={opt_seeds}" + # Per-term guarantee: every legacy winner with a non-empty seed slot is + # accounted for — either it appears in the seed list or another node with + # the same normalized label already claimed the slot (#1766 label dedup). + ref_seed_set = set(ref_seeds) + for term, nid in ref_best.items(): + if nid in ref_seed_set: + continue + nid_label = (G.nodes[nid].get("norm_label") + or G.nodes[nid].get("label") + or nid) + seeded_with_same_label = any( + (G.nodes[s].get("norm_label") or G.nodes[s].get("label") or s) == nid_label + for s in ref_seeds + ) + assert seeded_with_same_label, ( + f"term {term!r} winner {nid!r} dropped without label-dedup reason" + ) + + +def test_score_query_matches_legacy_across_random_deterministic_graphs(): + """Across many deterministic random graphs and many random multi-term + queries, the single-pass scorer's combined ranking, per-token winners, + and resulting seed list all match the legacy semantics. Exercises label + collisions, ties, broad terms, missing terms, and graph size variance.""" + import random + + rng = random.Random(42) + syllables = [ + "foo", "bar", "baz", "get", "set", "run", "user", "name", "path", + "build", "report", "extract", "router", "config", "service", + "handler", "token", "auth", "rate", "limit", "widget", "model", + ] + for trial in range(30): + n = rng.randint(20, 200) + G = _make_random_scoring_graph(n, seed=rng.randint(0, 10**9)) + nq = rng.randint(1, 5) + terms = [rng.choice(syllables) for _ in range(nq)] + ref_best = _reference_best_seed_by_term(G, terms) + opt = _score_query(G, terms, collect_per_term_seeds=True) + # (a) Combined ranking unchanged. + assert opt.ranked == _score_nodes(G, terms), ( + f"trial {trial}: combined ranking diverged for terms={terms}" + ) + # (b) Per-token winners match the legacy per-term rescoring loop. + assert opt.best_seed_by_term == ref_best, ( + f"trial {trial}: best_seed_by_term diverged; ref={ref_best} opt={opt.best_seed_by_term}" + ) + # (c) Final seed list is identical under the legacy semantics. + ref_seeds = _pick_seeds(opt.ranked, G=G, best_seed_by_term=ref_best) + opt_seeds = _pick_seeds(opt.ranked, G=G, best_seed_by_term=opt.best_seed_by_term) + assert opt_seeds == ref_seeds, ( + f"trial {trial}: seeds diverged; ref={ref_seeds} opt={opt_seeds}" + ) + + +def test_score_query_matches_legacy_under_full_scan_fallback(monkeypatch): + """When the trigram prefilter falls back to a full-graph scan, the + single-pass path still produces identical rankings and per-term winners. + + Forces `_trigram_candidates` to return None so the combined iterates the + whole graph — mirroring per-token `_score_nodes([token])` which would also + full-scan when its own trigram search isn't selective.""" + monkeypatch.setattr( + "graphify.serve._trigram_candidates", lambda G, needles: None + ) + terms = ["router", "service", "handler"] + G = _make_random_scoring_graph(80, seed=19) + ref_best = _reference_best_seed_by_term(G, terms) + opt = _score_query(G, terms, collect_per_term_seeds=True) + assert opt.ranked == _score_nodes(G, terms) + assert opt.best_seed_by_term == ref_best + + +def test_query_graph_text_makes_exactly_one_score_query_call(monkeypatch): + """`_query_graph_text` must invoke `_score_query` exactly once per query, + regardless of how many tokens the query has — eliminating the legacy + T+1-pass rescoring. `_score_nodes` must NOT be called from the query path + (only path/explain still call it).""" + G = _make_random_scoring_graph(60, seed=23) + original_sq = _score_query + original_sn = _score_nodes + + state = {"sq": 0, "sn": 0} + + def counting_sq(*a, **k): + state["sq"] += 1 + return original_sq(*a, **k) + + def counting_sn(*a, **k): + state["sn"] += 1 + return original_sn(*a, **k) + + monkeypatch.setattr("graphify.serve._score_query", counting_sq) + monkeypatch.setattr("graphify.serve._score_nodes", counting_sn) + + queries = [ + "foo", # one term + "foo bar", # two + "router service handler", # three (the scenario the RFC targets) + "get user run name path", # five + "extract build report router config service token rate limit widget", # ten + ] + for q in queries: + state["sq"] = 0 + state["sn"] = 0 + _query_graph_text(G, q, mode="bfs", depth=1) + assert state["sq"] == 1, ( + f"expected exactly one _score_query call for {q!r}, got {state['sq']}" + ) + assert state["sn"] == 0, ( + f"query path must not call _score_nodes; got {state['sn']} call(s) for {q!r}" + ) + + +def test_score_query_collect_per_term_seeds_false_omits_tracking(monkeypatch): + """`collect_per_term_seeds=False` returns empty `best_seed_by_term` and + does not pay for per-token best tracking — preserving the cost contract + for path/explain/tests callers that only want the combined ranking.""" + G = _make_random_scoring_graph(50, seed=29) + qs = _score_query(G, ["foo", "bar", "baz"], collect_per_term_seeds=False) + assert qs.best_seed_by_term == {} + # And the combined output is still byte-identical to _score_nodes. + assert qs.ranked == _score_nodes(G, ["foo", "bar", "baz"]) From c0875ccae9409c5b1cc1d09f004205342de723a6 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 15:09:17 +0100 Subject: [PATCH 39/42] test(serve): rebase #1900 test onto #1918 _score_query API; bench newline PR #1918 replaced _pick_seeds(terms=) with best_seed_by_term= from _score_query. Update the #1900 German-stopword seed test to the new single-traversal API and add the missing trailing newline in the (manual, non-collected) query-scoring benchmark. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/bench_query_scoring.py | 2 +- tests/test_serve.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/bench_query_scoring.py b/tests/bench_query_scoring.py index 375c9449b..68e6ce4be 100644 --- a/tests/bench_query_scoring.py +++ b/tests/bench_query_scoring.py @@ -277,4 +277,4 @@ def main(argv: list[str] | None = None) -> int: if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/tests/test_serve.py b/tests/test_serve.py index 855cdc414..302787187 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -393,7 +393,11 @@ def test_pick_seeds_german_query_seeds_content_node_not_heading_noise(): q = "Wie funktioniert die Authentifizierung?" terms = _query_terms(q) - seeds = _pick_seeds(_score_nodes(G, terms), G=G, terms=terms) + # #1918: _score_query does combined scoring + per-term singleton winners in + # one traversal; _pick_seeds consumes best_seed_by_term for the per-term + # guarantee (replaces the old terms= per-term rescoring). + qs = _score_query(G, terms, collect_per_term_seeds=True) + seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) assert "auth" in seeds assert "cfg" not in seeds assert "sec" not in seeds From 19c496dd6303548b2b7bc0aa5693d31992564c47 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 15:14:34 +0100 Subject: [PATCH 40/42] fix(build): make _semantic_id_remap idempotent to stop id accretion (#1917) An id whose canonical stem contains its legacy stem as a prefix (parent dir name == file stem, e.g. .claude/CLAUDE.md) re-matched the legacy branch every build and grew another stem segment, defeating the same_topology/no_change short-circuits and churning graph.json + clustering on every zero-delta update. Skip an id that already carries its canonical stem (mirrors graph_has_legacy_ids); genuine legacy migration still applies. Also records the #1889/#1918 query-scoring perf work in the changelog. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 ++++ graphify/build.py | 10 +++++++++ tests/test_semantic_id_remap_root.py | 33 ++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0cd98d3b..bd6306ee1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.17 (unreleased) +- Fix: `_semantic_id_remap` is now idempotent, so incremental rebuilds stop churning (#1917). When a file's canonical stem contained its own legacy stem as a prefix (parent dir name equals the file stem, e.g. `.claude/CLAUDE.md`, `docs/docs.md`), an already-migrated semantic node id re-matched the legacy branch and gained another stem segment on every build (`claude_x` -> `claude_claude_x` -> ...). Because `_origin` is persisted, every `graphify update` re-fed nodes through the remap, so the ids grew unboundedly and the `same_topology`/`same_graph`/`no_change` short-circuits never fired — rewriting `graph.json` and re-running clustering on every zero-delta update. The remap now skips an id that already carries its canonical stem (mirroring the `graph_has_legacy_ids` check), while a genuine one-time legacy migration still applies. (An already-corrupted graph needs one `graphify extract --force` to reset the grown ids.) +- Perf: `graphify query` now scores the graph once per query instead of T+1 times for a T-term query (#1889 / #1918, thanks @Sirhan1). The per-term-guarantee (#1445) previously re-scored the whole graph once per token; `_score_query` now computes the combined ranking and each token's singleton winner in a single traversal, feeding `_pick_seeds` via `best_seed_by_term`. Behavior is preserved (verified byte-identical against the old per-term scoring across a differential fuzz); ~1.3-1.4x faster and independent of query length. + + - Fix: `--mode deep` is now effective over a warm cache instead of a silent no-op (#1894). The semantic cache is namespaced by mode (`semantic` vs `semantic-deep`) so a shallow-cached file no longer satisfies a deep run; `graphify extract` gains `--force` (and honors `GRAPHIFY_FORCE`) to bypass the incremental gate and the cache read; and a deep incremental run widens its dispatch to the full live doc set so the deep namespace actually gets populated. Cache prune/clear now sweep both namespaces so the deep cache can't accumulate orphans. (The skill-side flow that passes the mode through is a follow-up; the CLI is complete and backward compatible — the new `mode` argument defaults to the existing behavior.) - Fix: the semantic cache no longer persists dangling edges/hyperedges (#1916). When a node group was skipped on write (out-of-scope per the batch guard, or a ghost `source_file`), edges/hyperedges in the kept groups that referenced those never-written nodes were saved anyway and re-surfaced on every cache replay. Those references are now pruned at write time (gated on the scoping allowlist, so unscoped callers are unchanged), and `build_from_json` validates hyperedge members against the node set so a dangling hyperedge can't reach `graph.json` even from a live extraction. - Fix: `graphify update`/watch no longer produces a bloated graph by double-representing documents (#1915). `_rebuild_code` AST-quick-scanned Markdown/doc files and then preserved their existing semantic (LLM) nodes on top, so each doc appeared twice (a real corpus came out ~4x). A doc that already has semantic nodes in the graph is no longer AST-quick-scanned (its semantic nodes are the sole representation), while a doc with no semantic layer still gets the structural quick-scan; incremental rebuilds now preserve a doc's semantic nodes instead of evicting them, and previously-bloated graphs self-heal on the next full rebuild. diff --git a/graphify/build.py b/graphify/build.py index b05c83978..caeaefdf9 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -287,6 +287,16 @@ def _semantic_id_remap(nodes: list, root: str | None) -> dict: if not new_stem: continue norm_nid = _normalize_id(nid) + # Idempotency guard (#1917): an id already carrying its canonical stem is + # done — do not re-run the legacy branch on it. When the canonical stem + # contains a shorter legacy stem as a prefix (parent dir name == file + # stem, e.g. `.claude/CLAUDE.md` -> `claude_claude` over legacy `claude`), + # an already-migrated id like `claude_claude_x` still matches the legacy + # `claude_` prefix below and would gain another stem segment on every + # build, defeating the same_topology/no_change short-circuits. Mirrors the + # canonical check in graph_has_legacy_ids. + if norm_nid == new_stem or norm_nid.startswith(new_stem + "_"): + continue new_id: str | None = None for old_stem in _old_file_stems(rel): if old_stem == new_stem: diff --git a/tests/test_semantic_id_remap_root.py b/tests/test_semantic_id_remap_root.py index ab85bd6d7..3a5fee626 100644 --- a/tests/test_semantic_id_remap_root.py +++ b/tests/test_semantic_id_remap_root.py @@ -48,3 +48,36 @@ def test_normal_semantic_remap_still_works(): remap = _semantic_id_remap( [{"id": "foo", "source_file": "src/foo.py", "_origin": "semantic"}], "/proj") assert isinstance(remap, dict) + + +# --- #1917: _semantic_id_remap must be idempotent (no id accretion) --- + +def test_semantic_id_remap_is_idempotent_when_stem_contains_legacy_stem(): + """A file whose parent dir name equals its stem (.claude/CLAUDE.md -> + canonical `claude_claude`, legacy `claude`) must not re-prefix an + already-canonical id on every build (#1917). Without the guard, ids grow + `claude_x` -> `claude_claude_x` -> `claude_claude_claude_x` ..., defeating + the same_topology/no_change short-circuits.""" + nodes = [{"id": "claude_graphify_trigger", + "source_file": ".claude/CLAUDE.md", "_origin": "semantic"}] + first = _semantic_id_remap(nodes, ".") + assert first == {"claude_graphify_trigger": "claude_claude_graphify_trigger"} + # Feed the migrated ids back through: a second pass must be a fixed point. + migrated = [{**n, "id": first.get(n["id"], n["id"])} for n in nodes] + assert _semantic_id_remap(migrated, ".") == {}, "id re-prefixed on second build (#1917)" + + +def test_semantic_id_remap_bare_file_node_is_idempotent(): + """The bare file node id follows the same fixed-point rule.""" + nodes = [{"id": "claude", "source_file": ".claude/CLAUDE.md", "_origin": "semantic"}] + first = _semantic_id_remap(nodes, ".") + assert first == {"claude": "claude_claude"} + migrated = [{"id": "claude_claude", "source_file": ".claude/CLAUDE.md", "_origin": "semantic"}] + assert _semantic_id_remap(migrated, ".") == {} + + +def test_semantic_id_remap_still_migrates_genuine_legacy_id(): + """The idempotency guard must not block a real one-time legacy migration: + a pre-scheme id under a normal path still remaps once to the canonical stem.""" + nodes = [{"id": "readme_booking", "source_file": "api/README.md", "_origin": "semantic"}] + assert _semantic_id_remap(nodes, ".") == {"readme_booking": "api_readme_booking"} From cb96bdaa0c367bec8d5c5aee5d7c9ebb727e9780 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Wed, 15 Jul 2026 19:51:53 +0100 Subject: [PATCH 41/42] fix: preserve semantic layer, stamp hyperedges, PHP namespaces, ignore diagnostic (#1925 #1920 #1923 #1922) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - #1925: a missing manifest.json no longer degrades `extract --code-only` into a full scan that discards the committed semantic layer. An existing graph.json is a sufficient incremental baseline (detect_incremental treats an absent manifest as "all new / none deleted"), so out-of-scope doc/paper/ image nodes are preserved while genuinely deleted sources still evict. - #1920: _stamped_manifest_files now counts hyperedge output, so a doc whose only chunk output is a hyperedge is stamped instead of re-extracted forever. - #1923: new namespace/use-aware PHP resolver (mirrors the Java resolver, runs before the unique-name rewire) so App\Models\Page and an imported Filament\Pages\Page stay distinct — no more false inherits/imports edge. - #1922: detect() records ignored files/dirs in a new `ignored` diagnostic field (the nested-ignore scoping bug itself shipped in 0.9.16 / #1873). Regression tests added for each; full suite 3325 passed, 3 skipped. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 + graphify/cli.py | 27 ++- graphify/detect.py | 21 ++- graphify/extract.py | 17 ++ graphify/extractors/resolution.py | 264 ++++++++++++++++++++++++++++++ tests/test_detect.py | 27 +++ tests/test_extract_cli.py | 135 +++++++++++++++ tests/test_php_type_resolution.py | 158 ++++++++++++++++++ 8 files changed, 645 insertions(+), 8 deletions(-) create mode 100644 tests/test_php_type_resolution.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bd6306ee1..84d865a6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.17 (unreleased) +- Fix: a missing `manifest.json` no longer degrades `graphify extract --code-only` into a full scan that discards the committed semantic layer (#1925). On a fresh clone (or when the manifest is deliberately untracked because its mtimes churn), the incremental gate required both `manifest.json` and `graph.json`; with only the graph present it fell to a full scan, and under `--code-only` that dropped every doc/paper/image node — silently replacing a curated graph with an AST-only skeleton. An existing `graph.json` is now a sufficient incremental baseline: `detect_incremental` already treats an absent manifest as "everything new / nothing deleted", so `build_merge` + `_stale_graph_sources` preserve files that are merely out of this run's scope while still evicting genuinely deleted sources. +- Fix: hyperedge-only documents are now stamped in the manifest instead of being re-extracted on every run (#1920). `_stamped_manifest_files` (#1897) decided a semantic doc "produced output" by inspecting only `nodes` and `edges`, never `hyperedges`, so a chunk whose only output for a doc was a hyperedge (3+ nodes sharing a concept) left that doc unstamped and perpetually re-queued. Stamping now counts hyperedge output too, mirroring the per-`source_file` keying the semantic cache already uses. +- Fix: the PHP extractor now disambiguates same-named classes across namespaces (#1923). A bare class reference (`extends Page`) collapsed onto the only internal class named `Page`, so `App\Models\Page` and an imported `Filament\Pages\Page` fused into one node, manufacturing a false `inherits`/`imports` edge and a bogus cross-community bridge. A new namespace/`use`-aware resolution pass (mirroring the Java resolver, running before the unique-name rewire) re-points supertype/import references to the real definition, or parks provably-external ones on a fully-qualified stub the bare-name rewire cannot collapse. Plain, non-namespaced PHP is unchanged. +- Fix: `detect()` now records files and directories dropped by a `.gitignore`/`.graphifyignore` rule in a new `ignored` diagnostic field (#1922). The nested-ignore scoping bug itself was fixed in 0.9.16 (#1873); this closes the remaining gap where an ignored path left no trace in any diagnostic, so an over-broad rule looked like a clean scan. Entries are per-directory where a subtree is pruned, keeping the list bounded. - Fix: `_semantic_id_remap` is now idempotent, so incremental rebuilds stop churning (#1917). When a file's canonical stem contained its own legacy stem as a prefix (parent dir name equals the file stem, e.g. `.claude/CLAUDE.md`, `docs/docs.md`), an already-migrated semantic node id re-matched the legacy branch and gained another stem segment on every build (`claude_x` -> `claude_claude_x` -> ...). Because `_origin` is persisted, every `graphify update` re-fed nodes through the remap, so the ids grew unboundedly and the `same_topology`/`same_graph`/`no_change` short-circuits never fired — rewriting `graph.json` and re-running clustering on every zero-delta update. The remap now skips an id that already carries its canonical stem (mirroring the `graph_has_legacy_ids` check), while a genuine one-time legacy migration still applies. (An already-corrupted graph needs one `graphify extract --force` to reset the grown ids.) - Perf: `graphify query` now scores the graph once per query instead of T+1 times for a T-term query (#1889 / #1918, thanks @Sirhan1). The per-term-guarantee (#1445) previously re-scored the whole graph once per token; `_score_query` now computes the combined ranking and each token's singleton winner in a single traversal, feeding `_pick_seeds` via `best_seed_by_term`. Behavior is preserved (verified byte-identical against the old per-term scoring across a differential fuzz); ~1.3-1.4x faster and independent of query length. diff --git a/graphify/cli.py b/graphify/cli.py index f2ae1da1b..d817401ac 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -65,11 +65,18 @@ def _stamped_manifest_files( empty so detect_incremental re-queues them (#933). Both sides of the membership test are resolved against the scan ``root`` - before comparing (#1897): node/edge ``source_file`` values are + before comparing (#1897): node/edge/hyperedge ``source_file`` values are root-relative on a fresh extraction while ``files_by_type`` entries are absolute (from detect()), so a raw string comparison never matched and every freshly-extracted semantic doc was dropped from the manifest. Mirrors the #1890 path normalization in graphify.llm. + + Hyperedges are counted as output (#1920): a chunk whose only result for a + document is a hyperedge (3+ nodes sharing a concept) is valid output that + the semantic cache persists per-``source_file`` — omitting it here left the + doc unstamped, so detect_incremental re-queued it on every run. The stamping + condition mirrors the cache-write keying (a hyperedge carries its own + ``source_file``); do not derive it from member nodes. """ root = Path(root) @@ -83,7 +90,7 @@ def _resolve(value: str) -> Path: return p sem_extracted: set[Path] = set() - for coll in ("nodes", "edges"): + for coll in ("nodes", "edges", "hyperedges"): for item in sem_result.get(coll, []): sf = item.get("source_file", "") if sf: @@ -2281,12 +2288,26 @@ def _parse_float(name: str, raw: str) -> float: ) manifest_path = graphify_out / "manifest.json" existing_graph_path = graphify_out / "graph.json" - incremental_mode = manifest_path.exists() and existing_graph_path.exists() if has_path else False + # #1925: a missing manifest.json must not degrade to a full scan that + # discards the existing graph's semantic layer. An existing graph.json + # is a sufficient incremental baseline: detect_incremental treats an + # absent manifest as "everything is new" (re-extract all, nothing + # deleted), and build_merge + _stale_graph_sources reconcile replaced + # and genuinely-deleted sources against the current corpus, so doc/ + # paper/image nodes survive a --code-only rebuild instead of being + # dropped with the rest of the committed graph. + incremental_mode = existing_graph_path.exists() if has_path else False # --force: full scan, not the manifest-gated incremental diff — a warm # unchanged tree would otherwise dispatch zero files (#1894). incremental_mode = incremental_mode and not force if force: print("[graphify extract] --force: full re-scan, semantic cache reads skipped") + elif incremental_mode and not manifest_path.exists(): + print( + "[graphify extract] manifest.json missing; using existing " + "graph.json as the incremental baseline (all files re-checked; " + "nodes for files outside this run's scope are preserved)" + ) if not has_path: code_files = [] diff --git a/graphify/detect.py b/graphify/detect.py index 1c1c27c3e..ce1aad856 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -1151,6 +1151,11 @@ def _wc(path: Path) -> int: skipped_sensitive: list[str] = [] unclassified: list[str] = [] + # Files/dirs dropped by a .gitignore/.graphifyignore rule. Recorded so an + # over-broad ignore (or a legitimately-ignored subtree) is visible instead + # of silently vanishing from the graph (#1922). Directory-level entries keep + # this bounded — a pruned `data/` is one entry, not one per contained file. + ignored: list[str] = [] ignore_patterns = _load_graphifyignore(root) ignore_cache: dict[Path, bool] = {} # shared across all _is_ignored calls in this scan # CLI --exclude patterns are anchored at the scan root and appended last @@ -1222,11 +1227,15 @@ def _on_walk_error(err: OSError) -> None: # any `!` rule existed — e.g. a single `!docs/**` made the walk descend # bin/, obj/, wwwroot/, generated/, … : a pathological slowdown on large # repos for no correctness gain. - dirnames[:] = [ - d for d in dirnames - if not _is_noise_dir(d, dp) - and not _is_ignored(dp / d, root, ignore_patterns, _cache=ignore_cache) - ] + kept_dirs: list[str] = [] + for d in dirnames: + if _is_noise_dir(d, dp): + continue + if _is_ignored(dp / d, root, ignore_patterns, _cache=ignore_cache): + ignored.append(str(dp / d) + os.sep) + continue + kept_dirs.append(d) + dirnames[:] = kept_dirs if follow_symlinks: safe_dirs: list[str] = [] for d in dirnames: @@ -1256,6 +1265,7 @@ def _on_walk_error(err: OSError) -> None: if str(p).startswith(str(converted_dir)): continue if not in_memory and _is_ignored(p, root, ignore_patterns, _cache=ignore_cache): + ignored.append(str(p)) continue if not _resolves_under_root(p, root): skipped_sensitive.append(str(p) + " [symlink target outside scan root]") @@ -1338,6 +1348,7 @@ def _on_walk_error(err: OSError) -> None: "skipped_sensitive": skipped_sensitive, "unclassified": sorted(unclassified), "walk_errors": walk_errors, + "ignored": sorted(ignored), "graphifyignore_patterns": len(ignore_patterns), "scan_root": str(root.resolve()), } diff --git a/graphify/extract.py b/graphify/extract.py index 12a4f108e..2ff5411cf 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -113,6 +113,7 @@ _resolve_cross_file_java_imports, _resolve_export_target, _resolve_java_type_references, + _resolve_php_type_references, _resolve_js_import_path, _resolve_js_import_target, _resolve_js_module_path, @@ -4581,6 +4582,22 @@ def extract( _merge_swift_extensions(per_file, all_nodes, all_edges) _disambiguate_colliding_node_ids(all_nodes, all_edges, all_raw_calls, root) _canonicalize_csharp_namespace_nodes(all_nodes, all_edges) + # PHP namespace/use disambiguation must run BEFORE the unique-stub rewire: + # the false merge (#1923) happens inside the rewire when a bare-name stub + # matches a unique internal class from a different namespace. + _php_exts = {".php", ".phtml", ".php3", ".php4", ".php5", ".php7", ".phps"} + _php_sel = [ + (r, p) for r, p in zip(per_file, paths) + if p.suffix.lower() in _php_exts and not p.name.lower().endswith(".blade.php") + ] + if _php_sel: + try: + _resolve_php_type_references( + [r for r, _ in _php_sel], [p for _, p in _php_sel], all_nodes, all_edges + ) + except Exception as exc: + import logging + logging.getLogger(__name__).warning("PHP type-reference resolution failed, skipping: %s", exc) _rewire_unique_stub_nodes(all_nodes, all_edges) # Add cross-file class-level edges (Python only - uses Python parser internally) diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index 9736cccef..a88abf2da 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -2213,6 +2213,270 @@ def walk(n) -> None: if node.get("id") not in repointed_from or node.get("id") in still_referenced ] + +_PHP_SUPERTYPE_RELATIONS = ("inherits", "implements", "mixes_in") +_PHP_REPOINT_RELATIONS = frozenset({"inherits", "implements", "mixes_in", "imports", "references"}) + + +def _php_fqn_from_raw(raw: str, ns: str, uses: dict[str, str]) -> str: + """Resolve a raw (possibly qualified) PHP class reference to an FQN. + + PHP name-resolution for class names: + \\A\\B -> absolute: A\\B + A\\B -> first segment through the `use` map (group-prefix semantics), + else relative to the current namespace + B -> `use` map, else current namespace (class names do NOT fall + back to the global namespace) + """ + raw = raw.strip() + if raw.startswith("\\"): + return raw[1:] + if "\\" in raw: + first, rest = raw.split("\\", 1) + mapped = uses.get(first.lower()) + if mapped: + return f"{mapped}\\{rest}" + return f"{ns}\\{raw}" if ns else raw + mapped = uses.get(raw.lower()) + if mapped: + return mapped + return f"{ns}\\{raw}" if ns else raw + + +def _resolve_php_type_references( + per_file: list[dict], + paths: list[Path], + all_nodes: list[dict], + all_edges: list[dict], +) -> None: + """Disambiguate PHP inherits/implements/mixes_in/imports/references targets + using each file's ``namespace`` declaration and ``use`` imports (#1923). + + Mirrors ``_resolve_java_type_references`` (a re-parse pass), but MUST run + BEFORE ``_rewire_unique_stub_nodes``: the false edge is manufactured by the + rewire itself — a bare ``Page`` stub collapses onto the only internal class + labeled ``Page`` even though the referencing file ``use``d a different + namespace (``Filament\\Pages\\Page`` vs ``App\\Models\\Page``). References + proven external by a ``use`` FQN or a qualified name are re-pointed to an + FQN-labeled sourceless stub, which the bare-label rewire cannot collapse. + References with no namespace facts are left untouched so the unique-label + rewire keeps handling plain (non-namespaced) PHP as before. + """ + try: + import tree_sitter_php as tsphp + from tree_sitter import Language, Parser + except ImportError: + return + + lang_fn = getattr(tsphp, "language_php", None) or getattr(tsphp, "language", None) + if lang_fn is None: + return + language = Language(lang_fn()) + parser = Parser(language) + + ns_by_file: dict[str, str] = {} + uses_by_file: dict[str, dict[str, str]] = {} # lower alias -> FQN + raw_by_file: dict[str, dict[tuple[str, str], str | None]] = {} # (relation, lower bare) -> raw | None(ambiguous) + + for path, result in zip(paths, per_file): + srcs = {n.get("source_file") for n in result.get("nodes", []) if n.get("source_file")} + if not srcs: + continue + try: + source = path.read_bytes() + tree = parser.parse(source) + except Exception: + continue + + namespaces: list[str] = [] + uses: dict[str, str] = {} + raws: dict[tuple[str, str], str | None] = {} + + def _record_raw(relation: str, raw: str) -> None: + bare = raw.rsplit("\\", 1)[-1].strip().lower() + if not bare: + return + key = (relation, bare) + if key in raws and raws[key] != raw: + raws[key] = None # e.g. `implements A\I, B\I` — never guess + else: + raws.setdefault(key, raw) + + def _record_use_clause(clause, prefix: str) -> None: + target = None + alias = None + saw_as = False + for c in clause.children: + if c.type in ("function", "const"): + return # not a class import + if c.type == "as": + saw_as = True + elif c.type in ("qualified_name", "name"): + if saw_as: + alias = _read_text(c, source) + elif target is None: + target = _read_text(c, source) + if not target: + return + fqn = (f"{prefix}\\{target}" if prefix else target).lstrip("\\") + key = (alias or fqn.rsplit("\\", 1)[-1]).strip().lower() + if key: + uses.setdefault(key, fqn) + + def walk(n) -> None: + t = n.type + if t == "namespace_definition": + for c in n.children: + if c.type == "namespace_name": + namespaces.append(_read_text(c, source)) + break + elif t == "namespace_use_declaration": + prefix = "" + group = None + for c in n.children: + if c.type == "namespace_name": + prefix = _read_text(c, source) # group-use prefix + elif c.type == "namespace_use_group": + group = c + elif c.type == "namespace_use_clause": + _record_use_clause(c, "") + if group is not None: + for c in group.children: + if c.type == "namespace_use_clause": + _record_use_clause(c, prefix) + return + elif t == "class_declaration": + for child in n.children: + if child.type == "base_clause": + for sub in child.children: + if sub.type in ("name", "qualified_name"): + _record_raw("inherits", _read_text(sub, source)) + elif child.type == "class_interface_clause": + for sub in child.children: + if sub.type in ("name", "qualified_name"): + _record_raw("implements", _read_text(sub, source)) + elif child.type == "declaration_list": + for member in child.children: + if member.type != "use_declaration": + continue + for sub in member.children: + if sub.type in ("name", "qualified_name"): + _record_raw("mixes_in", _read_text(sub, source)) + for child in n.children: + walk(child) + + walk(tree.root_node) + if len(set(namespaces)) > 1: + continue # multi-namespace file (PSR-1 violation): keep legacy behavior + ns = namespaces[0] if namespaces else "" + for s in srcs: + ns_by_file[s] = ns + uses_by_file[s] = uses + raw_by_file[s] = raws + + if not ns_by_file: + return + + # lower FQN -> definition node id (PHP class names are case-insensitive). + fqn_to_id: dict[str, str] = {} + for node in all_nodes: + label = node.get("label", "") + src = node.get("source_file", "") + nid = node.get("id", "") + if not (label and src and nid) or src not in ns_by_file: + continue + if label.endswith(")") or "." in label: # methods / file nodes + continue + ns = ns_by_file[src] + fqn = f"{ns}\\{label}" if ns else label + fqn_to_id.setdefault(fqn.lower(), nid) + + node_ids = {n.get("id") for n in all_nodes if n.get("id")} + stub_label: dict[str, str] = { + n["id"]: n.get("label", "") + for n in all_nodes + if n.get("id") and not n.get("source_file") and n.get("label") + } + + external_stub_ids: dict[str, str] = {} + new_nodes: list[dict] = [] + + def _external_stub(fqn: str) -> str: + key = fqn.lower() + nid = external_stub_ids.get(key) + if nid: + return nid + nid = _make_id(fqn) + if nid not in node_ids: + new_nodes.append({ + "id": nid, + "label": fqn, + "file_type": "code", + "source_file": "", + "source_location": "", + }) + node_ids.add(nid) + external_stub_ids[key] = nid + return nid + + repointed_from: set[str] = set() + for edge in all_edges: + relation = edge.get("relation") + if relation not in _PHP_REPOINT_RELATIONS: + continue + ref_file = edge.get("source_file", "") + if ref_file not in ns_by_file: + continue + tgt = edge.get("target") + label = stub_label.get(tgt) + if not label: + continue + bare = label.strip().lower() + ns = ns_by_file[ref_file] + uses = uses_by_file.get(ref_file, {}) + + raw = None + if relation in _PHP_SUPERTYPE_RELATIONS: + raw = raw_by_file.get(ref_file, {}).get((relation, bare)) + + explicit = False + if raw and "\\" in raw: + fqn = _php_fqn_from_raw(raw, ns, uses) + explicit = True + elif bare in uses: + fqn = uses[bare] + explicit = True + elif ns: + fqn = f"{ns}\\{label}" + else: + continue # no namespace facts: legacy unique-label rewire applies + + resolved = fqn_to_id.get(fqn.lower()) + if resolved and resolved != tgt: + edge["target"] = resolved + repointed_from.add(tgt) + elif explicit and resolved is None: + # Proven external: park the edge on an FQN-labeled stub the + # bare-name rewire cannot collapse (this is the #1923 fix). + edge["target"] = _external_stub(fqn) + repointed_from.add(tgt) + # non-explicit miss: leave the bare stub for the legacy rewire + + if new_nodes: + all_nodes.extend(new_nodes) + if not repointed_from: + return + + still_referenced: set[str] = set() + for edge in all_edges: + still_referenced.add(edge.get("source")) + still_referenced.add(edge.get("target")) + all_nodes[:] = [ + n for n in all_nodes + if n.get("id") not in repointed_from or n.get("id") in still_referenced + ] + + _pascal_unit_cache: dict[str, dict[str, str]] = {} _pascal_class_stem_cache: dict[str, dict[str, str]] = {} # root_key → {stem_lower: _file_stem} diff --git a/tests/test_detect.py b/tests/test_detect.py index a783eaf6a..498a9aab8 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -1853,6 +1853,33 @@ def test_nested_gitignore_patterns_still_apply_inside_their_dir(tmp_path): assert result["total_files"] == 2 # main.py + sub/keep.py; sub/noise.log ignored +def test_nested_gitignore_does_not_govern_sibling_project(tmp_path): + """A nested .gitignore ('data/') in one project must not drop a sibling + project's data/ files, and the drop must be recorded in the `ignored` + diagnostic field rather than silently vanishing (#1922).""" + (tmp_path / "run.py").write_text("x = 1") + pa = tmp_path / "project_a" / "data" + pa.mkdir(parents=True) + (pa / "loader.py").write_text("def load(): pass") + pb = tmp_path / "project_b" + (pb / "data").mkdir(parents=True) + (pb / ".gitignore").write_text("data/\n") + (pb / "data" / "dump.csv").write_text("a,b\n1,2\n") + + result = detect(tmp_path) + + all_paths = [f for v in result["files"].values() for f in v] + assert any( + f.endswith(os.path.join("project_a", "data", "loader.py")) for f in all_paths + ), "sibling project_a/data/loader.py must survive project_b's nested ignore" + assert not any(f.endswith("dump.csv") for f in all_paths) + # The legitimately-ignored subtree is recorded, not silently dropped. + assert any( + e.rstrip(os.sep).endswith(os.path.join("project_b", "data")) + for e in result["ignored"] + ), f"ignored subtree should be recorded in detect()['ignored']: {result['ignored']}" + + # --------------------------------------------------------------------------- # #1908: manifest must not retain scan-excluded files as permanent # "deleted" entries. Full-scan saves prune excluded-but-alive rows; subset diff --git a/tests/test_extract_cli.py b/tests/test_extract_cli.py index 6ef1b2c22..87a65e45e 100644 --- a/tests/test_extract_cli.py +++ b/tests/test_extract_cli.py @@ -222,6 +222,74 @@ def test_stamped_manifest_files_normalizes_both_sides(tmp_path): assert out["document"] == [str(fresh_doc), str(cached_doc)] +def test_stamped_manifest_files_counts_hyperedge_only_docs(tmp_path): + """#1920: a doc whose only chunk output is a hyperedge (3+ nodes sharing a + concept) is valid output — the semantic cache persists it per source_file — + so it must be stamped. Before the fix the stamping loop only inspected + ``nodes``/``edges``, leaving such a doc unstamped and re-queued forever.""" + from graphify.cli import _stamped_manifest_files + + hyper_doc = tmp_path / "hyper.md"; hyper_doc.write_text("# hyper") + omitted_doc = tmp_path / "omitted.md"; omitted_doc.write_text("# omitted") + + files_by_type = {"document": [str(hyper_doc), str(omitted_doc)]} + sem_result = { + "nodes": [], + "edges": [], + "hyperedges": [ + {"id": "h1", "label": "L", "nodes": ["a", "b", "c"], + "relation": "participate_in", "source_file": "hyper.md"}, + ], + } + + out = _stamped_manifest_files(files_by_type, sem_result, tmp_path) + assert str(hyper_doc) in out["document"], ( + "a hyperedge-only doc must be stamped (#1920)" + ) + # A doc with no output at all still stays unstamped (#933). + assert str(omitted_doc) not in out["document"] + + +def test_manifest_stamps_hyperedge_only_docs(monkeypatch, tmp_path): + """#1920 end-to-end: a fresh extraction whose only output for a doc is a + hyperedge stamps that doc's semantic_hash, so it is not re-dispatched.""" + import json + + corpus = _make_corpus(tmp_path) # main.go + README.md + out_dir = tmp_path / "out" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + + def _hyperedge_only(paths, **kwargs): + on_chunk = kwargs.get("on_chunk_done") + if on_chunk: + on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) + return { + "nodes": [], + "edges": [], + "hyperedges": [{"id": "h1", "label": "Shared", "nodes": ["a", "b", "c"], + "relation": "participate_in", "source_file": "README.md"}], + "input_tokens": 10, + "output_tokens": 5, + } + + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _hyperedge_only) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, "argv", + ["graphify", "extract", str(corpus), "--backend", "claude", + "--no-cluster", "--out", str(out_dir)], + ) + try: + mainmod.main() + except SystemExit as exc: + assert exc.code in (None, 0), f"unexpected exit code {exc.code}" + + manifest = json.loads((out_dir / "graphify-out" / "manifest.json").read_text()) + assert manifest.get("README.md", {}).get("semantic_hash"), ( + f"hyperedge-only doc must be stamped (#1920): {sorted(manifest)}" + ) + + # --- #1894: --force and deep-mode dispatch over a warm cache ----------------- def _recording_extractor(calls): @@ -429,6 +497,73 @@ def test_extract_codeonly_succeeds_without_api_key(monkeypatch, tmp_path): assert len(json.loads(graph.read_text()).get("nodes", [])) > 0 +def test_missing_manifest_code_only_preserves_semantic_layer(monkeypatch, tmp_path): + """#1925: `graphify extract --code-only` with a MISSING manifest.json must + not degrade to a full scan that discards the committed semantic layer. An + existing graph.json is a sufficient incremental baseline, so doc/paper/image + nodes (excluded by --code-only, not deleted) are preserved; a genuinely + deleted source is still evicted (#1909 semantics retained).""" + import json + + corpus = tmp_path / "proj"; corpus.mkdir() + (corpus / "keep.py").write_text("def keep():\n return 1\n") + (corpus / "README.md").write_text("# Notes\nCurated docs.\n") + out_dir = tmp_path / "out" + graphify_out = out_dir / "graphify-out" + _clear_backend_keys(monkeypatch) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + def _sem_doc_count(g): + return sum(1 for n in g["nodes"] if n.get("source_file") == "README.md") + + # 1) seed a code-only graph + _run_extract(monkeypatch, ["graphify", "extract", str(corpus), + "--code-only", "--out", str(out_dir)]) + graph_path = graphify_out / "graph.json" + graph = json.loads(graph_path.read_text()) + + # 2) inject a committed semantic layer for README.md (nodes + edge + hyperedge) + graph["nodes"].append({"id": "doc_readme_a", "label": "Concept A", + "source_file": "README.md", "file_type": "document"}) + graph["nodes"].append({"id": "doc_readme_b", "label": "Concept B", + "source_file": "README.md", "file_type": "document"}) + graph.setdefault("edges", []).append( + {"source": "doc_readme_a", "target": "doc_readme_b", + "relation": "relates_to", "source_file": "README.md"}) + graph.setdefault("hyperedges", []).append( + {"id": "h1", "label": "Shared", "nodes": ["doc_readme_a", "doc_readme_b"], + "relation": "participate_in", "source_file": "README.md"}) + graph_path.write_text(json.dumps(graph)) + (graphify_out / ".graphify_semantic_marker").write_text( + json.dumps({"output_tokens": 1})) + + # 3) manifest goes missing (fresh clone / deliberately untracked) + (graphify_out / "manifest.json").unlink() + + # 4) re-run the SAME code-only extract + _run_extract(monkeypatch, ["graphify", "extract", str(corpus), + "--code-only", "--out", str(out_dir)]) + after = json.loads(graph_path.read_text()) + assert _sem_doc_count(after) >= 2, ( + "committed semantic doc nodes must survive a missing-manifest " + f"--code-only rebuild (#1925); got {_sem_doc_count(after)}" + ) + assert any(h.get("id") == "h1" for h in after.get("hyperedges", [])), ( + "committed hyperedge must survive the rebuild" + ) + assert any("keep" in n["id"] for n in after["nodes"]), "code nodes intact" + + # 5) a genuine deletion still evicts the doc's semantic nodes + (corpus / "README.md").unlink() + (graphify_out / "manifest.json").unlink(missing_ok=True) + _run_extract(monkeypatch, ["graphify", "extract", str(corpus), + "--code-only", "--out", str(out_dir)]) + gone = json.loads(graph_path.read_text()) + assert _sem_doc_count(gone) == 0, ( + "a genuinely deleted doc must still be evicted (#1909 semantics preserved)" + ) + + def test_extract_out_keeps_project_root_clean(monkeypatch, tmp_path): """`extract --out DIR` routes every artifact to DIR/graphify-out/ and the scanned project must not grow a graphify-out/ (or anything else) beside diff --git a/tests/test_php_type_resolution.py b/tests/test_php_type_resolution.py new file mode 100644 index 000000000..0dff626d3 --- /dev/null +++ b/tests/test_php_type_resolution.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +from pathlib import Path + +from graphify.extract import extract + + +def _write(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _node_by_id(result: dict, nid: str) -> dict | None: + return next((n for n in result["nodes"] if n.get("id") == nid), None) + + +def _class_defs(result: dict, label: str) -> list[dict]: + return [ + n for n in result["nodes"] + if n.get("label") == label and n.get("source_file") + ] + + +def test_php_external_namespaced_base_does_not_collapse_onto_internal_class(tmp_path: Path): + # #1923: `App\Models\Page` (internal) and `Filament\Pages\Page` (external, + # via `use`) share the simple name `Page`. The bare-name rewire must NOT + # collapse the external supertype reference onto the only internal `Page`. + model = _write( + tmp_path / "app/Models/Page.php", + " Date: Wed, 15 Jul 2026 22:14:11 +0100 Subject: [PATCH 42/42] Add: R (.r/.R) language support via bespoke extract_r Adds a new bespoke R extractor under graphify/extractors/, following the Julia/Elixir precedent. Handles the five function-assignment forms (<-, <<-, =, ->, ->>), nested function scoping, top-level vs function-level call attribution, member ($/@) and package-qualified (::/:::) raw_calls, pipes (|>/%>%), static library/require/requireNamespace imports, and static source() cross-file linkage. Unqualified bare calls route through the shared raw_calls resolver with an R language-family guard that blocks binding to non-R definitions. Wires .r into _DISPATCH/_EXTRA_FOR_EXTENSION/_LANG_FAMILY_BY_EXT and Rscript into _SHEBANG_DISPATCH; .R reuses the existing case-insensitive suffix fallback. Ships under optional [r] extra via tree-sitter-language-pack (no maintained tree-sitter-r wheel fits the per-language model); the pack may fetch/cache the R grammar on first use (no source uploaded, R not executed). Fixes #1689 for .R (no longer a no-AST-extractor case) and reuses the existing #1745 install-extra warning with pip install "graphifyy[r]". Adds tests/fixtures/sample.r + 3 cross-file fixtures, 22 R tests in test_languages.py following the DM optional-grammar skip-when-unavailable convention, and 2 dispatch/missing-dep tests in test_extract.py. R package metadata (DESCRIPTION/NAMESPACE/exports) and package-qualified call resolution are deferred to a follow-up (#9). --- CHANGELOG.md | 2 + README.md | 3 +- graphify/extract.py | 9 +- graphify/extractors/__init__.py | 2 + graphify/extractors/r.py | 281 ++++++++++++++++++++++++++++++++ pyproject.toml | 7 +- tests/fixtures/r_caller.r | 5 + tests/fixtures/r_extra.r | 9 + tests/fixtures/r_other.r | 8 + tests/fixtures/sample.r | 43 +++++ tests/test_extract.py | 49 +++++- tests/test_languages.py | 232 +++++++++++++++++++++++++- 12 files changed, 637 insertions(+), 13 deletions(-) create mode 100644 graphify/extractors/r.py create mode 100644 tests/fixtures/r_caller.r create mode 100644 tests/fixtures/r_extra.r create mode 100644 tests/fixtures/r_other.r create mode 100644 tests/fixtures/sample.r diff --git a/CHANGELOG.md b/CHANGELOG.md index 84d865a6a..a6351a0b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu - Fix: non-English query sentences no longer pick wrong BFS seeds because their filler words were unfiltered (#1900). The query stopword set now covers German and the major Romance languages (curated to avoid clobbering English content words), so `Wie funktioniert die Authentifizierung?` seeds the keyword, not the stopwords. - Fix: Python calls to an imported module now resolve (already shipped in 0.9.16); `.skill` files (Markdown-with-frontmatter agent files) are now classified as documents instead of being silently dropped as an unsupported extension (#1901). - Fix: the `--postgres` missing-driver error now points at the correct PyPI package, `graphifyy[postgres]` (was the nonexistent `graphify[postgres]`) (#1906). +- Add: R (`.r`/`.R`) support — bespoke `extract_r` handles all five function-assignment forms (`<-`, `<<-`, `=`, `->`, `->>`), nested function scoping beneath the enclosing function, top-level vs function-level call attribution, member calls (`$`/`@`), package-qualified calls (`::`/`:::`), and pipes (`|>`/`%>%`). Named functions are assignments whose right- or left-hand side is an anonymous `function_definition`, so the generic extractor is not a good fit; the new bespoke extractor lives under `graphify/extractors/`, the established home for language-specific extractors (not the `extract.py` god node). `library()`/`require()`/`requireNamespace()` with a literal identifier or string emit `imports` edges; `pkg::fn`/`pkg:::fn` double as import evidence for `pkg`. `source("helper.r")` with a static `.r` path resolved against the caller file's directory emits `imports_from` when the target exists on disk (URLs, connections, variables, computed paths, and missing files are skipped — mirroring the Bash static-`source` policy). Unqualified bare calls funnel through the shared cross-file `raw_calls` resolver; an R language-family guard blocks binding to non-R definitions, so a cross-file candidate resolves `EXTRACTED` with `source()` import evidence or `INFERRED` without it, and ambiguous names remain unresolved. Ships under the optional `[r]` extra via `tree-sitter-language-pack` (the pack may fetch/cache the R grammar on first use; no source is uploaded and R itself is not executed). Install: `uv tool install "graphifyy[r]"`. R package metadata (`DESCRIPTION`/`NAMESPACE`/exports) and package-qualified call resolution are deferred to a follow-up (#9). +- Fix: `.R` files no longer trigger the "no AST extractor for `.r`/`.R`" warning (#1689) now that an `extract_r` dispatch exists; the missing-dependency path reuses the existing `#1745` install-extra warning with `pip install "graphifyy[r]"`. `Rscript`-shebang scripts (already detected as code) now dispatch to `extract_r` instead of falling through to the no-extractor warning. ## 0.9.16 (2026-07-14) diff --git a/README.md b/README.md index 81685cf72..c6eba647b 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,7 @@ Codex users also need `multi_agent = true` under `[features]` in `~/.codex/confi | `terraform` | Terraform / HCL `.tf`/`.tfvars`/`.hcl` AST extraction | `uv tool install "graphifyy[terraform]"` | | `pascal` | Pascal / Delphi `.pas`/`.dpr`/`.dpk`/`.inc` AST extraction (more accurate `calls`/`inherits` edges; falls back to a regex extractor when absent) | `uv tool install "graphifyy[pascal]"` | | `chinese` | Chinese query segmentation (jieba) | `uv tool install "graphifyy[chinese]"` | +| `r` | R `.r`/`.R` AST extraction (functions, calls, imports, static `source()`); ships via `tree-sitter-language-pack` which may fetch/cache the R grammar on first use (no source uploaded, R not executed) | `uv tool install "graphifyy[r]"` | | `all` | Everything above | `uv tool install "graphifyy[all]"` | @@ -327,7 +328,7 @@ To remove graphify from all platforms at once: `graphify uninstall` (add `--purg | Type | Extensions | |------|-----------| -| Code (36 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .m .mm .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | +| Code (37 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .m .mm .jl .r .R .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`; `.r`/`.R` requires `uv tool install "graphifyy[r]"`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | | Salesforce Apex | `.cls .trigger` (regex-based; classes, interfaces, enums, methods, triggers, SOQL/DML edges) | | Terraform / HCL | `.tf .tfvars .hcl` (requires `uv tool install graphifyy[terraform]`) | | MCP configs | `.mcp.json` `mcp.json` `mcp_servers.json` `claude_desktop_config.json` — extracts server nodes, package refs, env var requirements | diff --git a/graphify/extract.py b/graphify/extract.py index 2ff5411cf..5aa54eb05 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -139,6 +139,7 @@ from graphify.extractors.objc import _objc_local_var_types, extract_objc # noqa: E402,F401 from graphify.extractors.julia import extract_julia # noqa: E402,F401 +from graphify.extractors.r import extract_r # noqa: E402,F401 _RECURSION_LIMIT = 10_000 @@ -1788,6 +1789,7 @@ def _lang_is_case_insensitive(source_file: object) -> bool: ".zig": "zig", ".ex": "elixir", ".exs": "elixir", ".jl": "julia", + ".r": "r", ".dart": "dart", ".sh": "shell", ".bash": "shell", ".ps1": "powershell", ".psm1": "powershell", ".psd1": "powershell", @@ -3877,6 +3879,7 @@ def add_existing_edge(edge: dict) -> None: ".m": extract_objc, ".mm": extract_objc, ".jl": extract_julia, + ".r": extract_r, ".f": extract_fortran, ".F": extract_fortran, ".f90": extract_fortran, @@ -3943,6 +3946,7 @@ def add_existing_edge(edge: dict) -> None: ".hcl": "terraform", ".dm": "dm", ".dme": "dm", + ".r": "r", } @@ -3951,7 +3955,7 @@ def add_existing_edge(edge: dict) -> None: # routes them to the CODE path via _shebang_interpreter; _get_extractor must # honor the same signal or these files are classified as code and then silently # dropped by extraction. Only interpreters with a real extractor are mapped — -# detect's wider set (perl, fish, tcsh, Rscript) stays unmapped and skipped. +# detect's wider set (perl, fish, tcsh) stays unmapped and skipped. _SHEBANG_DISPATCH: dict[str, Any] = { "python": extract_python, "python2": extract_python, @@ -3967,6 +3971,7 @@ def add_existing_edge(edge: dict) -> None: "lua": extract_lua, "php": extract_php, "julia": extract_julia, + "Rscript": extract_r, } @@ -4403,7 +4408,7 @@ def extract( ) # #1689: a file counted as code (extension in CODE_EXTENSIONS) but with no AST - # extractor wired up (e.g. .r/.R — there is no tree-sitter-r dispatch) silently + # extractor wired up (e.g. .ejs) silently # contributes zero nodes. The #1666 warning above deliberately skips these (it # only fires when an extractor exists), so surface them explicitly, grouped by # extension, rather than reporting success as if the language were mapped. diff --git a/graphify/extractors/__init__.py b/graphify/extractors/__init__.py index ada517094..ffe6b5f09 100644 --- a/graphify/extractors/__init__.py +++ b/graphify/extractors/__init__.py @@ -23,6 +23,7 @@ from graphify.extractors.markdown import extract_markdown from graphify.extractors.objc import extract_objc from graphify.extractors.pascal import extract_pascal +from graphify.extractors.r import extract_r from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest from graphify.extractors.razor import extract_razor @@ -52,6 +53,7 @@ "markdown": extract_markdown, "objc": extract_objc, "pascal": extract_pascal, + "r": extract_r, "powershell": extract_powershell, "powershell_manifest": extract_powershell_manifest, "razor": extract_razor, diff --git a/graphify/extractors/r.py b/graphify/extractors/r.py new file mode 100644 index 000000000..55d8f03d4 --- /dev/null +++ b/graphify/extractors/r.py @@ -0,0 +1,281 @@ +"""R extractor. + +R's named functions are assignments whose left- or right-hand side is an +anonymous ``function_definition`` (``name <- function(...) ...`` and the +right-assignment forms ``function(...) -> name``), so a bespoke extractor is +required rather than the generic one. The contract mirrors the other +extractors: file/function nodes, ``contains``/``calls``/``imports`` edges and +``raw_calls`` for the shared cross-file bare-call resolver. + +Package metadata (DESCRIPTION/NAMESPACE/exports) is intentionally out of scope +for this initial integration; package-qualified ``pkg::fn`` calls are recorded +as member raw facts only. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS, _file_stem, _make_id + + +_ASSIGN_OPS_LEFT = frozenset({"<-", "<<-", "="}) +_ASSIGN_OPS_RIGHT = frozenset({"->", "->>"}) +_PKG_LOADERS = frozenset({"library", "require", "requireNamespace"}) + + +def extract_r(path: Path) -> dict: + """Extract functions, calls, imports, and static source() from a .r/.R file.""" + try: + from tree_sitter_language_pack import get_language + from tree_sitter import Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree-sitter-language-pack not installed"} + + try: + language = get_language("r") + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = _file_stem(path) + str_path = str(path) + del stem # current contract builds function ids from scope nids, not the file stem + + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + function_bodies: list[tuple[str, Any]] = [] + pkg_imports_seen: set[tuple[str, str]] = set() + + def _text(node) -> str: + return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + + def add_node(nid: str, label: str, line: int) -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "code", + "source_file": str_path, "source_location": f"L{line}"}) + + def add_edge(src: str, tgt: str, relation: str, line: int, + confidence: str = "EXTRACTED", weight: float = 1.0, + context: str | None = None) -> None: + if not src or not tgt or src == tgt: + return + edge = {"source": src, "target": tgt, "relation": relation, + "confidence": confidence, "source_file": str_path, + "source_location": f"L{line}", "weight": weight} + if context: + edge["context"] = context + edges.append(edge) + + file_nid = _make_id(str(path)) + add_node(file_nid, path.name, 1) + + def _op_of(binop) -> str: + for child in binop.children: + if child.type in _ASSIGN_OPS_LEFT or child.type in _ASSIGN_OPS_RIGHT: + return child.type + return "" + + def _parent_type(node) -> str: + parent = node.parent + return parent.type if parent is not None else "" + + def _named_children(node): + return [c for c in node.children if c.is_named] + + def _function_body(fn_def): + for c in _named_children(fn_def): + if c.type != "parameters": + return c + return None + + def walk(node, scope_nid: str) -> None: + if node.type == "binary_operator": + op = _op_of(node) + named = _named_children(node) + if op in _ASSIGN_OPS_LEFT and len(named) >= 2: + lhs, rhs = named[0], named[1] + in_call_arg = _parent_type(node) == "argument" + if (not in_call_arg and lhs.type == "identifier" + and rhs.type == "function_definition"): + name = _text(lhs) + line = node.start_point[0] + 1 + func_nid = _make_id(scope_nid, name) + add_node(func_nid, f"{name}()", line) + add_edge(scope_nid, func_nid, "contains", line) + body = _function_body(rhs) + if body is not None: + function_bodies.append((func_nid, body)) + walk(rhs, func_nid) + return + elif op in _ASSIGN_OPS_RIGHT and len(named) >= 2: + lhs, rhs = named[0], named[1] + if rhs.type == "identifier" and lhs.type == "function_definition": + name = _text(rhs) + line = node.start_point[0] + 1 + func_nid = _make_id(scope_nid, name) + add_node(func_nid, f"{name}()", line) + add_edge(scope_nid, func_nid, "contains", line) + body = _function_body(lhs) + if body is not None: + function_bodies.append((func_nid, body)) + walk(lhs, func_nid) + return + for c in node.children: + walk(c, scope_nid) + return + + if node.type == "function_definition": + parent_op = _op_of(node.parent) if node.parent is not None and node.parent.type == "binary_operator" else "" + if parent_op in _ASSIGN_OPS_LEFT or parent_op in _ASSIGN_OPS_RIGHT: + for c in node.children: + walk(c, scope_nid) + return + body = None + for c in _named_children(node): + if c.type not in ("parameters",): + body = c + break + if (body is not None and body.type == "binary_operator" + and _op_of(body) in _ASSIGN_OPS_RIGHT + and _named_children(body)[-1].type == "identifier"): + body_named = _named_children(body) + rhs = body_named[-1] + name = _text(rhs) + line = node.start_point[0] + 1 + func_nid = _make_id(scope_nid, name) + add_node(func_nid, f"{name}()", line) + add_edge(scope_nid, func_nid, "contains", line) + fbody = _function_body(node) + if fbody is not None: + function_bodies.append((func_nid, fbody)) + if len(body_named) >= 2: + walk(body_named[0], func_nid) + return + for c in node.children: + walk(c, scope_nid) + return + + for c in node.children: + walk(c, scope_nid) + + walk(root, file_nid) + + label_to_nid: dict[str, str] = {} + for n in nodes: + normalised = n["label"].strip("()").lstrip(".") + label_to_nid[normalised] = n["id"] + + seen_call_pairs: set[tuple[str, str]] = set() + raw_calls: list[dict] = [] + + def _first_string(node) -> str | None: + for c in node.children: + if c.type == "string": + inner = c.children[1] if len(c.children) > 1 else c + return _text(inner) + if c.type == "argument": + return _first_string(c) + return None + + def _first_identifier(node) -> str | None: + for c in node.children: + if c.type == "identifier": + return _text(c) + if c.type == "argument": + return _first_identifier(c) + return None + + def _emit_pkg_import(scope_nid: str, pkg: str, line: int) -> None: + key = (scope_nid, pkg) + if pkg and key not in pkg_imports_seen: + pkg_imports_seen.add(key) + add_edge(scope_nid, _make_id(pkg), "imports", line, context="import") + + def walk_calls(node, caller_nid: str) -> None: + if node.type == "function_definition": + return + if node.type != "call": + for c in node.children: + walk_calls(c, caller_nid) + return + + line = node.start_point[0] + 1 + fn_node = None + arguments_node = None + for c in node.children: + if c.type in ("identifier", "namespace_operator", "extract_operator"): + fn_node = c + elif c.type == "arguments": + arguments_node = c + + if fn_node is not None and fn_node.type == "identifier": + callee = _text(fn_node) + if callee in _PKG_LOADERS: + if arguments_node is not None: + pkg = _first_identifier(arguments_node) or _first_string(arguments_node) + if pkg: + _emit_pkg_import(caller_nid, pkg, line) + for c in node.children: + walk_calls(c, caller_nid) + return + if callee == "source": + if arguments_node is not None: + raw = _first_string(arguments_node) + if raw and raw.lower().endswith(".r"): + if not raw.startswith(("http://", "https://", "ftp://")): + resolved = (path.parent / raw).resolve() + if resolved.exists(): + add_edge(file_nid, _make_id(str(resolved)), "imports_from", line, context="import") + for c in node.children: + walk_calls(c, caller_nid) + return + is_member_call = False + elif fn_node is not None and fn_node.type == "namespace_operator": + named = _named_children(fn_node) + pkg = _text(named[0]) if named else "" + callee = _text(named[-1]) if named else "" + is_member_call = True + if pkg: + _emit_pkg_import(caller_nid, pkg, line) + elif fn_node is not None and fn_node.type == "extract_operator": + named = _named_children(fn_node) + callee = _text(named[-1]) if named else "" + is_member_call = True + else: + callee = "" + is_member_call = False + + if callee and callee not in _LANGUAGE_BUILTIN_GLOBALS: + tgt_nid = label_to_nid.get(callee) + if tgt_nid and tgt_nid != caller_nid: + pair = (caller_nid, tgt_nid) + if pair not in seen_call_pairs: + seen_call_pairs.add(pair) + add_edge(caller_nid, tgt_nid, "calls", line, + confidence="EXTRACTED", weight=1.0, context="call") + else: + raw_calls.append({ + "caller_nid": caller_nid, + "callee": callee, + "is_member_call": is_member_call, + "source_file": str_path, + "source_location": f"L{line}", + }) + + for c in node.children: + walk_calls(c, caller_nid) + + walk_calls(root, file_nid) + for caller_nid, body in function_bodies: + walk_calls(body, caller_nid) + + clean_edges = [e for e in edges if e["source"] in seen_ids and + (e["target"] in seen_ids or e["relation"] in ("imports", "imports_from"))] + return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls, + "input_tokens": 0, "output_tokens": 0} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 2e5340d39..02daf5a02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,12 @@ pascal = ["tree-sitter-pascal"] # avoids breaking the default `uv tool install graphifyy` for everyone (#1104). dm = ["tree-sitter-dm"] terraform = ["tree-sitter-hcl"] -all = ["mcp", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal"] +# No maintained tree-sitter-r wheel fits the repo's per-language wheel model, +# so the R grammar ships via tree-sitter-language-pack (lazy-loaded in +# extract_r). The pack may fetch/cache the R grammar on first use; no source +# is uploaded and R itself is never executed (#9). +r = ["tree-sitter-language-pack>=1.12,<2"] +all = ["mcp", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal", "tree-sitter-language-pack>=1.12,<2"] [project.scripts] graphify = "graphify.__main__:main" diff --git a/tests/fixtures/r_caller.r b/tests/fixtures/r_caller.r new file mode 100644 index 000000000..99d6ba043 --- /dev/null +++ b/tests/fixtures/r_caller.r @@ -0,0 +1,5 @@ +# R fixture: top-level bare calls with no own definition and no source() link, +# so `dup()` is ambiguous (defined in r_other.R and r_extra.R) and `helper()` +# resolves INFERRED to a single unique cross-file candidate. +dup() +helper() \ No newline at end of file diff --git a/tests/fixtures/r_extra.r b/tests/fixtures/r_extra.r new file mode 100644 index 000000000..72c3604ea --- /dev/null +++ b/tests/fixtures/r_extra.r @@ -0,0 +1,9 @@ +# R fixture: defines a second `dup` (ambiguity with r_other.R) and a no-source +# caller scope, so `helper()` resolves INFERRED unless source() linkage exists. +extra_fn <- function() { + helper() +} + +dup <- function() { + NA +} \ No newline at end of file diff --git a/tests/fixtures/r_other.r b/tests/fixtures/r_other.r new file mode 100644 index 000000000..dd8232809 --- /dev/null +++ b/tests/fixtures/r_other.r @@ -0,0 +1,8 @@ +# R fixture: helper + dup definitions for cross-file resolution tests. +helper <- function() { + invisible() +} + +dup <- function() { + NA +} \ No newline at end of file diff --git a/tests/fixtures/sample.r b/tests/fixtures/sample.r new file mode 100644 index 000000000..8c85741b1 --- /dev/null +++ b/tests/fixtures/sample.r @@ -0,0 +1,43 @@ +# Graphify R fixture: exercises every construct the extractor handles. +# Function assignment forms (left, super, equals, right, super-right): +library(dplyr) +requireNamespace("utils") +library(installed.packages()) # dynamic arg -> NO import edge +source("r_other.r") # static source() -> imports_from edge + +foo <- function(x) { # `<-` form + helper() # cross-file call (defined in r_other.R) + print(x) # builtin in raw_calls +} + +bar <<- function(y) y * 2 # `<<-` super-assignment form + +baz = function(z) { # `=` form + inner <- function(w) { # nested function + helper() # attributed to inner, not baz + z + w + } + inner(z) # same-file call -> EXTRACTED +} + +function(a) a + 1 -> qux # `->` right-assignment form +function(b) b * 2 ->> quux # `->>` super-right-assignment form + +# Member calls (do NOT resolve via the bare-name resolver): +obj$method() +obj@field() + +# Package-qualified calls (recorded as member raw_calls; pkg import edge emitted): +base::summary() +dplyr::filter() +utils:::deep_fn() + +# Pipes (traversed so contained calls are captured): +1 |> qux() |> foo() +"data" %>% process() %>% save() + +# Named call argument with `=` is NOT a function definition: +options(config = function() 42) + +# Top-level call (attributed to the file node): +foo(1) \ No newline at end of file diff --git a/tests/test_extract.py b/tests/test_extract.py index 32c8e8e13..066481f6c 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -1999,18 +1999,19 @@ def test_case_insensitive_suffix_filtering(tmp_path): def test_extract_warns_on_code_files_with_no_ast_extractor(tmp_path, capsys): - # #1689: .r/.R is in CODE_EXTENSIONS (counted as code) but has no AST extractor, - # so R files silently contribute nothing. extract() must surface that instead of - # reporting success as if the language were mapped. - r1 = tmp_path / "analysis.R"; r1.write_text("f <- function(x) x + 1\n") - r2 = tmp_path / "helper.r"; r2.write_text("g <- function(y) y * 2\n") + # #1689: .ejs is in CODE_EXTENSIONS (counted as code) but has no AST extractor, + # so such files silently contribute nothing. extract() must surface that instead + # of reporting success as if the language were mapped. (.R was the original + # motivating case but now has extract_r; .ejs is the canonical no-extractor one.) + r1 = tmp_path / "view.ejs"; r1.write_text("

<%= name %>

\n") + r2 = tmp_path / "page.ejs"; r2.write_text("

Title

\n") py = tmp_path / "main.py"; py.write_text("def main():\n return 1\n") result = extract([r1, r2, py], cache_root=tmp_path) err = capsys.readouterr().err assert "no AST extractor" in err - assert ".r (2)" in err # both R files grouped under the lowercased ext + assert ".ejs (2)" in err assert "#1689" in err # the Python file still extracts normally labels = [n.get("label") for n in result["nodes"]] @@ -2054,6 +2055,38 @@ def test_extract_no_missing_dep_warning_when_sql_installed(tmp_path, capsys): assert "#1745" not in err +def test_extract_warns_when_r_extra_missing(tmp_path, capsys, monkeypatch): + # #1745: .r HAS a dispatch entry, so the #1689 warning can't fire, and + # extract_r returns an "error" result when tree-sitter-language-pack is + # absent, so the files must not vanish silently — extract() surfaces them + # with the [r] extra named. + monkeypatch.setitem(sys.modules, "tree_sitter_language_pack", None) + r1 = tmp_path / "analysis.R"; r1.write_text("f <- function(x) x + 1\n") + r2 = tmp_path / "helper.r"; r2.write_text("g <- function(y) y * 2\n") + py = tmp_path / "main.py"; py.write_text("def main():\n return 1\n") + + result = extract([r1, r2, py], cache_root=tmp_path) + err = capsys.readouterr().err + + assert "2 .r file(s)" in err + assert "tree-sitter-language-pack not installed" in err + assert 'graphifyy[r]' in err + assert "#1745" in err + assert "#1689" not in err # .r/.R is now mapped; no #1689 no-extractor warning + # the Python file still extracts normally + labels = [n.get("label") for n in result["nodes"]] + assert any(str(l).startswith("main") for l in labels) + + +def test_extract_no_warning_when_r_extra_installed(tmp_path, capsys): + pytest.importorskip("tree_sitter_language_pack") + r = tmp_path / "module.R"; r.write_text("f <- function(x) x + 1\n") + extract([r], cache_root=tmp_path) + err = capsys.readouterr().err + assert "#1745" not in err + assert "no AST extractor" not in err + + def test_extract_progress_final_line_uses_consistent_denominator(tmp_path, capsys): # #1693: intermediate progress lines count against uncached_work; the final # "100%" line must NOT switch to total_files (which includes cached hits and @@ -2061,8 +2094,8 @@ def test_extract_progress_final_line_uses_consistent_denominator(tmp_path, capsy for i in range(100): (tmp_path / f"m{i}.py").write_text(f"def f{i}():\n return {i}\n") for i in range(5): - (tmp_path / f"s{i}.r").write_text(f"g{i} <- function(x) x\n") # no extractor - paths = sorted(tmp_path.glob("*.py")) + sorted(tmp_path.glob("*.r")) # total 105 + (tmp_path / f"s{i}.ejs").write_text(f"hello {i}\n") # no extractor + paths = sorted(tmp_path.glob("*.py")) + sorted(tmp_path.glob("*.ejs")) # total 105 extract(paths, cache_root=tmp_path, parallel=False) out = capsys.readouterr().out diff --git a/tests/test_languages.py b/tests/test_languages.py index f36ea7603..3cac22410 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -9,7 +9,7 @@ extract_groovy, extract_sln, extract_csproj, extract_xaml, extract_razor, extract_dm, extract_dmi, extract_dmm, extract_dmf, extract_powershell, extract_apex, extract_verilog, - extract_powershell_manifest, + extract_powershell_manifest, extract_r, ) FIXTURES = Path(__file__).parent / "fixtures" @@ -22,6 +22,10 @@ _ilu.find_spec("tree_sitter_dm") is None, reason="tree-sitter-dm not installed (optional [dm] extra)", ) +_needs_r = pytest.mark.skipif( + _ilu.find_spec("tree_sitter_language_pack") is None, + reason="tree-sitter-language-pack not installed (optional [r] extra)", +) def _labels(r): @@ -2966,3 +2970,229 @@ def test_decldef_merge_does_not_merge_same_name_same_dir_distinct_files(): r = _corpus("cpp_samedir/Alpha.h", "cpp_samedir/Beta.h") dups = _nodes_with_label(r, "Dup") assert len(dups) == 2, f"same-dir distinct Dups must stay distinct, got {[n['id'] for n in dups]}" + + +# --- R --------------------------------------------------------------- + +def _r_assert_no_dangling(r): + """R-aware variant of _assert_no_dangling: `imports` targets may be + external package ids (e.g. dplyr) with no matching node, so they're + allowed to dangle like other languages' external imports.""" + ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + assert e["source"] in ids, f"dangling source: {e}" + if e["relation"] not in ("imports", "imports_from"): + assert e["target"] in ids, f"dangling target: {e}" + + +def test_r_dispatch_lower(): + from graphify.extract import _get_extractor + assert _get_extractor(Path("foo.r")) is extract_r + + +def test_r_dispatch_upper_uses_case_insensitive_fallback(): + # .R is not in _DISPATCH directly; _get_extractor lowercases the suffix + # via the existing fallback (extract.py ~4019-4020). + from graphify.extract import _get_extractor + assert _get_extractor(Path("FOO.R")) is extract_r + + +def test_r_script_shebang_dispatch(tmp_path): + from graphify.extract import _get_extractor + p = tmp_path / "rscript_no_ext" + p.write_text("#!/usr/bin/env Rscript\nx <- 1\n") + assert _get_extractor(p) is extract_r + + +def test_r_missing_dependency_returns_error(tmp_path, monkeypatch): + import sys as _sys + monkeypatch.setitem(_sys.modules, "tree_sitter_language_pack", None) + p = tmp_path / "x.r"; p.write_text("f <- function() 1\n") + r = extract_r(p) + assert "error" in r and "not installed" in r["error"] + + +@_needs_r +def test_r_no_error(): + r = extract_r(FIXTURES / "sample.r") + assert "error" not in r + + +@_needs_r +def test_r_left_assignment_forms_create_functions(): + r = extract_r(FIXTURES / "sample.r") + labels = _labels(r) + assert "foo()" in labels # <- + assert "bar()" in labels # <<- + assert "baz()" in labels # = + + +@_needs_r +def test_r_right_assignment_forms_create_functions(): + r = extract_r(FIXTURES / "sample.r") + labels = _labels(r) + assert "qux()" in labels # -> + assert "quux()" in labels # ->> + + +@_needs_r +def test_r_named_call_argument_with_equals_not_a_function(): + # options(config = function() 42) — the `=` is a named call arg, not an + # assignment binding a function to a name; must NOT produce a `config()` node. + r = extract_r(FIXTURES / "sample.r") + labels = _labels(r) + assert "config()" not in labels + + +@_needs_r +def test_r_nested_functions_get_scoped_ids_and_contains(): + r = extract_r(FIXTURES / "sample.r") + baz = _node_by_label(r, "baz") + inner_nodes = _nodes_with_label(r, "inner()") + assert len(inner_nodes) == 1 + inner = inner_nodes[0] + assert baz["id"] != inner["id"] + contains = _edge_labels(r, "contains") + assert (baz["id"], inner["id"]) in contains or \ + (_normalize_symbol_label("baz()"), _normalize_symbol_label("inner()")) in \ + {(_normalize_symbol_label(a), _normalize_symbol_label(b)) for a, b in contains} + + +@_needs_r +def test_r_nested_call_not_attributed_to_outer_function(): + # inner()'s body calls helper(); this call must attribute to `inner`, not `baz` + r = extract_r(FIXTURES / "sample.r") + baz_id = _node_by_label(r, "baz")["id"] + inner_id = _node_by_label(r, "inner")["id"] + # find a raw_call to helper whose caller is inner (not baz) + helper_rcs = [rc for rc in r["raw_calls"] if rc["callee"] == "helper"] + assert any(rc["caller_nid"] == inner_id for rc in helper_rcs), \ + "inner()'s call to helper must appear in raw_calls from inner, not baz" + assert not any(rc["caller_nid"] == baz_id for rc in helper_rcs), \ + "helper() call must not be attributed to the enclosing baz()" + + +@_needs_r +def test_r_same_file_calls_extracted(): + r = extract_r(FIXTURES / "sample.r") + calls = _calls(r) # set of (raw_src_label, raw_tgt_label) pairs + file_label = "sample.r" + # top-level call to foo() attributed to the file node + assert (file_label, "foo()") in calls + # baz() contains inner(); nested call to inner() attributed to baz + assert ("baz()", "inner()") in calls + # pipe chain captures qux() call at top level + assert (file_label, "qux()") in calls + + +@_needs_r +def test_r_member_calls_recorded_as_member_raw_calls(): + r = extract_r(FIXTURES / "sample.r") + member_callees = {rc["callee"] for rc in r["raw_calls"] if rc.get("is_member_call")} + # $ and @ receivers are member calls + assert "method" in member_callees # obj$method() + assert "field" in member_callees # obj@field + # pkg::fn / pkg:::fn are qualified-member (is_member_call=True) so the + # shared resolver doesn't bind them to a same-named def + assert "summary" in member_callees # pkg::summary + assert "deep_fn" in member_callees # base:::deep_fn + # NOTE: `dplyr::filter` — filter is a Python builtin, filtered from raw_calls + # by _LANGUAGE_BUILTIN_GLOBALS, so it does NOT appear here. + + +@_needs_r +def test_r_member_calls_never_bind_to_same_named_def(): + r = extract_r(FIXTURES / "sample.r") + calls = _calls(r) + # No calls edges to method/field/summary/deep_fn — they're member raw_calls + for callee in ("method", "field", "summary", "deep_fn"): + assert not any(tgt == f"{callee}()" for _, tgt in calls), \ + f"member call {callee}() must not bind to a same-named def" + + +@_needs_r +def test_r_imports_from_static_loaders(): + r = extract_r(FIXTURES / "sample.r") + imports_targets = {e["target"] for e in r["edges"] if e["relation"] == "imports"} + assert "dplyr" in imports_targets # library(dplyr) + assert "utils" in imports_targets # requireNamespace("utils") + assert "base" in imports_targets # pkg::fn emits pkg import evidence + + +@_needs_r +def test_r_imports_skipped_for_dynamic_loader_arg(): + # library(installed.packages()) — the arg is a call expression, not a + # literal identifier/string; must NOT produce an imports edge. + r = extract_r(FIXTURES / "sample.r") + imports_targets = {e["target"] for e in r["edges"] if e["relation"] == "imports"} + assert "installed.packages" not in imports_targets + assert "installed" not in imports_targets + + +@_needs_r +def test_r_static_source_emits_imports_from_to_existing_target(): + r = extract_r(FIXTURES / "sample.r") + ifs = [e for e in r["edges"] if e["relation"] == "imports_from"] + assert len(ifs) == 1 + assert "r_other" in ifs[0]["target"] # sibling r_other.r exists on disk + + +@_needs_r +def test_r_source_ignores_url_missing_and_non_r_targets(tmp_path): + src = tmp_path / "src.r" + src.write_text( + 'source("https://example.com/remote.R")\n' # URL — skip + 'source("missing.r")\n' # missing — skip + 'source("helper.py")\n' # not .r — skip + ) + r = extract_r(src) + assert not [e for e in r["edges"] if e["relation"] == "imports_from"] + + +@_needs_r +def test_r_no_dangling_edges(): + r = extract_r(FIXTURES / "sample.r") + _r_assert_no_dangling(r) + + +@_needs_r +def test_r_pipes_capture_contained_calls(): + r = extract_r(FIXTURES / "sample.r") + callees = {rc["callee"] for rc in r["raw_calls"]} + # pipe chain `1 |> process() |> save()` captures process & save + assert "process" in callees + assert "save" in callees + + +@_needs_r +def test_r_cross_file_call_resolves_with_source_link_extracted(): + # sample.r sources r_other.r, creating imports_from evidence; the cross-file + # helper() call resolves EXTRACTED (single candidate + import evidence). + r = _corpus("sample.r", "r_other.r") + calls = _calls(r) + assert ("foo()", "helper()") in calls or ("inner()", "helper()") in calls, \ + "cross-file helper() must resolve via source() import evidence" + + +@_needs_r +def test_r_cross_file_call_resolves_inferred_without_source(): + # r_caller.r has no source() linkage; helper() in r_other.r has a single + # cross-file candidate → INFERRED (confidence 0.8). + r = _corpus("r_caller.r", "r_other.r") + # Find any calls edge whose normalised target label is "helper" + helper_calls = [e for e in r["edges"] if e["relation"] == "calls" + and "helper" in _normalize_symbol_label(e.get("target", ""))] + assert helper_calls, "unique cross-file bare call must resolve as INFERRED" + # Verify the resolved target is in r_other.r + r_other_helper = [n for n in r["nodes"] if n.get("label") == "helper()"][0] + assert Path(r_other_helper["source_file"]).name == "r_other.r" + + +@_needs_r +def test_r_cross_file_ambiguous_call_remains_unresolved(): + # r_caller.r calls dup(); r_other.r AND r_extra.r both define dup(). + # Ambiguity guard must keep this unresolved (no calls edge to dup()). + r = _corpus("r_caller.r", "r_other.r", "r_extra.r") + dup_calls = [e for e in r["edges"] if e["relation"] == "calls" + and "dup" in _normalize_symbol_label(e.get("target", ""))] + assert len(dup_calls) == 0, f"ambiguous dup() must not resolve, got {dup_calls}"