fix(detect): let .graphifyinclude rescue generic keyword skips (#1225)

graphify/detect.py

@@ -182,17 +182,25 @@ def _generic_keyword_hit(name: str) -> bool:
 _PAPER_SIGNAL_THRESHOLD = 3  # need at least this many signals to call it a paper
 
 
-def _is_sensitive(path: Path) -> bool:
-    """Return True if this file likely contains secrets and should be skipped."""
+# Reasons returned by _sensitive_reason. Stages 1-2 match specific, near-certain
+# secret locations; Stage 3 is a heuristic guess on the name, so it is the only
+# stage an explicit .graphifyinclude entry may override (#1225).
+_SENSITIVE_DIR = "dir"
+_SENSITIVE_PATTERN = "pattern"
+_SENSITIVE_KEYWORD = "keyword"
+
+
+def _sensitive_reason(path: Path) -> str | None:
+    """Return which stage flags this file as likely containing secrets, or None."""
     # Stage 1: any PARENT directory is a known secrets dir (parts[:-1] excludes
     # the filename itself so a root-level file named "credentials" is not falsely
     # skipped — the name patterns in Stage 2 handle the filename).
     if any(part in _SENSITIVE_DIRS for part in path.parts[:-1]):
-        return True
+        return _SENSITIVE_DIR
     # Stage 2: filename pattern match
     name = path.name
     if any(p.search(name) for p in _SENSITIVE_PATTERNS):
-        return True
+        return _SENSITIVE_PATTERN
     # Stage 3: generic keywords, only when load-bearing in the name. Do NOT let a
     # bare name keyword silently drop a genuine programming-language source file:
     # a .rb/.py named device_token or passwords_controller is a module, not a secret
@@ -204,8 +212,13 @@ def _is_sensitive(path: Path) -> bool:
     if _generic_keyword_hit(name):
         ext = path.suffix.lower()
         is_source_code = classify_file(path) == FileType.CODE and ext not in _SECRET_PRONE_DATA_EXTS
-        return not is_source_code
-    return False
+        return None if is_source_code else _SENSITIVE_KEYWORD
+    return None
+
+
+def _is_sensitive(path: Path) -> bool:
+    """Return True if this file likely contains secrets and should be skipped."""
+    return _sensitive_reason(path) is not None
 
 
 def _looks_like_paper(path: Path) -> bool:
@@ -994,8 +1007,10 @@ def _matches(rel: str, p: str, anchored: bool) -> bool:
 def _load_graphifyinclude(root: Path) -> list[tuple[Path, str]]:
     """Read .graphifyinclude allowlist patterns from root and ancestors.
 
-    Include patterns opt matching hidden files/dirs into traversal. Sensitive
-    files and hard-skipped noise directories are still excluded later.
+    An include entry states explicit user intent to graph a file: wildcard-free
+    entries rescue files from the generic secrets keyword heuristic (#1225).
+    The specific sensitive patterns (secrets dirs, .env, key files) and
+    hard-skipped noise directories are still always excluded.
     Uses the same VCS-root ceiling logic as _load_graphifyignore.
     """
     root = root.resolve()
@@ -1150,6 +1165,7 @@ def _wc(path: Path) -> int:
         return _cache.cached_word_count(path, root, count_words, cache_root=cache_root)
 
     skipped_sensitive: list[str] = []
+    keyword_skipped: list[str] = []
     unclassified: list[str] = []
     ignore_patterns = _load_graphifyignore(root)
     ignore_cache: dict[Path, bool] = {}  # shared across all _is_ignored calls in this scan
@@ -1160,7 +1176,15 @@ def _wc(path: Path) -> int:
             line = _parse_gitignore_line(pat)
             if line:
                 ignore_patterns.append((root, line))
-    include_patterns = _load_graphifyinclude(root)
+    # Only wildcard-free .graphifyinclude entries may rescue a file from the
+    # secrets keyword heuristic: a broad glob written to opt hidden files into
+    # traversal must not silently start ingesting keyword-named credential
+    # stores like api_token.txt (#1225).
+    explicit_includes = [
+        (anchor, pat)
+        for anchor, pat in _load_graphifyinclude(root)
+        if not any(ch in pat for ch in "*?[")
+    ]
 
     # Always include graphify-out/memory/ - query results filed back into the graph
     memory_dir = root / GRAPHIFY_OUT / "memory"
@@ -1260,8 +1284,18 @@ def _on_walk_error(err: OSError) -> None:
         if not _resolves_under_root(p, root):
             skipped_sensitive.append(str(p) + " [symlink target outside scan root]")
             continue
-        if _is_sensitive(p):
-            skipped_sensitive.append(str(p))
+        sensitive = _sensitive_reason(p)
+        if sensitive == _SENSITIVE_KEYWORD and _is_included(p, root, explicit_includes):
+            sensitive = None  # explicit allowlist entry beats the name heuristic (#1225)
+        if sensitive is not None:
+            if sensitive == _SENSITIVE_KEYWORD:
+                keyword_skipped.append(str(p))
+                skipped_sensitive.append(
+                    str(p) + " [name matches a secrets keyword - add the exact "
+                    "path to .graphifyinclude to include it]"
+                )
+            else:
+                skipped_sensitive.append(str(p))
             continue
         ftype = classify_file(p)
         if not ftype:
@@ -1309,6 +1343,17 @@ def _on_walk_error(err: OSError) -> None:
             if ftype != FileType.VIDEO:
                 total_words += _wc(p)
 
+    if keyword_skipped:
+        import sys as _sys
+        _names = ", ".join(Path(f).name for f in keyword_skipped[:6])
+        _more = f" (+{len(keyword_skipped) - 6} more)" if len(keyword_skipped) > 6 else ""
+        print(
+            f"[graphify] {len(keyword_skipped)} file(s) skipped because their "
+            f"names match secrets keywords: {_names}{_more}. False positive? "
+            f"Add the exact path to .graphifyinclude to include it.",
+            file=_sys.stderr,
+        )
+
     for ftype in files:
         files[ftype].sort()
 

graphify/report.py

@@ -113,6 +113,17 @@ def generate(
             "- Verdict: corpus is large enough that graph structure adds value.",
         ]
 
+    # Every skipped file by name, not a bare count: one benign doc swallowed by
+    # the secrets heuristic must stay findable by grepping the report (#1225).
+    skipped = detection_result.get("skipped_sensitive") or []
+    if skipped:
+        lines += [
+            "",
+            "## Skipped Files",
+            f"{len(skipped)} file(s) were skipped during the scan and are absent from the graph:",
+        ]
+        lines += [f"- `{entry}`" for entry in skipped]
+
     from .analyze import _is_file_node as _ifn
     non_empty = {cid: nodes for cid, nodes in communities.items()
                  if any(not _ifn(G, n) for n in nodes)}

tests/test_detect.py

@@ -1099,6 +1099,66 @@ def test_sensitive_flags_plural_tokens_txt():
     assert _is_sensitive(Path("tokens.txt"))
 
 
+# ── Issue #1225: .graphifyinclude rescues generic keyword skips ───────────────
+# The keyword stage is the only heuristic guess; an exact .graphifyinclude entry
+# states user intent and overrides it. The specific credential patterns and
+# secrets dirs are never overridden, and glob entries don't count as explicit.
+
+def test_include_rescues_keyword_skipped_doc(tmp_path):
+    docs = tmp_path / "docs"
+    docs.mkdir()
+    doc = docs / "derive-locked-tokens.md"
+    doc.write_text("A prompt template about locking proper-noun tokens.\n" * 20)
+    (tmp_path / ".graphifyinclude").write_text("docs/derive-locked-tokens.md\n")
+
+    result = detect(tmp_path)
+
+    assert any("derive-locked-tokens.md" in f for f in result["files"]["document"])
+    assert not any("derive-locked-tokens" in item for item in result["skipped_sensitive"])
+
+
+def test_keyword_skip_is_annotated_and_names_escape_hatch(tmp_path, capsys):
+    doc = tmp_path / "deploy-tokens.md"
+    doc.write_text("Runbook for the deploy token rotation schedule.\n" * 20)
+
+    result = detect(tmp_path)
+
+    assert not any("deploy-tokens.md" in f for f in result["files"]["document"])
+    assert any(
+        "deploy-tokens.md" in item and ".graphifyinclude" in item
+        for item in result["skipped_sensitive"]
+    )
+    err = capsys.readouterr().err
+    assert "deploy-tokens.md" in err
+    assert ".graphifyinclude" in err
+
+
+def test_include_rescue_requires_exact_path_not_glob(tmp_path):
+    doc = tmp_path / "deploy-tokens.md"
+    doc.write_text("Runbook for the deploy token rotation schedule.\n" * 20)
+    (tmp_path / ".graphifyinclude").write_text("*\n")
+
+    result = detect(tmp_path)
+
+    assert not any("deploy-tokens.md" in f for f in result["files"]["document"])
+    assert any("deploy-tokens.md" in item for item in result["skipped_sensitive"])
+
+
+def test_include_cannot_rescue_credential_stores(tmp_path):
+    secrets = tmp_path / "secrets"
+    secrets.mkdir()
+    (secrets / "db.json").write_text('{"password": "hunter2"}')
+    (tmp_path / "server.pem").write_text("-----BEGIN PRIVATE KEY-----\n")
+    (tmp_path / ".graphifyinclude").write_text("secrets/db.json\nserver.pem\n")
+
+    result = detect(tmp_path)
+
+    assert any("db.json" in item for item in result["skipped_sensitive"])
+    assert any("server.pem" in item for item in result["skipped_sensitive"])
+    for flist in result["files"].values():
+        assert not any("db.json" in f or "server.pem" in f for f in flist)
+
+
 # ── Issue #933: failed-chunk files must not be frozen in manifest ─────────────
 
 def test_save_manifest_skips_semantic_hash_for_files_without_cache(tmp_path):

tests/test_report.py

@@ -154,3 +154,23 @@ def test_report_hubs_use_wikilinks_when_obsidian():
     labels = {cid: f"Widget {cid}" for cid in communities}
     report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project", min_community_size=1, obsidian=True)
     assert "[[_COMMUNITY_" in report
+
+
+def test_report_lists_skipped_sensitive_files():
+    # #1225: skipped files must be listed by name, not reduced to a count, so a
+    # silently dropped file is findable by grepping the report.
+    G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs()
+    detection["skipped_sensitive"] = [
+        "/proj/references/prompts/derive-locked-tokens.md [name matches a secrets keyword]",
+        "/proj/.ssh/id_rsa",
+    ]
+    report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project")
+    assert "## Skipped Files" in report
+    assert "derive-locked-tokens.md" in report
+    assert "id_rsa" in report
+
+
+def test_report_omits_skipped_section_when_nothing_skipped():
+    G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs()
+    report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project")
+    assert "## Skipped Files" not in report