Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -706,6 +706,7 @@ graphify extract ./docs --backend claude-cli # route through Claude Code CLI -
graphify extract ./docs --backend azure # Azure OpenAI (set AZURE_OPENAI_API_KEY + AZURE_OPENAI_ENDPOINT)
graphify extract ./docs --max-workers 16 # AST parallelism (also GRAPHIFY_MAX_WORKERS)
graphify extract --postgres "postgresql://user:pass@host/db" # introspect live PostgreSQL schema directly
graphify extract --postgres-ddl schema.sql # ...or from a pre-generated DDL file (pg_dump --schema-only, no live DSN)
graphify extract ./my-workspace --cargo # introspect Rust Cargo workspace dependencies directly
graphify extract ./docs --token-budget 30000 # smaller semantic chunks for local/small models
graphify extract ./docs --max-concurrency 2 # fewer parallel LLM calls (useful for local inference)
Expand Down
4 changes: 4 additions & 0 deletions graphify/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -608,6 +608,10 @@ def _run_cli() -> None:
print(" --postgres DSN extract schema from a live PostgreSQL database")
print(" maps tables, views, functions + FK relationships;")
print(" column-level detail is not represented in the graph")
print(" --postgres-ddl FILE same schema graph, but from a pre-generated DDL file")
print(" (pg_dump --schema-only, or DDL from any SQL runner) —")
print(" for when a live DSN isn't available; '-' reads stdin.")
print(" Needs only tree-sitter-sql (graphifyy[sql]), not psycopg")
print(" --cargo extract crate→crate deps from Cargo.toml")
print(" --global also merge the resulting graph into the global graph")
print(" --as <tag> repo tag for --global (default: target directory name)")
Expand Down
28 changes: 25 additions & 3 deletions graphify/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -1910,7 +1910,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph":
"Usage: graphify extract <path> [--backend gemini|kimi|claude|openai|deepseek|ollama] "
"[--model M] [--mode deep] [--out DIR] [--google-workspace] [--no-cluster] "
"[--max-workers N] [--token-budget N] [--max-concurrency N] "
"[--api-timeout S] [--postgres DSN] [--cargo] [--timing]",
"[--api-timeout S] [--postgres DSN] [--postgres-ddl FILE] [--cargo] [--timing]",
file=sys.stderr,
)
sys.exit(1)
Expand All @@ -1930,6 +1930,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph":
extract_mode: str | None = None
out_dir: Path | None = None
cli_postgres_dsn: str | None = None
cli_postgres_ddl: str | None = None
cli_cargo: bool = False
no_cluster = False
dedup_llm = False
Expand Down Expand Up @@ -2030,6 +2031,10 @@ def _parse_float(name: str, raw: str) -> float:
cli_excludes.append(args[i + 1]); i += 2
elif a.startswith("--exclude="):
cli_excludes.append(a.split("=", 1)[1]); i += 1
elif a == "--postgres-ddl" and i + 1 < len(args):
cli_postgres_ddl = args[i + 1]; i += 2
elif a.startswith("--postgres-ddl="):
cli_postgres_ddl = a.split("=", 1)[1]; i += 1
elif a == "--postgres" and i + 1 < len(args):
cli_postgres_dsn = args[i + 1]; i += 2
elif a.startswith("--postgres="):
Expand All @@ -2042,8 +2047,12 @@ def _parse_float(name: str, raw: str) -> float:
else:
i += 1

if not has_path and cli_postgres_dsn is None:
print("error: must specify a path to scan or a --postgres DSN", file=sys.stderr)
if cli_postgres_dsn is not None and cli_postgres_ddl is not None:
print("error: pass either --postgres or --postgres-ddl, not both", file=sys.stderr)
sys.exit(1)

if not has_path and cli_postgres_dsn is None and cli_postgres_ddl is None:
print("error: must specify a path to scan, a --postgres DSN, or --postgres-ddl <file>", file=sys.stderr)
sys.exit(1)

_VALID_MODES = {"deep"}
Expand Down Expand Up @@ -2402,6 +2411,19 @@ def _progress(idx: int, total: int, _result: dict) -> None:
sys.exit(1)
print(f"[graphify extract] PostgreSQL: {len(pg_result['nodes'])} nodes, "
f"{len(pg_result['edges'])} edges")
elif cli_postgres_ddl is not None:
from graphify.pg_introspect import introspect_postgres_ddl
print("[graphify extract] building PostgreSQL schema graph from DDL file...")
try:
ddl_text = (sys.stdin.read() if cli_postgres_ddl == "-"
else Path(cli_postgres_ddl).read_text(encoding="utf-8"))
except OSError as exc:
print(f"error: could not read --postgres-ddl file: {exc}", file=sys.stderr)
sys.exit(1)
dbname = "schema" if cli_postgres_ddl == "-" else Path(cli_postgres_ddl).stem
pg_result = introspect_postgres_ddl(ddl_text, dbname=dbname)
print(f"[graphify extract] PostgreSQL DDL: {len(pg_result['nodes'])} nodes, "
f"{len(pg_result['edges'])} edges")

cargo_result: dict = {"nodes": [], "edges": []}
if cli_cargo:
Expand Down
34 changes: 30 additions & 4 deletions graphify/pg_introspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,34 @@ def _quote_ident(name: str) -> str:
return '"' + name.replace('"', '""') + '"'


def _schema_ddl_to_graph(ddl: str, *, host: str = "localhost", dbname: str = "db") -> dict:
"""Turn a schema DDL string into a graph via ``extract_sql``.

Shared by :func:`introspect_postgres` (live connection) and
:func:`introspect_postgres_ddl` (pre-generated DDL). The ``postgresql://``
virtual path is what frames the resulting nodes as a DB schema rather than a
local file; it never carries credentials.
"""
virtual_path = PurePosixPath(f"postgresql://{host}/{dbname}")
return extract_sql(virtual_path, content=ddl)


def introspect_postgres_ddl(ddl: str, *, dbname: str = "schema") -> dict:
"""Build the schema graph from a pre-generated DDL string — no live connection.

Use this when a live DSN isn't available: DB access mediated by a tool/MCP
server that runs SQL but never exposes a ``postgresql://`` string, a
``pg_dump --schema-only`` artifact, or an air-gapped build. ``ddl`` is any
sequence of ``CREATE TABLE`` / ``CREATE VIEW`` / ``CREATE FUNCTION`` and
``ALTER TABLE … ADD FOREIGN KEY`` statements — the same shape
:func:`introspect_postgres` synthesizes from the live catalog. Reuses the
identical downstream framing, so the nodes are indistinguishable from a live
``--postgres`` run. Requires only tree-sitter-sql (``graphifyy[sql]``), not
psycopg.
"""
return _schema_ddl_to_graph(ddl, dbname=dbname)


def introspect_postgres(dsn: str | None = None) -> dict:
"""Connect to PostgreSQL, reconstruct DDL, and extract via extract_sql()."""
try:
Expand Down Expand Up @@ -144,8 +172,6 @@ def introspect_postgres(dsn: str | None = None) -> dict:
info = psycopg.conninfo.conninfo_to_dict(dsn or "")
host = info.get("host", "localhost")
dbname = info.get("dbname", "db")
virtual_path = PurePosixPath(f"postgresql://{host}/{dbname}")

# Pass virtual path and in-memory DDL content to extract_sql
result = extract_sql(virtual_path, content=ddl_string)
return result
# Same DDL → graph step as the DDL-file path (introspect_postgres_ddl).
return _schema_ddl_to_graph(ddl_string, host=host, dbname=dbname)
86 changes: 84 additions & 2 deletions tests/test_pg_introspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

pytest.importorskip("tree_sitter_sql", reason="tree-sitter-sql not installed; skip pg_introspect tests")

from graphify.pg_introspect import introspect_postgres
from graphify.pg_introspect import introspect_postgres, introspect_postgres_ddl
from graphify.validate import validate_extraction


Expand Down Expand Up @@ -322,10 +322,92 @@ def test_pg_introspect_uri_forward_slashes():
mock_psycopg = _make_mock_psycopg([], [], [], [], host="some-host", dbname="some-db")
with patch.dict("sys.modules", {"psycopg": mock_psycopg}):
res = introspect_postgres("postgresql://some-host/some-db")

# We should have at least the file node
assert len(res["nodes"]) > 0
for node in res["nodes"]:
assert "\\" not in node["source_file"]
assert "postgresql:/some-host/some-db" in node["source_file"]


# ---------------------------------------------------------------------------
# introspect_postgres_ddl — build the schema graph from pre-generated DDL,
# with no live connection (MCP-only access, pg_dump artifact, air-gapped).
# ---------------------------------------------------------------------------

# The same shape introspect_postgres synthesizes from the live catalog:
# quoted identifiers, stub function bodies, one ALTER TABLE per FK.
_SAMPLE_DDL = (
'CREATE TABLE "public"."users" (id INT);\n'
'CREATE TABLE "public"."orders" (id INT);\n'
'CREATE VIEW "public"."active_users" AS SELECT 1;\n'
'CREATE FUNCTION "public"."calc_total"() RETURNS void'
' AS $gfx$ BEGIN SELECT 1; END; $gfx$ LANGUAGE plpgsql;\n'
'ALTER TABLE "public"."orders" ADD CONSTRAINT fk_orders_user'
' FOREIGN KEY (user_id) REFERENCES "public"."users"(id);\n'
)


def test_introspect_postgres_ddl_from_string():
"""A DDL string produces the same graph shape as a live introspection:
table/view/function nodes, an FK references edge, and the sanitized
postgresql:// virtual source path — no psycopg, no connection."""
res = introspect_postgres_ddl(_SAMPLE_DDL, dbname="mydb")

errors = validate_extraction(res)
assert errors == [], f"Validation errors: {errors}"

# Same virtual-path framing as --postgres (host defaults to localhost).
for node in res["nodes"]:
assert node["source_file"] == "postgresql:/localhost/mydb"
for edge in res["edges"]:
assert edge["source_file"] == "postgresql:/localhost/mydb"

node_labels = {n["label"] for n in res["nodes"]}
assert _q("public", "users") in node_labels, f"users missing; got {node_labels}"
assert _q("public", "orders") in node_labels, f"orders missing; got {node_labels}"
assert _q("public", "active_users") in node_labels, f"view missing; got {node_labels}"
assert f'{_q("public", "calc_total")}()' in node_labels, f"function missing; got {node_labels}"

# File node carries the dbname label.
file_nodes = [n for n in res["nodes"] if n["file_type"] == "code" and n["label"] == "mydb"]
assert len(file_nodes) == 1

# FK -> a single references edge, orders -> users.
users_nid = next(n["id"] for n in res["nodes"] if n["label"] == _q("public", "users"))
orders_nid = next(n["id"] for n in res["nodes"] if n["label"] == _q("public", "orders"))
ref_edges = [
e for e in res["edges"]
if e["source"] == orders_nid and e["target"] == users_nid and e["relation"] == "references"
]
assert len(ref_edges) == 1, f"Expected exactly 1 references edge, got {len(ref_edges)}"


def test_introspect_postgres_ddl_needs_no_psycopg():
"""The DDL path must not require psycopg — that's the whole point. Even with
psycopg unimportable, the graph still builds."""
with patch.dict("sys.modules", {"psycopg": None}):
res = introspect_postgres_ddl('CREATE TABLE "public"."t" (id INT);', dbname="db")
assert any(n["label"] == _q("public", "t") for n in res["nodes"])


def test_introspect_postgres_ddl_matches_live_shape():
"""introspect_postgres_ddl and introspect_postgres yield identical nodes/edges
for the same schema — the DDL path is just the live path without the connection."""
mock_tables = [("public", "users", "BASE TABLE"), ("public", "orders", "BASE TABLE")]
mock_views = [("public", "active_users", "SELECT 1")]
mock_routines = [("public", "calc_total", "FUNCTION", "SELECT 1;", "PLPGSQL")]
mock_fks = [("fk_orders_user", "public", "orders", ["user_id"], "public", "users", ["id"])]
mock_psycopg = _make_mock_psycopg(mock_tables, mock_views, mock_routines, mock_fks,
host="localhost", dbname="mydb")
with patch.dict("sys.modules", {"psycopg": mock_psycopg}):
live = introspect_postgres("postgresql://u:p@localhost/mydb")

ddl = introspect_postgres_ddl(_SAMPLE_DDL, dbname="mydb")

assert {n["label"] for n in ddl["nodes"]} == {n["label"] for n in live["nodes"]}
assert (
sorted((e["relation"] for e in ddl["edges"]))
== sorted((e["relation"] for e in live["edges"]))
)