diff --git a/README.md b/README.md index 0a883e75b..ac4da1158 100644 --- a/README.md +++ b/README.md @@ -706,6 +706,7 @@ graphify extract ./docs --backend claude-cli # route through Claude Code CLI - graphify extract ./docs --backend azure # Azure OpenAI (set AZURE_OPENAI_API_KEY + AZURE_OPENAI_ENDPOINT) graphify extract ./docs --max-workers 16 # AST parallelism (also GRAPHIFY_MAX_WORKERS) graphify extract --postgres "postgresql://user:pass@host/db" # introspect live PostgreSQL schema directly +graphify extract --postgres-ddl schema.sql # ...or from a pre-generated DDL file (pg_dump --schema-only, no live DSN) graphify extract ./my-workspace --cargo # introspect Rust Cargo workspace dependencies directly graphify extract ./docs --token-budget 30000 # smaller semantic chunks for local/small models graphify extract ./docs --max-concurrency 2 # fewer parallel LLM calls (useful for local inference) diff --git a/graphify/__main__.py b/graphify/__main__.py index 82e422e0d..cbb69b2e3 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -608,6 +608,10 @@ def _run_cli() -> None: print(" --postgres DSN extract schema from a live PostgreSQL database") print(" maps tables, views, functions + FK relationships;") print(" column-level detail is not represented in the graph") + print(" --postgres-ddl FILE same schema graph, but from a pre-generated DDL file") + print(" (pg_dump --schema-only, or DDL from any SQL runner) —") + print(" for when a live DSN isn't available; '-' reads stdin.") + print(" Needs only tree-sitter-sql (graphifyy[sql]), not psycopg") print(" --cargo extract crate→crate deps from Cargo.toml") print(" --global also merge the resulting graph into the global graph") print(" --as repo tag for --global (default: target directory name)") diff --git a/graphify/cli.py b/graphify/cli.py index 0bb124777..e07460960 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -1910,7 +1910,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " "[--model M] [--mode deep] [--out DIR] [--google-workspace] [--no-cluster] " "[--max-workers N] [--token-budget N] [--max-concurrency N] " - "[--api-timeout S] [--postgres DSN] [--cargo] [--timing]", + "[--api-timeout S] [--postgres DSN] [--postgres-ddl FILE] [--cargo] [--timing]", file=sys.stderr, ) sys.exit(1) @@ -1930,6 +1930,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": extract_mode: str | None = None out_dir: Path | None = None cli_postgres_dsn: str | None = None + cli_postgres_ddl: str | None = None cli_cargo: bool = False no_cluster = False dedup_llm = False @@ -2030,6 +2031,10 @@ def _parse_float(name: str, raw: str) -> float: cli_excludes.append(args[i + 1]); i += 2 elif a.startswith("--exclude="): cli_excludes.append(a.split("=", 1)[1]); i += 1 + elif a == "--postgres-ddl" and i + 1 < len(args): + cli_postgres_ddl = args[i + 1]; i += 2 + elif a.startswith("--postgres-ddl="): + cli_postgres_ddl = a.split("=", 1)[1]; i += 1 elif a == "--postgres" and i + 1 < len(args): cli_postgres_dsn = args[i + 1]; i += 2 elif a.startswith("--postgres="): @@ -2042,8 +2047,12 @@ def _parse_float(name: str, raw: str) -> float: else: i += 1 - if not has_path and cli_postgres_dsn is None: - print("error: must specify a path to scan or a --postgres DSN", file=sys.stderr) + if cli_postgres_dsn is not None and cli_postgres_ddl is not None: + print("error: pass either --postgres or --postgres-ddl, not both", file=sys.stderr) + sys.exit(1) + + if not has_path and cli_postgres_dsn is None and cli_postgres_ddl is None: + print("error: must specify a path to scan, a --postgres DSN, or --postgres-ddl ", file=sys.stderr) sys.exit(1) _VALID_MODES = {"deep"} @@ -2402,6 +2411,19 @@ def _progress(idx: int, total: int, _result: dict) -> None: sys.exit(1) print(f"[graphify extract] PostgreSQL: {len(pg_result['nodes'])} nodes, " f"{len(pg_result['edges'])} edges") + elif cli_postgres_ddl is not None: + from graphify.pg_introspect import introspect_postgres_ddl + print("[graphify extract] building PostgreSQL schema graph from DDL file...") + try: + ddl_text = (sys.stdin.read() if cli_postgres_ddl == "-" + else Path(cli_postgres_ddl).read_text(encoding="utf-8")) + except OSError as exc: + print(f"error: could not read --postgres-ddl file: {exc}", file=sys.stderr) + sys.exit(1) + dbname = "schema" if cli_postgres_ddl == "-" else Path(cli_postgres_ddl).stem + pg_result = introspect_postgres_ddl(ddl_text, dbname=dbname) + print(f"[graphify extract] PostgreSQL DDL: {len(pg_result['nodes'])} nodes, " + f"{len(pg_result['edges'])} edges") cargo_result: dict = {"nodes": [], "edges": []} if cli_cargo: diff --git a/graphify/pg_introspect.py b/graphify/pg_introspect.py index dc7b2bbf8..f8a5c3853 100644 --- a/graphify/pg_introspect.py +++ b/graphify/pg_introspect.py @@ -8,6 +8,34 @@ def _quote_ident(name: str) -> str: return '"' + name.replace('"', '""') + '"' +def _schema_ddl_to_graph(ddl: str, *, host: str = "localhost", dbname: str = "db") -> dict: + """Turn a schema DDL string into a graph via ``extract_sql``. + + Shared by :func:`introspect_postgres` (live connection) and + :func:`introspect_postgres_ddl` (pre-generated DDL). The ``postgresql://`` + virtual path is what frames the resulting nodes as a DB schema rather than a + local file; it never carries credentials. + """ + virtual_path = PurePosixPath(f"postgresql://{host}/{dbname}") + return extract_sql(virtual_path, content=ddl) + + +def introspect_postgres_ddl(ddl: str, *, dbname: str = "schema") -> dict: + """Build the schema graph from a pre-generated DDL string — no live connection. + + Use this when a live DSN isn't available: DB access mediated by a tool/MCP + server that runs SQL but never exposes a ``postgresql://`` string, a + ``pg_dump --schema-only`` artifact, or an air-gapped build. ``ddl`` is any + sequence of ``CREATE TABLE`` / ``CREATE VIEW`` / ``CREATE FUNCTION`` and + ``ALTER TABLE … ADD FOREIGN KEY`` statements — the same shape + :func:`introspect_postgres` synthesizes from the live catalog. Reuses the + identical downstream framing, so the nodes are indistinguishable from a live + ``--postgres`` run. Requires only tree-sitter-sql (``graphifyy[sql]``), not + psycopg. + """ + return _schema_ddl_to_graph(ddl, dbname=dbname) + + def introspect_postgres(dsn: str | None = None) -> dict: """Connect to PostgreSQL, reconstruct DDL, and extract via extract_sql().""" try: @@ -144,8 +172,6 @@ def introspect_postgres(dsn: str | None = None) -> dict: info = psycopg.conninfo.conninfo_to_dict(dsn or "") host = info.get("host", "localhost") dbname = info.get("dbname", "db") - virtual_path = PurePosixPath(f"postgresql://{host}/{dbname}") - # Pass virtual path and in-memory DDL content to extract_sql - result = extract_sql(virtual_path, content=ddl_string) - return result \ No newline at end of file + # Same DDL → graph step as the DDL-file path (introspect_postgres_ddl). + return _schema_ddl_to_graph(ddl_string, host=host, dbname=dbname) \ No newline at end of file diff --git a/tests/test_pg_introspect.py b/tests/test_pg_introspect.py index 919d7ba40..b925a2c84 100644 --- a/tests/test_pg_introspect.py +++ b/tests/test_pg_introspect.py @@ -5,7 +5,7 @@ pytest.importorskip("tree_sitter_sql", reason="tree-sitter-sql not installed; skip pg_introspect tests") -from graphify.pg_introspect import introspect_postgres +from graphify.pg_introspect import introspect_postgres, introspect_postgres_ddl from graphify.validate import validate_extraction @@ -322,10 +322,92 @@ def test_pg_introspect_uri_forward_slashes(): mock_psycopg = _make_mock_psycopg([], [], [], [], host="some-host", dbname="some-db") with patch.dict("sys.modules", {"psycopg": mock_psycopg}): res = introspect_postgres("postgresql://some-host/some-db") - + # We should have at least the file node assert len(res["nodes"]) > 0 for node in res["nodes"]: assert "\\" not in node["source_file"] assert "postgresql:/some-host/some-db" in node["source_file"] + +# --------------------------------------------------------------------------- +# introspect_postgres_ddl — build the schema graph from pre-generated DDL, +# with no live connection (MCP-only access, pg_dump artifact, air-gapped). +# --------------------------------------------------------------------------- + +# The same shape introspect_postgres synthesizes from the live catalog: +# quoted identifiers, stub function bodies, one ALTER TABLE per FK. +_SAMPLE_DDL = ( + 'CREATE TABLE "public"."users" (id INT);\n' + 'CREATE TABLE "public"."orders" (id INT);\n' + 'CREATE VIEW "public"."active_users" AS SELECT 1;\n' + 'CREATE FUNCTION "public"."calc_total"() RETURNS void' + ' AS $gfx$ BEGIN SELECT 1; END; $gfx$ LANGUAGE plpgsql;\n' + 'ALTER TABLE "public"."orders" ADD CONSTRAINT fk_orders_user' + ' FOREIGN KEY (user_id) REFERENCES "public"."users"(id);\n' +) + + +def test_introspect_postgres_ddl_from_string(): + """A DDL string produces the same graph shape as a live introspection: + table/view/function nodes, an FK references edge, and the sanitized + postgresql:// virtual source path — no psycopg, no connection.""" + res = introspect_postgres_ddl(_SAMPLE_DDL, dbname="mydb") + + errors = validate_extraction(res) + assert errors == [], f"Validation errors: {errors}" + + # Same virtual-path framing as --postgres (host defaults to localhost). + for node in res["nodes"]: + assert node["source_file"] == "postgresql:/localhost/mydb" + for edge in res["edges"]: + assert edge["source_file"] == "postgresql:/localhost/mydb" + + node_labels = {n["label"] for n in res["nodes"]} + assert _q("public", "users") in node_labels, f"users missing; got {node_labels}" + assert _q("public", "orders") in node_labels, f"orders missing; got {node_labels}" + assert _q("public", "active_users") in node_labels, f"view missing; got {node_labels}" + assert f'{_q("public", "calc_total")}()' in node_labels, f"function missing; got {node_labels}" + + # File node carries the dbname label. + file_nodes = [n for n in res["nodes"] if n["file_type"] == "code" and n["label"] == "mydb"] + assert len(file_nodes) == 1 + + # FK -> a single references edge, orders -> users. + users_nid = next(n["id"] for n in res["nodes"] if n["label"] == _q("public", "users")) + orders_nid = next(n["id"] for n in res["nodes"] if n["label"] == _q("public", "orders")) + ref_edges = [ + e for e in res["edges"] + if e["source"] == orders_nid and e["target"] == users_nid and e["relation"] == "references" + ] + assert len(ref_edges) == 1, f"Expected exactly 1 references edge, got {len(ref_edges)}" + + +def test_introspect_postgres_ddl_needs_no_psycopg(): + """The DDL path must not require psycopg — that's the whole point. Even with + psycopg unimportable, the graph still builds.""" + with patch.dict("sys.modules", {"psycopg": None}): + res = introspect_postgres_ddl('CREATE TABLE "public"."t" (id INT);', dbname="db") + assert any(n["label"] == _q("public", "t") for n in res["nodes"]) + + +def test_introspect_postgres_ddl_matches_live_shape(): + """introspect_postgres_ddl and introspect_postgres yield identical nodes/edges + for the same schema — the DDL path is just the live path without the connection.""" + mock_tables = [("public", "users", "BASE TABLE"), ("public", "orders", "BASE TABLE")] + mock_views = [("public", "active_users", "SELECT 1")] + mock_routines = [("public", "calc_total", "FUNCTION", "SELECT 1;", "PLPGSQL")] + mock_fks = [("fk_orders_user", "public", "orders", ["user_id"], "public", "users", ["id"])] + mock_psycopg = _make_mock_psycopg(mock_tables, mock_views, mock_routines, mock_fks, + host="localhost", dbname="mydb") + with patch.dict("sys.modules", {"psycopg": mock_psycopg}): + live = introspect_postgres("postgresql://u:p@localhost/mydb") + + ddl = introspect_postgres_ddl(_SAMPLE_DDL, dbname="mydb") + + assert {n["label"] for n in ddl["nodes"]} == {n["label"] for n in live["nodes"]} + assert ( + sorted((e["relation"] for e in ddl["edges"])) + == sorted((e["relation"] for e in live["edges"])) + ) +