From b1d02384163258cf756195203f97f2ab9c485ed5 Mon Sep 17 00:00:00 2001 From: Tom Dupuis <60640908+tomdps@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:20:13 -0700 Subject: [PATCH] fix(python): share import resolution with graph --- AGENTS.md | 2 + CLAUDE.md | 2 + .../src/validation-composition.ts | 5 +- .../python/python.expected.json | 745 +++++++++++++++++- .../python/src/cases/alias_case.py | 3 + .../python/src/cases/conditional_case.py | 6 + .../python/src/cases/docstring_case.py | 8 + .../python/src/cases/multiline_case.py | 6 + .../python/src/cases/namespace_case.py | 3 + .../python/src/cases/package_case.py | 3 + .../python/src/cases/star_case.py | 1 + .../python/src/cases/stub_case.py | 3 + .../python/src/cases/submodule_case.py | 3 + .../python/src/namespace_pkg/tool.py | 1 + .../python/src/pkg/alias_target.py | 1 + .../python/src/pkg/conditional_target.py | 1 + .../source-extraction/python/src/pkg/ghost.py | 1 + .../python/src/pkg/models.py | 5 +- .../python/src/pkg/multiline.py | 1 + .../python/src/pkg/relative_case.py | 3 + .../python/src/pkg/relative_target.py | 1 + .../python/src/pkg/star_target.py | 1 + .../python/src/pkg/submodule.py | 1 + packages/fixtures/src/index.ts | 23 +- packages/graph/README.md | 2 + packages/graph/src/index.ts | 17 + packages/graph/src/node-shims.d.ts | 2 + packages/graph/src/python-import-analysis.ts | 101 +++ packages/opcore/src/repo-validation-policy.ts | 2 + packages/validation-policy/README.md | 2 + packages/validation-policy/src/factory.ts | 24 +- packages/validation-policy/src/types.ts | 3 +- packages/validation-python/README.md | 2 + .../validation-python/src/dead-code-check.ts | 14 +- .../src/graph-requirements.ts | 74 +- .../validation-python/src/import-analysis.ts | 51 ++ .../src/import-graph-check.ts | 15 +- packages/validation-python/src/index.ts | 18 +- .../src/project-config-files.ts | 14 + .../src/project-discovery.ts | 2 +- .../src/project-workspace.ts | 24 +- .../src/relevant-tests-check.ts | 12 +- .../validation-python/src/source-files.ts | 370 ++++----- .../validation-python/src/static-config.ts | 10 +- .../validation-python/src/syntax-check.ts | 1 + packages/validation-python/src/type-check.ts | 21 +- tests/conformance.test.mjs | 16 + tests/fixtures/package-packlists.json | 9 + tests/graph-python-import-analysis.test.mjs | 112 +++ tests/graph-store-conformance.test.mjs | 67 +- tests/installed-bins.test.mjs | 10 +- tests/python-import-composition.test.mjs | 70 ++ tests/validation-python.test.mjs | 352 ++++++++- 53 files changed, 1924 insertions(+), 322 deletions(-) create mode 100644 packages/fixtures/source-extraction/python/src/cases/alias_case.py create mode 100644 packages/fixtures/source-extraction/python/src/cases/conditional_case.py create mode 100644 packages/fixtures/source-extraction/python/src/cases/docstring_case.py create mode 100644 packages/fixtures/source-extraction/python/src/cases/multiline_case.py create mode 100644 packages/fixtures/source-extraction/python/src/cases/namespace_case.py create mode 100644 packages/fixtures/source-extraction/python/src/cases/package_case.py create mode 100644 packages/fixtures/source-extraction/python/src/cases/star_case.py create mode 100644 packages/fixtures/source-extraction/python/src/cases/stub_case.py create mode 100644 packages/fixtures/source-extraction/python/src/cases/submodule_case.py create mode 100644 packages/fixtures/source-extraction/python/src/namespace_pkg/tool.py create mode 100644 packages/fixtures/source-extraction/python/src/pkg/alias_target.py create mode 100644 packages/fixtures/source-extraction/python/src/pkg/conditional_target.py create mode 100644 packages/fixtures/source-extraction/python/src/pkg/ghost.py create mode 100644 packages/fixtures/source-extraction/python/src/pkg/multiline.py create mode 100644 packages/fixtures/source-extraction/python/src/pkg/relative_case.py create mode 100644 packages/fixtures/source-extraction/python/src/pkg/relative_target.py create mode 100644 packages/fixtures/source-extraction/python/src/pkg/star_target.py create mode 100644 packages/fixtures/source-extraction/python/src/pkg/submodule.py create mode 100644 packages/graph/src/python-import-analysis.ts create mode 100644 packages/validation-python/src/import-analysis.ts create mode 100644 packages/validation-python/src/project-config-files.ts create mode 100644 tests/graph-python-import-analysis.test.mjs create mode 100644 tests/python-import-composition.test.mjs diff --git a/AGENTS.md b/AGENTS.md index 6366542..473ee37 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -91,6 +91,7 @@ Opcore is the code-intelligence and robustness monorepo for graph context, edit - #17 Python export metadata is best-effort: `__all__` wins when present, otherwise the leading-underscore convention marks module-level public names; file `exports[]` entries must include policy and supportedSymbol - WHY: Python has no enforced export boundary. - #244 `python.syntax` resolves one concrete interpreter through the Python project resolver, executes that exact interpreter with an isolated versioned JSON compile protocol over sorted `.py`/`.pyi` after-state content, and reports ranged/provenanced compiler diagnostics plus fine-grained outcomes. Parser-success overrides and hand-written grammar heuristics are forbidden; malformed output, nonzero exits, signals, and timeouts fail closed - WHY: syntax truth, target-version truth, and tool provenance must come from the same compiler invocation without mutating the source repo. - #246 makes `@the-open-engine/opcore-validation-python` the sole dynamic owner of `opcore.python.project-context.v1`: every Python target resolves against its nearest project boundary through an injected read/list/exists/realpath workspace view, smol-toml AST config/build metadata, exact interpreter/tool/build probes, and after-state content. Missing realpath evidence is ambiguous, deleted overlays cannot remain discovery markers, and declared constraints never become invented exact versions. Validation, status, scan, init/install preview, metrics, ASP, and installed execution must reuse the resulting project key, context fingerprint, outcome, and provenance; ASP workspace/config reads must remain host-callback-only. Static descriptors advertise only the schema, outcome vocabulary, read-only behavior, and no-install guarantee - WHY: root-scoped and duplicate project/environment discovery validates nested monorepo files with the wrong interpreter and makes surfaces disagree. +- #209 makes Rust graph-core the sole parser/resolver for Python repo imports. `@the-open-engine/opcore-graph` materializes supplied `.py`/`.pyi` after-state files only in an isolated temporary repo and returns canonical directed `IMPORTS_FROM` file edges; validation-python owns only the structural analyzer contract, visible-file enumeration, cached target/transitive closure, and edge consumption. Opcore, advanced validation, validation-policy, and ASP inject the graph adapter; missing/failed/malformed analysis is an infrastructure failure, never empty success - WHY: a second TypeScript import grammar/resolver diverges on multiline syntax, overlays, packages, stubs, namespaces, and src layouts. - #19 requires graph status to preserve real WAL checkpoint evidence from the latest pipeline summary and release gates to fail missing/fabricated WAL evidence - WHY: freshness and checkpoint pressure must remain host-visible provider facts. - #19 treats `opcore graph serve` as the stdio/MCP hot-query replacement, not a Unix socket, with parallel independent serve sessions as the supported concurrency evidence. - #19 keeps current external CRG receipts as non-implementation compatibility evidence only - WHY: old CRG remains a guardrail until downstream cutover issues consume the Opcore proof. @@ -125,6 +126,7 @@ Opcore is the code-intelligence and robustness monorepo for graph context, edit - ALWAYS emit boolean `attributes.exported` on supported TS/JS graph symbol nodes and file-node `attributes.exports[]` for default/re-export/barrel forms - WHY: dead-export metrics must distinguish unsupported export coverage from zero dead exports. - ALWAYS make validation checks read file content through `ValidationCheckContext.fileView` - WHY: validation overlays are hypothetical and checks must see the same before/after file state without mutating disk. - ALWAYS make validation checks consume GraphProvider facts through `ValidationCheckContext.graph` and injected `ValidationGraphProviderClient` sessions - WHY: validation must depend only on public contracts, not graph package internals, CLI execution, or store layouts. +- ALWAYS make Python import-dependent validation consume an injected `PythonImportAnalyzer` backed by graph-core and derive closure from its directed after-state edges - WHY: validation-owned parsing or module candidates create a second semantic owner and cannot represent overlays consistently. - ALWAYS preserve validation diagnostic source ranges, tool provenance, and fine-grained check outcomes across contracts, manifests, sorting/fingerprints, reporting, and ASP mapping when an adapter supplies them - WHY: transport layers must not collapse actionable compiler evidence or turn tool failures into clean passes. - Native repo validation policy lives under `.opcore/config` `validation`; active checks, thresholds, path policy, docs policy, clone policy, TypeScript policy, Rust command gates, and check packs must be parsed and composed through `packages/validation-policy`. - Repo-owned validation extensions live in `.opcore/config` `validation.checks.packs`, resolve from the target repo root, and must export current `ValidationCheckDefinition` objects; Opcore owns loading/registry validation, repos own policy content. diff --git a/CLAUDE.md b/CLAUDE.md index 6366542..473ee37 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -91,6 +91,7 @@ Opcore is the code-intelligence and robustness monorepo for graph context, edit - #17 Python export metadata is best-effort: `__all__` wins when present, otherwise the leading-underscore convention marks module-level public names; file `exports[]` entries must include policy and supportedSymbol - WHY: Python has no enforced export boundary. - #244 `python.syntax` resolves one concrete interpreter through the Python project resolver, executes that exact interpreter with an isolated versioned JSON compile protocol over sorted `.py`/`.pyi` after-state content, and reports ranged/provenanced compiler diagnostics plus fine-grained outcomes. Parser-success overrides and hand-written grammar heuristics are forbidden; malformed output, nonzero exits, signals, and timeouts fail closed - WHY: syntax truth, target-version truth, and tool provenance must come from the same compiler invocation without mutating the source repo. - #246 makes `@the-open-engine/opcore-validation-python` the sole dynamic owner of `opcore.python.project-context.v1`: every Python target resolves against its nearest project boundary through an injected read/list/exists/realpath workspace view, smol-toml AST config/build metadata, exact interpreter/tool/build probes, and after-state content. Missing realpath evidence is ambiguous, deleted overlays cannot remain discovery markers, and declared constraints never become invented exact versions. Validation, status, scan, init/install preview, metrics, ASP, and installed execution must reuse the resulting project key, context fingerprint, outcome, and provenance; ASP workspace/config reads must remain host-callback-only. Static descriptors advertise only the schema, outcome vocabulary, read-only behavior, and no-install guarantee - WHY: root-scoped and duplicate project/environment discovery validates nested monorepo files with the wrong interpreter and makes surfaces disagree. +- #209 makes Rust graph-core the sole parser/resolver for Python repo imports. `@the-open-engine/opcore-graph` materializes supplied `.py`/`.pyi` after-state files only in an isolated temporary repo and returns canonical directed `IMPORTS_FROM` file edges; validation-python owns only the structural analyzer contract, visible-file enumeration, cached target/transitive closure, and edge consumption. Opcore, advanced validation, validation-policy, and ASP inject the graph adapter; missing/failed/malformed analysis is an infrastructure failure, never empty success - WHY: a second TypeScript import grammar/resolver diverges on multiline syntax, overlays, packages, stubs, namespaces, and src layouts. - #19 requires graph status to preserve real WAL checkpoint evidence from the latest pipeline summary and release gates to fail missing/fabricated WAL evidence - WHY: freshness and checkpoint pressure must remain host-visible provider facts. - #19 treats `opcore graph serve` as the stdio/MCP hot-query replacement, not a Unix socket, with parallel independent serve sessions as the supported concurrency evidence. - #19 keeps current external CRG receipts as non-implementation compatibility evidence only - WHY: old CRG remains a guardrail until downstream cutover issues consume the Opcore proof. @@ -125,6 +126,7 @@ Opcore is the code-intelligence and robustness monorepo for graph context, edit - ALWAYS emit boolean `attributes.exported` on supported TS/JS graph symbol nodes and file-node `attributes.exports[]` for default/re-export/barrel forms - WHY: dead-export metrics must distinguish unsupported export coverage from zero dead exports. - ALWAYS make validation checks read file content through `ValidationCheckContext.fileView` - WHY: validation overlays are hypothetical and checks must see the same before/after file state without mutating disk. - ALWAYS make validation checks consume GraphProvider facts through `ValidationCheckContext.graph` and injected `ValidationGraphProviderClient` sessions - WHY: validation must depend only on public contracts, not graph package internals, CLI execution, or store layouts. +- ALWAYS make Python import-dependent validation consume an injected `PythonImportAnalyzer` backed by graph-core and derive closure from its directed after-state edges - WHY: validation-owned parsing or module candidates create a second semantic owner and cannot represent overlays consistently. - ALWAYS preserve validation diagnostic source ranges, tool provenance, and fine-grained check outcomes across contracts, manifests, sorting/fingerprints, reporting, and ASP mapping when an adapter supplies them - WHY: transport layers must not collapse actionable compiler evidence or turn tool failures into clean passes. - Native repo validation policy lives under `.opcore/config` `validation`; active checks, thresholds, path policy, docs policy, clone policy, TypeScript policy, Rust command gates, and check packs must be parsed and composed through `packages/validation-policy`. - Repo-owned validation extensions live in `.opcore/config` `validation.checks.packs`, resolve from the target repo root, and must export current `ValidationCheckDefinition` objects; Opcore owns loading/registry validation, repos own policy content. diff --git a/packages/asp-provider/src/validation-composition.ts b/packages/asp-provider/src/validation-composition.ts index c052b39..f3382a8 100644 --- a/packages/asp-provider/src/validation-composition.ts +++ b/packages/asp-provider/src/validation-composition.ts @@ -37,11 +37,12 @@ import { graphProviderNamedQuery, graphProviderQuery, graphProviderReviewContext, - graphProviderStatus + graphProviderStatus, + graphPythonImportAnalyzer } from "@the-open-engine/opcore-graph"; import type { PythonProjectWorkspace } from "@the-open-engine/opcore-validation-python"; -const aspProviderPolicyOptions = { clone: false } as const; +const aspProviderPolicyOptions = { clone: false, pythonImportAnalyzer: graphPythonImportAnalyzer } as const; export const defaultAspProviderValidationChecks = createBuiltInValidationChecks(undefined, aspProviderPolicyOptions); diff --git a/packages/fixtures/source-extraction/python/python.expected.json b/packages/fixtures/source-extraction/python/python.expected.json index 6f4bea0..9495ecd 100644 --- a/packages/fixtures/source-extraction/python/python.expected.json +++ b/packages/fixtures/source-extraction/python/python.expected.json @@ -20,13 +20,31 @@ "class:src/pkg/base.py#BaseModel", "class:src/pkg/models.py#PublicModel", "class:tests/test_models.py#TestPublicModel", + "file:src/cases/alias_case.py", + "file:src/cases/conditional_case.py", + "file:src/cases/docstring_case.py", + "file:src/cases/multiline_case.py", + "file:src/cases/namespace_case.py", + "file:src/cases/package_case.py", + "file:src/cases/star_case.py", + "file:src/cases/stub_case.py", + "file:src/cases/submodule_case.py", + "file:src/namespace_pkg/tool.py", "file:src/pkg/__init__.py", + "file:src/pkg/alias_target.py", "file:src/pkg/async_tools.py", "file:src/pkg/base.py", "file:src/pkg/broken.py", + "file:src/pkg/conditional_target.py", + "file:src/pkg/ghost.py", "file:src/pkg/helpers.py", "file:src/pkg/models.py", + "file:src/pkg/multiline.py", + "file:src/pkg/relative_case.py", + "file:src/pkg/relative_target.py", + "file:src/pkg/star_target.py", "file:src/pkg/stubs.pyi", + "file:src/pkg/submodule.py", "file:src/pkg/uses_stub.py", "file:tests/test_models.py", "function:src/pkg/async_tools.py#load_name", @@ -41,17 +59,52 @@ "function:src/pkg/uses_stub.py#call_stub", "function:tests/test_models.py#TestPublicModel.test_render", "function:tests/test_models.py#test_make_model", + "module:src/cases/alias_case.py#src.cases.alias_case", + "module:src/cases/conditional_case.py#src.cases.conditional_case", + "module:src/cases/docstring_case.py#src.cases.docstring_case", + "module:src/cases/multiline_case.py#src.cases.multiline_case", + "module:src/cases/namespace_case.py#src.cases.namespace_case", + "module:src/cases/package_case.py#src.cases.package_case", + "module:src/cases/star_case.py#src.cases.star_case", + "module:src/cases/stub_case.py#src.cases.stub_case", + "module:src/cases/submodule_case.py#src.cases.submodule_case", + "module:src/namespace_pkg/tool.py#src.namespace_pkg.tool", "module:src/pkg/__init__.py#src.pkg", + "module:src/pkg/alias_target.py#src.pkg.alias_target", "module:src/pkg/async_tools.py#src.pkg.async_tools", "module:src/pkg/base.py#src.pkg.base", "module:src/pkg/broken.py#src.pkg.broken", + "module:src/pkg/conditional_target.py#src.pkg.conditional_target", + "module:src/pkg/ghost.py#src.pkg.ghost", "module:src/pkg/helpers.py#src.pkg.helpers", "module:src/pkg/models.py#src.pkg.models", + "module:src/pkg/multiline.py#src.pkg.multiline", + "module:src/pkg/relative_case.py#src.pkg.relative_case", + "module:src/pkg/relative_target.py#src.pkg.relative_target", + "module:src/pkg/star_target.py#src.pkg.star_target", "module:src/pkg/stubs.pyi#src.pkg.stubs", + "module:src/pkg/submodule.py#src.pkg.submodule", "module:src/pkg/uses_stub.py#src.pkg.uses_stub", "module:tests/test_models.py#tests.test_models", + "variable:src/cases/alias_case.py#VALUE", + "variable:src/cases/conditional_case.py#VALUE", + "variable:src/cases/docstring_case.py#VALUE", + "variable:src/cases/multiline_case.py#VALUE", + "variable:src/cases/namespace_case.py#VALUE", + "variable:src/cases/package_case.py#VALUE", + "variable:src/cases/stub_case.py#VALUE", + "variable:src/cases/submodule_case.py#VALUE", + "variable:src/namespace_pkg/tool.py#VALUE", "variable:src/pkg/__init__.py#PACKAGE_VALUE", - "variable:src/pkg/models.py#_private" + "variable:src/pkg/alias_target.py#VALUE", + "variable:src/pkg/conditional_target.py#VALUE", + "variable:src/pkg/ghost.py#VALUE", + "variable:src/pkg/models.py#_private", + "variable:src/pkg/multiline.py#VALUE", + "variable:src/pkg/relative_case.py#VALUE", + "variable:src/pkg/relative_target.py#VALUE", + "variable:src/pkg/star_target.py#STAR_VALUE", + "variable:src/pkg/submodule.py#VALUE" ], "edgeTriples": [ [ @@ -104,6 +157,11 @@ "function:tests/test_models.py#TestPublicModel.test_render", "class:src/pkg/models.py#PublicModel" ], + [ + "CALLS", + "variable:src/cases/stub_case.py#VALUE", + "function:src/pkg/stubs.pyi#stubbed" + ], [ "CALLS", "variable:src/pkg/__init__.py#PACKAGE_VALUE", @@ -124,11 +182,66 @@ "class:tests/test_models.py#TestPublicModel", "function:tests/test_models.py#TestPublicModel.test_render" ], + [ + "CONTAINS", + "file:src/cases/alias_case.py", + "module:src/cases/alias_case.py#src.cases.alias_case" + ], + [ + "CONTAINS", + "file:src/cases/conditional_case.py", + "module:src/cases/conditional_case.py#src.cases.conditional_case" + ], + [ + "CONTAINS", + "file:src/cases/docstring_case.py", + "module:src/cases/docstring_case.py#src.cases.docstring_case" + ], + [ + "CONTAINS", + "file:src/cases/multiline_case.py", + "module:src/cases/multiline_case.py#src.cases.multiline_case" + ], + [ + "CONTAINS", + "file:src/cases/namespace_case.py", + "module:src/cases/namespace_case.py#src.cases.namespace_case" + ], + [ + "CONTAINS", + "file:src/cases/package_case.py", + "module:src/cases/package_case.py#src.cases.package_case" + ], + [ + "CONTAINS", + "file:src/cases/star_case.py", + "module:src/cases/star_case.py#src.cases.star_case" + ], + [ + "CONTAINS", + "file:src/cases/stub_case.py", + "module:src/cases/stub_case.py#src.cases.stub_case" + ], + [ + "CONTAINS", + "file:src/cases/submodule_case.py", + "module:src/cases/submodule_case.py#src.cases.submodule_case" + ], + [ + "CONTAINS", + "file:src/namespace_pkg/tool.py", + "module:src/namespace_pkg/tool.py#src.namespace_pkg.tool" + ], [ "CONTAINS", "file:src/pkg/__init__.py", "module:src/pkg/__init__.py#src.pkg" ], + [ + "CONTAINS", + "file:src/pkg/alias_target.py", + "module:src/pkg/alias_target.py#src.pkg.alias_target" + ], [ "CONTAINS", "file:src/pkg/async_tools.py", @@ -144,6 +257,16 @@ "file:src/pkg/broken.py", "module:src/pkg/broken.py#src.pkg.broken" ], + [ + "CONTAINS", + "file:src/pkg/conditional_target.py", + "module:src/pkg/conditional_target.py#src.pkg.conditional_target" + ], + [ + "CONTAINS", + "file:src/pkg/ghost.py", + "module:src/pkg/ghost.py#src.pkg.ghost" + ], [ "CONTAINS", "file:src/pkg/helpers.py", @@ -154,11 +277,36 @@ "file:src/pkg/models.py", "module:src/pkg/models.py#src.pkg.models" ], + [ + "CONTAINS", + "file:src/pkg/multiline.py", + "module:src/pkg/multiline.py#src.pkg.multiline" + ], + [ + "CONTAINS", + "file:src/pkg/relative_case.py", + "module:src/pkg/relative_case.py#src.pkg.relative_case" + ], + [ + "CONTAINS", + "file:src/pkg/relative_target.py", + "module:src/pkg/relative_target.py#src.pkg.relative_target" + ], + [ + "CONTAINS", + "file:src/pkg/star_target.py", + "module:src/pkg/star_target.py#src.pkg.star_target" + ], [ "CONTAINS", "file:src/pkg/stubs.pyi", "module:src/pkg/stubs.pyi#src.pkg.stubs" ], + [ + "CONTAINS", + "file:src/pkg/submodule.py", + "module:src/pkg/submodule.py#src.pkg.submodule" + ], [ "CONTAINS", "file:src/pkg/uses_stub.py", @@ -174,11 +322,61 @@ "function:src/pkg/async_tools.py#load_name", "function:src/pkg/async_tools.py#load_name.inner" ], + [ + "CONTAINS", + "module:src/cases/alias_case.py#src.cases.alias_case", + "variable:src/cases/alias_case.py#VALUE" + ], + [ + "CONTAINS", + "module:src/cases/conditional_case.py#src.cases.conditional_case", + "variable:src/cases/conditional_case.py#VALUE" + ], + [ + "CONTAINS", + "module:src/cases/docstring_case.py#src.cases.docstring_case", + "variable:src/cases/docstring_case.py#VALUE" + ], + [ + "CONTAINS", + "module:src/cases/multiline_case.py#src.cases.multiline_case", + "variable:src/cases/multiline_case.py#VALUE" + ], + [ + "CONTAINS", + "module:src/cases/namespace_case.py#src.cases.namespace_case", + "variable:src/cases/namespace_case.py#VALUE" + ], + [ + "CONTAINS", + "module:src/cases/package_case.py#src.cases.package_case", + "variable:src/cases/package_case.py#VALUE" + ], + [ + "CONTAINS", + "module:src/cases/stub_case.py#src.cases.stub_case", + "variable:src/cases/stub_case.py#VALUE" + ], + [ + "CONTAINS", + "module:src/cases/submodule_case.py#src.cases.submodule_case", + "variable:src/cases/submodule_case.py#VALUE" + ], + [ + "CONTAINS", + "module:src/namespace_pkg/tool.py#src.namespace_pkg.tool", + "variable:src/namespace_pkg/tool.py#VALUE" + ], [ "CONTAINS", "module:src/pkg/__init__.py#src.pkg", "variable:src/pkg/__init__.py#PACKAGE_VALUE" ], + [ + "CONTAINS", + "module:src/pkg/alias_target.py#src.pkg.alias_target", + "variable:src/pkg/alias_target.py#VALUE" + ], [ "CONTAINS", "module:src/pkg/async_tools.py#src.pkg.async_tools", @@ -194,6 +392,16 @@ "module:src/pkg/base.py#src.pkg.base", "class:src/pkg/base.py#BaseModel" ], + [ + "CONTAINS", + "module:src/pkg/conditional_target.py#src.pkg.conditional_target", + "variable:src/pkg/conditional_target.py#VALUE" + ], + [ + "CONTAINS", + "module:src/pkg/ghost.py#src.pkg.ghost", + "variable:src/pkg/ghost.py#VALUE" + ], [ "CONTAINS", "module:src/pkg/helpers.py#src.pkg.helpers", @@ -219,11 +427,36 @@ "module:src/pkg/models.py#src.pkg.models", "variable:src/pkg/models.py#_private" ], + [ + "CONTAINS", + "module:src/pkg/multiline.py#src.pkg.multiline", + "variable:src/pkg/multiline.py#VALUE" + ], + [ + "CONTAINS", + "module:src/pkg/relative_case.py#src.pkg.relative_case", + "variable:src/pkg/relative_case.py#VALUE" + ], + [ + "CONTAINS", + "module:src/pkg/relative_target.py#src.pkg.relative_target", + "variable:src/pkg/relative_target.py#VALUE" + ], + [ + "CONTAINS", + "module:src/pkg/star_target.py#src.pkg.star_target", + "variable:src/pkg/star_target.py#STAR_VALUE" + ], [ "CONTAINS", "module:src/pkg/stubs.pyi#src.pkg.stubs", "function:src/pkg/stubs.pyi#stubbed" ], + [ + "CONTAINS", + "module:src/pkg/submodule.py#src.pkg.submodule", + "variable:src/pkg/submodule.py#VALUE" + ], [ "CONTAINS", "module:src/pkg/uses_stub.py#src.pkg.uses_stub", @@ -239,6 +472,56 @@ "module:tests/test_models.py#tests.test_models", "function:tests/test_models.py#test_make_model" ], + [ + "DEPENDS_ON", + "file:src/cases/alias_case.py", + "file:src/pkg/alias_target.py" + ], + [ + "DEPENDS_ON", + "file:src/cases/conditional_case.py", + "file:src/pkg/conditional_target.py" + ], + [ + "DEPENDS_ON", + "file:src/cases/docstring_case.py", + "file:src/pkg/__init__.py" + ], + [ + "DEPENDS_ON", + "file:src/cases/multiline_case.py", + "file:src/pkg/multiline.py" + ], + [ + "DEPENDS_ON", + "file:src/cases/multiline_case.py", + "file:src/pkg/relative_target.py" + ], + [ + "DEPENDS_ON", + "file:src/cases/namespace_case.py", + "file:src/namespace_pkg/tool.py" + ], + [ + "DEPENDS_ON", + "file:src/cases/package_case.py", + "file:src/pkg/__init__.py" + ], + [ + "DEPENDS_ON", + "file:src/cases/star_case.py", + "file:src/pkg/star_target.py" + ], + [ + "DEPENDS_ON", + "file:src/cases/stub_case.py", + "file:src/pkg/stubs.pyi" + ], + [ + "DEPENDS_ON", + "file:src/cases/submodule_case.py", + "file:src/pkg/submodule.py" + ], [ "DEPENDS_ON", "file:src/pkg/__init__.py", @@ -259,6 +542,11 @@ "file:src/pkg/models.py", "file:src/pkg/helpers.py" ], + [ + "DEPENDS_ON", + "file:src/pkg/relative_case.py", + "file:src/pkg/relative_target.py" + ], [ "DEPENDS_ON", "file:src/pkg/uses_stub.py", @@ -274,6 +562,56 @@ "file:tests/test_models.py", "file:src/pkg/models.py" ], + [ + "IMPORTS_FROM", + "file:src/cases/alias_case.py", + "file:src/pkg/alias_target.py" + ], + [ + "IMPORTS_FROM", + "file:src/cases/conditional_case.py", + "file:src/pkg/conditional_target.py" + ], + [ + "IMPORTS_FROM", + "file:src/cases/docstring_case.py", + "file:src/pkg/__init__.py" + ], + [ + "IMPORTS_FROM", + "file:src/cases/multiline_case.py", + "file:src/pkg/multiline.py" + ], + [ + "IMPORTS_FROM", + "file:src/cases/multiline_case.py", + "file:src/pkg/relative_target.py" + ], + [ + "IMPORTS_FROM", + "file:src/cases/namespace_case.py", + "file:src/namespace_pkg/tool.py" + ], + [ + "IMPORTS_FROM", + "file:src/cases/package_case.py", + "file:src/pkg/__init__.py" + ], + [ + "IMPORTS_FROM", + "file:src/cases/star_case.py", + "file:src/pkg/star_target.py" + ], + [ + "IMPORTS_FROM", + "file:src/cases/stub_case.py", + "file:src/pkg/stubs.pyi" + ], + [ + "IMPORTS_FROM", + "file:src/cases/submodule_case.py", + "file:src/pkg/submodule.py" + ], [ "IMPORTS_FROM", "file:src/pkg/__init__.py", @@ -294,6 +632,11 @@ "file:src/pkg/models.py", "file:src/pkg/helpers.py" ], + [ + "IMPORTS_FROM", + "file:src/pkg/relative_case.py", + "file:src/pkg/relative_target.py" + ], [ "IMPORTS_FROM", "file:src/pkg/uses_stub.py", @@ -330,6 +673,80 @@ "function:tests/test_models.py#test_make_model" ] ], + "pythonImportEdges": [ + { + "fromPath": "src/cases/alias_case.py", + "toPath": "src/pkg/alias_target.py" + }, + { + "fromPath": "src/cases/conditional_case.py", + "toPath": "src/pkg/conditional_target.py" + }, + { + "fromPath": "src/cases/docstring_case.py", + "toPath": "src/pkg/__init__.py" + }, + { + "fromPath": "src/cases/multiline_case.py", + "toPath": "src/pkg/multiline.py" + }, + { + "fromPath": "src/cases/multiline_case.py", + "toPath": "src/pkg/relative_target.py" + }, + { + "fromPath": "src/cases/namespace_case.py", + "toPath": "src/namespace_pkg/tool.py" + }, + { + "fromPath": "src/cases/package_case.py", + "toPath": "src/pkg/__init__.py" + }, + { + "fromPath": "src/cases/star_case.py", + "toPath": "src/pkg/star_target.py" + }, + { + "fromPath": "src/cases/stub_case.py", + "toPath": "src/pkg/stubs.pyi" + }, + { + "fromPath": "src/cases/submodule_case.py", + "toPath": "src/pkg/submodule.py" + }, + { + "fromPath": "src/pkg/__init__.py", + "toPath": "src/pkg/models.py" + }, + { + "fromPath": "src/pkg/async_tools.py", + "toPath": "src/pkg/helpers.py" + }, + { + "fromPath": "src/pkg/models.py", + "toPath": "src/pkg/base.py" + }, + { + "fromPath": "src/pkg/models.py", + "toPath": "src/pkg/helpers.py" + }, + { + "fromPath": "src/pkg/relative_case.py", + "toPath": "src/pkg/relative_target.py" + }, + { + "fromPath": "src/pkg/uses_stub.py", + "toPath": "src/pkg/stubs.pyi" + }, + { + "fromPath": "tests/test_models.py", + "toPath": "src/pkg/__init__.py" + }, + { + "fromPath": "tests/test_models.py", + "toPath": "src/pkg/models.py" + } + ], "nodeAttributes": { "class:src/pkg/base.py#BaseModel": { "decorators": [], @@ -457,9 +874,42 @@ "exported": false, "isTest": true }, + "module:src/cases/alias_case.py#src.cases.alias_case": { + "dottedName": "src.cases.alias_case" + }, + "module:src/cases/conditional_case.py#src.cases.conditional_case": { + "dottedName": "src.cases.conditional_case" + }, + "module:src/cases/docstring_case.py#src.cases.docstring_case": { + "dottedName": "src.cases.docstring_case" + }, + "module:src/cases/multiline_case.py#src.cases.multiline_case": { + "dottedName": "src.cases.multiline_case" + }, + "module:src/cases/namespace_case.py#src.cases.namespace_case": { + "dottedName": "src.cases.namespace_case" + }, + "module:src/cases/package_case.py#src.cases.package_case": { + "dottedName": "src.cases.package_case" + }, + "module:src/cases/star_case.py#src.cases.star_case": { + "dottedName": "src.cases.star_case" + }, + "module:src/cases/stub_case.py#src.cases.stub_case": { + "dottedName": "src.cases.stub_case" + }, + "module:src/cases/submodule_case.py#src.cases.submodule_case": { + "dottedName": "src.cases.submodule_case" + }, + "module:src/namespace_pkg/tool.py#src.namespace_pkg.tool": { + "dottedName": "src.namespace_pkg.tool" + }, "module:src/pkg/__init__.py#src.pkg": { "dottedName": "src.pkg" }, + "module:src/pkg/alias_target.py#src.pkg.alias_target": { + "dottedName": "src.pkg.alias_target" + }, "module:src/pkg/async_tools.py#src.pkg.async_tools": { "dottedName": "src.pkg.async_tools" }, @@ -469,33 +919,246 @@ "module:src/pkg/broken.py#src.pkg.broken": { "dottedName": "src.pkg.broken" }, + "module:src/pkg/conditional_target.py#src.pkg.conditional_target": { + "dottedName": "src.pkg.conditional_target" + }, + "module:src/pkg/ghost.py#src.pkg.ghost": { + "dottedName": "src.pkg.ghost" + }, "module:src/pkg/helpers.py#src.pkg.helpers": { "dottedName": "src.pkg.helpers" }, "module:src/pkg/models.py#src.pkg.models": { "dottedName": "src.pkg.models" }, + "module:src/pkg/multiline.py#src.pkg.multiline": { + "dottedName": "src.pkg.multiline" + }, + "module:src/pkg/relative_case.py#src.pkg.relative_case": { + "dottedName": "src.pkg.relative_case" + }, + "module:src/pkg/relative_target.py#src.pkg.relative_target": { + "dottedName": "src.pkg.relative_target" + }, + "module:src/pkg/star_target.py#src.pkg.star_target": { + "dottedName": "src.pkg.star_target" + }, "module:src/pkg/stubs.pyi#src.pkg.stubs": { "dottedName": "src.pkg.stubs" }, + "module:src/pkg/submodule.py#src.pkg.submodule": { + "dottedName": "src.pkg.submodule" + }, "module:src/pkg/uses_stub.py#src.pkg.uses_stub": { "dottedName": "src.pkg.uses_stub" }, "module:tests/test_models.py#tests.test_models": { "dottedName": "tests.test_models" }, + "variable:src/cases/alias_case.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, + "variable:src/cases/conditional_case.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, + "variable:src/cases/docstring_case.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, + "variable:src/cases/multiline_case.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, + "variable:src/cases/namespace_case.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, + "variable:src/cases/package_case.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, + "variable:src/cases/stub_case.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, + "variable:src/cases/submodule_case.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, + "variable:src/namespace_pkg/tool.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, "variable:src/pkg/__init__.py#PACKAGE_VALUE": { "exportKind": "named", "exportName": "PACKAGE_VALUE", "exportPolicy": "__all__", "exported": true }, + "variable:src/pkg/alias_target.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, + "variable:src/pkg/conditional_target.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, + "variable:src/pkg/ghost.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, "variable:src/pkg/models.py#_private": { "exportPolicy": "__all__", "exported": false + }, + "variable:src/pkg/multiline.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, + "variable:src/pkg/relative_case.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, + "variable:src/pkg/relative_target.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, + "variable:src/pkg/star_target.py#STAR_VALUE": { + "exportKind": "named", + "exportName": "STAR_VALUE", + "exportPolicy": "underscore_convention", + "exported": true + }, + "variable:src/pkg/submodule.py#VALUE": { + "exportKind": "named", + "exportName": "VALUE", + "exportPolicy": "underscore_convention", + "exported": true } }, "fileExports": { + "file:src/cases/alias_case.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], + "file:src/cases/conditional_case.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], + "file:src/cases/docstring_case.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], + "file:src/cases/multiline_case.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], + "file:src/cases/namespace_case.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], + "file:src/cases/package_case.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], + "file:src/cases/stub_case.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], + "file:src/cases/submodule_case.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], + "file:src/namespace_pkg/tool.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], "file:src/pkg/__init__.py": [ { "exported": "PACKAGE_VALUE", @@ -514,6 +1177,16 @@ "supportedSymbol": false } ], + "file:src/pkg/alias_target.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], "file:src/pkg/async_tools.py": [ { "exported": "trace", @@ -542,6 +1215,26 @@ "supportedSymbol": true } ], + "file:src/pkg/conditional_target.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], + "file:src/pkg/ghost.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], "file:src/pkg/helpers.py": [ { "exported": "build_name", @@ -570,6 +1263,46 @@ "supportedSymbol": true } ], + "file:src/pkg/multiline.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], + "file:src/pkg/relative_case.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], + "file:src/pkg/relative_target.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], + "file:src/pkg/star_target.py": [ + { + "exported": "STAR_VALUE", + "kind": "named", + "local": "STAR_VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], "file:src/pkg/stubs.pyi": [ { "exported": "stubbed", @@ -580,6 +1313,16 @@ "supportedSymbol": true } ], + "file:src/pkg/submodule.py": [ + { + "exported": "VALUE", + "kind": "named", + "local": "VALUE", + "policy": "underscore_convention", + "source": null, + "supportedSymbol": true + } + ], "file:src/pkg/uses_stub.py": [ { "exported": "call_stub", diff --git a/packages/fixtures/source-extraction/python/src/cases/alias_case.py b/packages/fixtures/source-extraction/python/src/cases/alias_case.py new file mode 100644 index 0000000..fa234d1 --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/cases/alias_case.py @@ -0,0 +1,3 @@ +import pkg.alias_target as aliased_target + +VALUE = aliased_target.VALUE diff --git a/packages/fixtures/source-extraction/python/src/cases/conditional_case.py b/packages/fixtures/source-extraction/python/src/cases/conditional_case.py new file mode 100644 index 0000000..d5e5e0f --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/cases/conditional_case.py @@ -0,0 +1,6 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pkg import conditional_target + +VALUE = True diff --git a/packages/fixtures/source-extraction/python/src/cases/docstring_case.py b/packages/fixtures/source-extraction/python/src/cases/docstring_case.py new file mode 100644 index 0000000..f072645 --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/cases/docstring_case.py @@ -0,0 +1,8 @@ +""" +This example is documentation, not syntax: +from pkg import ghost +""" + +import pkg + +VALUE = pkg.PACKAGE_VALUE diff --git a/packages/fixtures/source-extraction/python/src/cases/multiline_case.py b/packages/fixtures/source-extraction/python/src/cases/multiline_case.py new file mode 100644 index 0000000..b92e3f6 --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/cases/multiline_case.py @@ -0,0 +1,6 @@ +from pkg import ( + multiline, + relative_target as renamed_target, +) + +VALUE = multiline.VALUE + renamed_target.VALUE diff --git a/packages/fixtures/source-extraction/python/src/cases/namespace_case.py b/packages/fixtures/source-extraction/python/src/cases/namespace_case.py new file mode 100644 index 0000000..29cd242 --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/cases/namespace_case.py @@ -0,0 +1,3 @@ +from namespace_pkg import tool + +VALUE = tool.VALUE diff --git a/packages/fixtures/source-extraction/python/src/cases/package_case.py b/packages/fixtures/source-extraction/python/src/cases/package_case.py new file mode 100644 index 0000000..6754f11 --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/cases/package_case.py @@ -0,0 +1,3 @@ +import pkg + +VALUE = pkg.PACKAGE_VALUE diff --git a/packages/fixtures/source-extraction/python/src/cases/star_case.py b/packages/fixtures/source-extraction/python/src/cases/star_case.py new file mode 100644 index 0000000..f8c1d4a --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/cases/star_case.py @@ -0,0 +1 @@ +from pkg.star_target import * diff --git a/packages/fixtures/source-extraction/python/src/cases/stub_case.py b/packages/fixtures/source-extraction/python/src/cases/stub_case.py new file mode 100644 index 0000000..2f4efcf --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/cases/stub_case.py @@ -0,0 +1,3 @@ +from pkg import stubs + +VALUE = stubs.stubbed() diff --git a/packages/fixtures/source-extraction/python/src/cases/submodule_case.py b/packages/fixtures/source-extraction/python/src/cases/submodule_case.py new file mode 100644 index 0000000..e62ec9c --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/cases/submodule_case.py @@ -0,0 +1,3 @@ +from pkg import submodule + +VALUE = submodule.VALUE diff --git a/packages/fixtures/source-extraction/python/src/namespace_pkg/tool.py b/packages/fixtures/source-extraction/python/src/namespace_pkg/tool.py new file mode 100644 index 0000000..b15b1b0 --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/namespace_pkg/tool.py @@ -0,0 +1 @@ +VALUE = 1 diff --git a/packages/fixtures/source-extraction/python/src/pkg/alias_target.py b/packages/fixtures/source-extraction/python/src/pkg/alias_target.py new file mode 100644 index 0000000..b15b1b0 --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/pkg/alias_target.py @@ -0,0 +1 @@ +VALUE = 1 diff --git a/packages/fixtures/source-extraction/python/src/pkg/conditional_target.py b/packages/fixtures/source-extraction/python/src/pkg/conditional_target.py new file mode 100644 index 0000000..b15b1b0 --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/pkg/conditional_target.py @@ -0,0 +1 @@ +VALUE = 1 diff --git a/packages/fixtures/source-extraction/python/src/pkg/ghost.py b/packages/fixtures/source-extraction/python/src/pkg/ghost.py new file mode 100644 index 0000000..b15b1b0 --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/pkg/ghost.py @@ -0,0 +1 @@ +VALUE = 1 diff --git a/packages/fixtures/source-extraction/python/src/pkg/models.py b/packages/fixtures/source-extraction/python/src/pkg/models.py index eb133f1..2224b6b 100644 --- a/packages/fixtures/source-extraction/python/src/pkg/models.py +++ b/packages/fixtures/source-extraction/python/src/pkg/models.py @@ -1,6 +1,9 @@ +import sys + from .base import BaseModel from .helpers import build_name -from .missing import MissingLocal +if sys.version_info < (0, 0): + from .missing import MissingLocal _private = 1 __all__ = ["PublicModel", "make_model"] diff --git a/packages/fixtures/source-extraction/python/src/pkg/multiline.py b/packages/fixtures/source-extraction/python/src/pkg/multiline.py new file mode 100644 index 0000000..b15b1b0 --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/pkg/multiline.py @@ -0,0 +1 @@ +VALUE = 1 diff --git a/packages/fixtures/source-extraction/python/src/pkg/relative_case.py b/packages/fixtures/source-extraction/python/src/pkg/relative_case.py new file mode 100644 index 0000000..64e7691 --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/pkg/relative_case.py @@ -0,0 +1,3 @@ +from . import relative_target + +VALUE = relative_target.VALUE diff --git a/packages/fixtures/source-extraction/python/src/pkg/relative_target.py b/packages/fixtures/source-extraction/python/src/pkg/relative_target.py new file mode 100644 index 0000000..b15b1b0 --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/pkg/relative_target.py @@ -0,0 +1 @@ +VALUE = 1 diff --git a/packages/fixtures/source-extraction/python/src/pkg/star_target.py b/packages/fixtures/source-extraction/python/src/pkg/star_target.py new file mode 100644 index 0000000..a50cc06 --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/pkg/star_target.py @@ -0,0 +1 @@ +STAR_VALUE = 1 diff --git a/packages/fixtures/source-extraction/python/src/pkg/submodule.py b/packages/fixtures/source-extraction/python/src/pkg/submodule.py new file mode 100644 index 0000000..b15b1b0 --- /dev/null +++ b/packages/fixtures/source-extraction/python/src/pkg/submodule.py @@ -0,0 +1 @@ +VALUE = 1 diff --git a/packages/fixtures/src/index.ts b/packages/fixtures/src/index.ts index 645ae94..eba1d69 100644 --- a/packages/fixtures/src/index.ts +++ b/packages/fixtures/src/index.ts @@ -166,6 +166,11 @@ export interface SyntheticFixtureMetadata { nodeKinds: readonly string[]; edgeKinds: readonly string[]; diagnostics: readonly string[]; + importResolution?: { + owner: "graph-core"; + expectedEdgeField: "pythonImportEdges"; + cases: readonly string[]; + }; }; graphPipeline?: { operations: readonly string[]; @@ -568,7 +573,23 @@ export const conformanceFixtureMetadata = [ languages: ["py", "pyi"], nodeKinds: ["File", "Module", "Class", "Function", "Variable"], edgeKinds: ["CONTAINS", "IMPORTS_FROM", "DEPENDS_ON", "CALLS", "TESTED_BY", "INHERITS"], - diagnostics: ["parse_error", "unresolved_import"] + diagnostics: ["parse_error", "unresolved_import"], + importResolution: { + owner: "graph-core", + expectedEdgeField: "pythonImportEdges", + cases: [ + "multiline-parenthesized", + "alias", + "conditional", + "submodule", + "star", + "relative", + "package-initializer", + "stub", + "namespace", + "src-layout" + ] + } } }, { diff --git a/packages/graph/README.md b/packages/graph/README.md index 58388e6..47f00c6 100644 --- a/packages/graph/README.md +++ b/packages/graph/README.md @@ -1,3 +1,5 @@ # @the-open-engine/opcore-graph Opcore graph provider package for repository graph extraction, query, search, and impact surfaces. + +The package also owns canonical Python repo-import analysis. `analyzePythonImports` accepts only supplied `.py`/`.pyi` after-state files, materializes them in an isolated temporary repository, runs the installed graph-core build/query path, returns sorted directed `IMPORTS_FROM` file edges, and always removes temporary state. It never reads or writes the target worktree. diff --git a/packages/graph/src/index.ts b/packages/graph/src/index.ts index c83b174..dca0cb3 100644 --- a/packages/graph/src/index.ts +++ b/packages/graph/src/index.ts @@ -20,6 +20,11 @@ import type { } from "@the-open-engine/opcore-contracts"; import { createCommandRouterResult, graphNamedQueryKinds } from "@the-open-engine/opcore-contracts"; import { providerFailureStatus } from "./artifact.js"; +import { + analyzePythonImportsWithGraph, + type PythonImportAnalysisEdge, + type PythonImportAnalysisFile +} from "./python-import-analysis.js"; import { graphServeRouterResult, isServeTransportArgv, runGraphServeCli } from "./serve.js"; import { invokeGraphCoreSidecar } from "./sidecar.js"; @@ -41,10 +46,22 @@ export { export { invokeGraphCoreSidecar } from "./sidecar.js"; export { graphServeRouterResult, isServeTransportArgv, runGraphServeCli } from "./serve.js"; export type { GraphServeFrameTimingEvent, GraphServeTelemetry } from "./serve.js"; +export type { PythonImportAnalysisEdge, PythonImportAnalysisFile } from "./python-import-analysis.js"; export const graphProviderName = "opcore-graph"; export const graphProviderSchemaVersion = 1; +export function analyzePythonImports( + files: readonly PythonImportAnalysisFile[] +): Promise { + return analyzePythonImportsWithGraph(files, { + build: (repo) => graphProviderBuild(repo), + query: (repo) => graphProviderQuery(repo, { kind: "edges", edgeKinds: ["IMPORTS_FROM"] }) + }); +} + +export const graphPythonImportAnalyzer = { analyze: analyzePythonImports } as const; + export function graphProviderStatus( repo: RepoIdentity | string = { repoRoot: process.cwd() }, options: { paths?: readonly string[]; watchPaths?: readonly string[] } = {} diff --git a/packages/graph/src/node-shims.d.ts b/packages/graph/src/node-shims.d.ts index 4d0a98b..3216679 100644 --- a/packages/graph/src/node-shims.d.ts +++ b/packages/graph/src/node-shims.d.ts @@ -62,6 +62,7 @@ declare module "node:crypto" { declare module "node:fs" { export function closeSync(fd: number): void; export function existsSync(path: string): boolean; + export const mkdirSync: (path: string, options?: { recursive?: boolean }) => void; export function mkdtempSync(prefix: string): string; export function openSync(path: string, flags: string): number; export function readFileSync(path: string): Uint8Array; @@ -69,6 +70,7 @@ declare module "node:fs" { export function readSync(fd: number, buffer: Uint8Array, offset: number, length: number, position: number): number; export function rmSync(path: string, options?: { recursive?: boolean; force?: boolean }): void; export function statSync(path: string): { size: number; isFile: () => boolean }; + export const writeFileSync: (path: string, content: string) => void; } declare module "node:os" { diff --git a/packages/graph/src/python-import-analysis.ts b/packages/graph/src/python-import-analysis.ts new file mode 100644 index 0000000..0aaf643 --- /dev/null +++ b/packages/graph/src/python-import-analysis.ts @@ -0,0 +1,101 @@ +import type { GraphFactEdge, GraphFactQueryResult, GraphPipelineResult, RepoIdentity } from "@the-open-engine/opcore-contracts"; +import { validateRepoRelativePath } from "@the-open-engine/opcore-contracts"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; + +export interface PythonImportAnalysisFile { + path: string; + content: string; +} + +export interface PythonImportAnalysisEdge { + fromPath: string; + toPath: string; +} + +export interface PythonImportAnalysisGraph { + build(repo: RepoIdentity): GraphPipelineResult; + query(repo: RepoIdentity): GraphFactQueryResult; +} + +export async function analyzePythonImportsWithGraph( + files: readonly PythonImportAnalysisFile[], + graph: PythonImportAnalysisGraph +): Promise { + const normalizedFiles = normalizeSuppliedFiles(files); + const tempRoot = mkdtempSync(join(tmpdir(), "opcore-python-import-analysis-")); + const repoRoot = join(tempRoot, "repo"); + try { + mkdirSync(repoRoot, { recursive: true }); + for (const file of normalizedFiles) writeSuppliedFile(repoRoot, file); + const repo = { repoRoot }; + const build = graph.build(repo); + if (build.status.state !== "available") throw graphAnalysisFailure("build", build.status); + const query = graph.query(repo); + if (query.status.state !== "available" || !("edges" in query)) throw graphAnalysisFailure("query", query.status); + return importEdges(query.edges, new Set(normalizedFiles.map((file) => file.path))); + } finally { + rmSync(tempRoot, { recursive: true, force: true }); + } +} + +function normalizeSuppliedFiles(files: readonly PythonImportAnalysisFile[]): readonly PythonImportAnalysisFile[] { + if (!Array.isArray(files)) throw new Error("Python import analysis files must be an array"); + const byPath = new Map(); + for (const [index, candidate] of files.entries()) { + if (candidate === null || typeof candidate !== "object") { + throw new Error(`Python import analysis file at index ${index} is malformed`); + } + if (typeof candidate.path !== "string" || typeof candidate.content !== "string") { + throw new Error(`Python import analysis file at index ${index} requires string path and content`); + } + const path = validateRepoRelativePath(candidate.path); + if (!isPythonSourcePath(path)) throw new Error(`Python import analysis accepts only .py/.pyi files: ${path}`); + if (byPath.has(path)) throw new Error(`Python import analysis received duplicate path: ${path}`); + byPath.set(path, { path, content: candidate.content }); + } + return [...byPath.values()].sort((left, right) => left.path.localeCompare(right.path)); +} + +function writeSuppliedFile(repoRoot: string, file: PythonImportAnalysisFile): void { + const absolutePath = resolveRepoPath(repoRoot, file.path); + mkdirSync(dirname(absolutePath), { recursive: true }); + writeFileSync(absolutePath, file.content); +} + +function resolveRepoPath(repoRoot: string, path: string): string { + return resolve(repoRoot, validateRepoRelativePath(path)); +} + +function importEdges( + edges: readonly GraphFactEdge[], + suppliedPaths: ReadonlySet +): readonly PythonImportAnalysisEdge[] { + const byKey = new Map(); + for (const edge of edges) { + if (edge.kind !== "IMPORTS_FROM") continue; + const fromPath = fileEndpointPath(edge.from); + const toPath = fileEndpointPath(edge.to); + if (!suppliedPaths.has(fromPath) || !suppliedPaths.has(toPath)) { + throw new Error(`Graph-core Python import analysis returned an edge outside supplied files: ${fromPath} -> ${toPath}`); + } + byKey.set(`${fromPath}\0${toPath}`, { fromPath, toPath }); + } + return [...byKey.values()].sort((left, right) => + `${left.fromPath}\0${left.toPath}`.localeCompare(`${right.fromPath}\0${right.toPath}`) + ); +} + +function fileEndpointPath(endpoint: string): string { + if (!endpoint.startsWith("file:")) throw new Error(`Graph-core Python import analysis returned a non-file endpoint: ${endpoint}`); + return validateRepoRelativePath(endpoint.slice("file:".length)); +} + +function graphAnalysisFailure(operation: "build" | "query", status: { state: string; message?: string }): Error { + return new Error(`Graph-core Python import analysis ${operation} failed (${status.state}): ${status.message ?? "no failure message"}`); +} + +function isPythonSourcePath(path: string): boolean { + return path.endsWith(".py") || path.endsWith(".pyi"); +} diff --git a/packages/opcore/src/repo-validation-policy.ts b/packages/opcore/src/repo-validation-policy.ts index d733c3b..ee2622b 100644 --- a/packages/opcore/src/repo-validation-policy.ts +++ b/packages/opcore/src/repo-validation-policy.ts @@ -3,10 +3,12 @@ import { validationChecksForRepoPolicy as validationChecksForSharedRepoPolicy, validationChecksForRepoPolicyAndCoverage as validationChecksForSharedRepoPolicyAndCoverage } from "@the-open-engine/opcore-validation-policy"; +import { graphPythonImportAnalyzer } from "@the-open-engine/opcore-graph"; import { invokeCloneAnalysis } from "./clone-invoker.js"; import { createNodePythonProjectWorkspace } from "@the-open-engine/opcore-validation-python"; const opcoreValidationPolicyOptions = { + pythonImportAnalyzer: graphPythonImportAnalyzer, clone: { invoke: invokeCloneAnalysis } diff --git a/packages/validation-policy/README.md b/packages/validation-policy/README.md index 94d9955..1a1a5a1 100644 --- a/packages/validation-policy/README.md +++ b/packages/validation-policy/README.md @@ -1,3 +1,5 @@ # Opcore Validation Policy Shared repo validation policy parsing and check composition for Opcore-owned validation entrypoints. + +Python import-dependent checks require `pythonImportAnalyzer` composition. Opcore and ASP inject the graph package adapter; validation-policy forwards the structural interface without importing graph implementation internals. diff --git a/packages/validation-policy/src/factory.ts b/packages/validation-policy/src/factory.ts index d8f2fc1..ac1ae36 100644 --- a/packages/validation-policy/src/factory.ts +++ b/packages/validation-policy/src/factory.ts @@ -52,9 +52,10 @@ export function createBuiltInValidationChecks( deadCode: checksConfig?.typescript?.deadCode }), ...createRustValidationChecks(rustValidationOptions(config)), - ...createPythonValidationChecks( - pythonWorkspace === undefined ? {} : { nodeWorkspace: pythonWorkspace } - ), + ...createPythonValidationChecks({ + ...(pythonWorkspace === undefined ? {} : { nodeWorkspace: pythonWorkspace }), + ...(options.pythonImportAnalyzer === undefined ? {} : { importAnalyzer: options.pythonImportAnalyzer }) + }), ...createDocsValidationChecks(docsValidationOptions(checksConfig?.docs)), ...cloneChecks(checksConfig?.clone, options) ]; @@ -80,11 +81,12 @@ function validationChecksForRepoConfig( const adapters = config.validation.adapters === undefined ? undefined : new Set(config.validation.adapters); const disabled = new Set(configuredDisabled); const defaults = new Set(configuredDefaults); + const filteredContexts = new WeakMap[0]>(); const checks = available .filter((check) => adapters === undefined || adapters.has(check.adapter)) .filter((check) => !disabled.has(check.id)) .map((check) => applyDefaultScopePolicy(check, defaults)) - .map((check) => applyPathPolicy(check, config.validation.pathPolicy)); + .map((check) => applyPathPolicy(check, config.validation.pathPolicy, filteredContexts)); createValidationCheckRegistry(checks); return checks; } @@ -151,13 +153,21 @@ function applyDefaultScopePolicy(check: ValidationCheckDefinition, defaults: Rea function applyPathPolicy( check: ValidationCheckDefinition, - pathPolicy: OpcorePathPolicy | undefined + pathPolicy: OpcorePathPolicy | undefined, + filteredContexts: WeakMap[0]> ): ValidationCheckDefinition { if (pathPolicy === undefined) return check; - const run: ValidationCheckDefinition["run"] = (context) => check.run(withFilteredFileView(context, pathPolicy)); + const filteredContext = (context: Parameters[0]) => { + const existing = filteredContexts.get(context.fileView); + if (existing !== undefined) return existing; + const filtered = withFilteredFileView(context, pathPolicy); + filteredContexts.set(context.fileView, filtered); + return filtered; + }; + const run: ValidationCheckDefinition["run"] = (context) => check.run(filteredContext(context)); if (check.graphRequirements === undefined) return { ...check, run }; const graphRequirements: NonNullable = (context) => - check.graphRequirements?.(withFilteredFileView(context, pathPolicy)) ?? []; + check.graphRequirements?.(filteredContext(context)) ?? []; return { ...check, graphRequirements, run }; } diff --git a/packages/validation-policy/src/types.ts b/packages/validation-policy/src/types.ts index 8bcef84..b9d8f7b 100644 --- a/packages/validation-policy/src/types.ts +++ b/packages/validation-policy/src/types.ts @@ -1,5 +1,5 @@ import type { ValidationCheckDefinition } from "@the-open-engine/opcore-validation"; -import type { PythonProjectWorkspace } from "@the-open-engine/opcore-validation-python"; +import type { PythonImportAnalyzer, PythonProjectWorkspace } from "@the-open-engine/opcore-validation-python"; import type { CloneNativeInvoker } from "@the-open-engine/opcore-validation-clone"; export const configPath = ".opcore/config"; @@ -114,6 +114,7 @@ export interface OpcoreCheckPack { export interface OpcoreRepoValidationPolicyOptions { pythonWorkspace?: PythonProjectWorkspace; + pythonImportAnalyzer?: PythonImportAnalyzer; clone?: false | { invoke?: CloneNativeInvoker["invoke"]; }; diff --git a/packages/validation-python/README.md b/packages/validation-python/README.md index 243e269..108f0dd 100644 --- a/packages/validation-python/README.md +++ b/packages/validation-python/README.md @@ -6,6 +6,8 @@ This package owns the canonical, read-only `opcore.python.project-context.v1` re Consumers must inject a workspace view with read/list/exists/realpath operations. Missing realpath evidence is an ambiguous context, not an assumed non-symlink. Validation uses `ValidationFileView` after-state adapters so written and deleted config/source overlays share one identity; ASP uses host workspace callbacks only. Node status and installed-artifact surfaces use the read-only Node adapter. TOML decisions consume the parsed `smol-toml` AST, static build metadata is retained, and build availability is probed independently. All consumers reuse the resolver result instead of scanning project/config/environment filenames independently. +Python import grammar and repo-module resolution belong exclusively to Rust graph-core. This package exposes only the structural `PythonImportAnalyzer` injection contract. Import-dependent checks enumerate every visible `.py`/`.pyi` after-state file through `ValidationFileView`, invoke the required analyzer once per file view, and derive target/transitive source closure from its directed edges. Missing, failed, or malformed analysis fails as infrastructure failure; validation never parses Python imports or treats unavailable analysis as an empty graph. + Precedence is nearest project, explicit per-tool override, compatible active environment, project-local `.venv`/`venv`/`env`, safe installed manager evidence, then PATH. Conflicting same-root manager or target evidence is ambiguous. Missing tools and probe timeout/signal/spawn/exit/malformed-output failures remain typed and visible; unresolved required contexts cannot produce a clean Python check. Run `npm run python:resolver-matrix -- --json` for checked fixture evidence covering real POSIX execution and explicitly simulated POSIX/Windows layouts. diff --git a/packages/validation-python/src/dead-code-check.ts b/packages/validation-python/src/dead-code-check.ts index da2d478..fb976ea 100644 --- a/packages/validation-python/src/dead-code-check.ts +++ b/packages/validation-python/src/dead-code-check.ts @@ -10,10 +10,10 @@ import { } from "@the-open-engine/opcore-validation"; import { PYTHON_DEAD_CODE_CHECK_ID } from "./check-ids.js"; import { pythonCheckAdapter, pythonCheckOwner, supportedPythonValidationScopes } from "./check-constants.js"; -import { deadCodeGraphRequirements } from "./graph-requirements.js"; -import { materializePythonSources } from "./source-files.js"; +import { createDeadCodeGraphRequirements } from "./graph-requirements.js"; +import type { PythonSourceRootResolver } from "./source-files.js"; -export function createDeadCodeCheck(): ValidationCheckDefinition { +export function createDeadCodeCheck(resolveRoots: PythonSourceRootResolver): ValidationCheckDefinition { return { id: PYTHON_DEAD_CODE_CHECK_ID, owner: pythonCheckOwner, @@ -21,15 +21,15 @@ export function createDeadCodeCheck(): ValidationCheckDefinition { defaultSeverity: "warning", supportedScopes: supportedPythonValidationScopes, requiresGraph: true, - graphRequirements: deadCodeGraphRequirements, + graphRequirements: createDeadCodeGraphRequirements(resolveRoots), run: async (context) => { - const sourceSet = await materializePythonSources(context); + const rootPaths = await resolveRoots(context); const [symbolFacts, calls, fileNodes] = await Promise.all([ context.graph.facts({ kind: "symbols" }), context.graph.calls(), - context.graph.fileNodes(sourceSet.rootPaths) + context.graph.fileNodes(rootPaths) ]); - const scopedPaths = new Set(sourceSet.rootPaths); + const scopedPaths = new Set(rootPaths); const scopedSymbols = symbolFacts.nodes.filter((node) => isScopedSymbol(node, scopedPaths)); const unsupportedFileExports = fileNodes.flatMap(graphFactUnsupportedFileExportMetadata); if (!scopedSymbols.some(graphFactHasExportMetadata) && unsupportedFileExports.length === 0) { diff --git a/packages/validation-python/src/graph-requirements.ts b/packages/validation-python/src/graph-requirements.ts index 62441ee..ebb560a 100644 --- a/packages/validation-python/src/graph-requirements.ts +++ b/packages/validation-python/src/graph-requirements.ts @@ -1,53 +1,51 @@ import type { GraphEdgeKind } from "@the-open-engine/opcore-contracts"; import type { ValidationCheckContext, ValidationGraphQueryRequirement } from "@the-open-engine/opcore-validation"; -import { materializePythonSources, toFileNodeId } from "./source-files.js"; +import { toFileNodeId, type PythonSourceRootResolver, type PythonSourceSetResolver } from "./source-files.js"; -export async function importGraphRequirements( - context: ValidationCheckContext -): Promise { - return edgeAndScopedFileRequirements(context, ["IMPORTS_FROM"]); +export function createImportGraphRequirements(resolveSources: PythonSourceSetResolver) { + return async (context: ValidationCheckContext): Promise => + edgeAndScopedFileRequirements(context, ["IMPORTS_FROM"], (await resolveSources(context)).rootPaths); } -export async function deadCodeGraphRequirements( - context: ValidationCheckContext -): Promise { - const sourceSet = await materializePythonSources(context); - return [ - { - operation: "factQuery", - selector: { - kind: "edges", - edgeKinds: ["CALLS"] - } - }, - { - operation: "factQuery", - selector: { - kind: "nodes", - nodeKinds: ["File", "file"], - ids: sourceSet.rootPaths.map(toFileNodeId) +export function createDeadCodeGraphRequirements(resolveRoots: PythonSourceRootResolver) { + return async (context: ValidationCheckContext): Promise => { + const rootPaths = await resolveRoots(context); + return [ + { + operation: "factQuery", + selector: { + kind: "edges", + edgeKinds: ["CALLS"] + } + }, + { + operation: "factQuery", + selector: { + kind: "nodes", + nodeKinds: ["File", "file"], + ids: rootPaths.map(toFileNodeId) + } + }, + { + operation: "factQuery", + selector: { + kind: "symbols" + } } - }, - { - operation: "factQuery", - selector: { - kind: "symbols" - } - } - ]; + ]; + }; } -export async function relevantTestsGraphRequirements( - context: ValidationCheckContext -): Promise { - return edgeAndScopedFileRequirements(context, ["TESTED_BY"]); +export function createRelevantTestsGraphRequirements(resolveRoots: PythonSourceRootResolver) { + return async (context: ValidationCheckContext): Promise => + edgeAndScopedFileRequirements(context, ["TESTED_BY"], await resolveRoots(context)); } async function edgeAndScopedFileRequirements( context: ValidationCheckContext, - edgeKinds: readonly GraphEdgeKind[] + edgeKinds: readonly GraphEdgeKind[], + rootPaths: readonly string[] ): Promise { - const sourceSet = await materializePythonSources(context); return [ { operation: "factQuery", @@ -61,7 +59,7 @@ async function edgeAndScopedFileRequirements( selector: { kind: "nodes", nodeKinds: ["File", "file"], - ids: sourceSet.rootPaths.map(toFileNodeId) + ids: rootPaths.map(toFileNodeId) } } ]; diff --git a/packages/validation-python/src/import-analysis.ts b/packages/validation-python/src/import-analysis.ts new file mode 100644 index 0000000..9b3d61c --- /dev/null +++ b/packages/validation-python/src/import-analysis.ts @@ -0,0 +1,51 @@ +import { normalizeValidationFileViewPath } from "@the-open-engine/opcore-validation"; + +export interface PythonImportSourceFile { + path: string; + content: string; +} + +export interface PythonImportEdge { + fromPath: string; + toPath: string; +} + +export interface PythonImportAnalyzer { + analyze(files: readonly PythonImportSourceFile[]): Promise; +} + +export function requirePythonImportAnalyzer(analyzer: PythonImportAnalyzer | undefined): PythonImportAnalyzer { + if (analyzer === undefined || typeof analyzer.analyze !== "function") { + throw new Error("A canonical Python import analyzer is required for Python import-dependent validation"); + } + return analyzer; +} + +export function validatePythonImportEdges( + value: unknown, + sourcePaths: ReadonlySet +): readonly PythonImportEdge[] { + if (!Array.isArray(value)) throw new Error("Python import analyzer returned malformed edges: expected an array"); + const byKey = new Map(); + for (const [index, candidate] of value.entries()) { + if (candidate === null || typeof candidate !== "object") { + throw new Error(`Python import analyzer returned malformed edge at index ${index}`); + } + const fromValue = Reflect.get(candidate, "fromPath"); + const toValue = Reflect.get(candidate, "toPath"); + if (typeof fromValue !== "string" || typeof toValue !== "string") { + throw new Error(`Python import analyzer returned malformed edge endpoints at index ${index}`); + } + const fromPath = normalizeValidationFileViewPath(fromValue); + const toPath = normalizeValidationFileViewPath(toValue); + if (!sourcePaths.has(fromPath) || !sourcePaths.has(toPath)) { + throw new Error(`Python import analyzer returned an edge outside the supplied after-state: ${fromPath} -> ${toPath}`); + } + byKey.set(`${fromPath}\0${toPath}`, { fromPath, toPath }); + } + return [...byKey.values()].sort(comparePythonImportEdges); +} + +function comparePythonImportEdges(left: PythonImportEdge, right: PythonImportEdge): number { + return `${left.fromPath}\0${left.toPath}`.localeCompare(`${right.fromPath}\0${right.toPath}`); +} diff --git a/packages/validation-python/src/import-graph-check.ts b/packages/validation-python/src/import-graph-check.ts index aa04625..ea04931 100644 --- a/packages/validation-python/src/import-graph-check.ts +++ b/packages/validation-python/src/import-graph-check.ts @@ -2,10 +2,10 @@ import type { GraphFactEdge, ValidationDiagnostic } from "@the-open-engine/opcor import type { ValidationCheckDefinition } from "@the-open-engine/opcore-validation"; import { PYTHON_IMPORT_GRAPH_CHECK_ID } from "./check-ids.js"; import { pythonCheckAdapter, pythonCheckOwner, supportedPythonValidationScopes } from "./check-constants.js"; -import { importGraphRequirements } from "./graph-requirements.js"; -import { materializePythonSources, toFileNodeId, type PythonProjectContextResolver } from "./source-files.js"; +import { createImportGraphRequirements } from "./graph-requirements.js"; +import { toFileNodeId, type PythonSourceSetResolver } from "./source-files.js"; -export function createImportGraphCheck(resolveContexts?: PythonProjectContextResolver): ValidationCheckDefinition { +export function createImportGraphCheck(resolveSources: PythonSourceSetResolver): ValidationCheckDefinition { return { id: PYTHON_IMPORT_GRAPH_CHECK_ID, owner: pythonCheckOwner, @@ -13,18 +13,17 @@ export function createImportGraphCheck(resolveContexts?: PythonProjectContextRes defaultSeverity: "warning", supportedScopes: supportedPythonValidationScopes, requiresGraph: true, - graphRequirements: importGraphRequirements, + graphRequirements: createImportGraphRequirements(resolveSources), run: async (context) => { - const [contexts, edges] = await Promise.all([resolveContexts?.(context) ?? [], context.graph.importsFrom()]); - const sourceSet = await materializePythonSources(context, contexts); + const [sourceSet, edges] = await Promise.all([resolveSources(context), context.graph.importsFrom()]); const diagnostics = sourceSet.repoImports - .filter((repoImport) => !edges.some((edge) => matchesDirectedFileEdge(edge, repoImport.fromPath, repoImport.resolvedPath))) + .filter((repoImport) => !edges.some((edge) => matchesDirectedFileEdge(edge, repoImport.fromPath, repoImport.toPath))) .map((repoImport): ValidationDiagnostic => ({ category: "graph", severity: "warning", path: repoImport.fromPath, code: "PY_IMPORT_GRAPH_MISSING_EDGE", - message: `Missing IMPORTS_FROM graph edge for ${repoImport.fromPath} -> ${repoImport.resolvedPath}` + message: `Missing IMPORTS_FROM graph edge for ${repoImport.fromPath} -> ${repoImport.toPath}` })); return { diagnostics }; } diff --git a/packages/validation-python/src/index.ts b/packages/validation-python/src/index.ts index 69dfff6..1ff1cbc 100644 --- a/packages/validation-python/src/index.ts +++ b/packages/validation-python/src/index.ts @@ -5,8 +5,9 @@ import { createImportGraphCheck } from "./import-graph-check.js"; import { createRelevantTestsCheck } from "./relevant-tests-check.js"; import { createSourceHygieneCheck } from "./source-hygiene-check.js"; import { createSyntaxCheck, type PythonSyntaxCheckOptions } from "./syntax-check.js"; -import { createPythonProjectContextResolver, pythonInputSet } from "./source-files.js"; +import { createPythonProjectContextResolver, createPythonSourceRootResolver, createPythonSourceSetResolver, pythonInputSet } from "./source-files.js"; import { createTypeCheck, type PythonTypeCheckOptions } from "./type-check.js"; +import type { PythonImportAnalyzer } from "./import-analysis.js"; export { PYTHON_DEAD_CODE_CHECK_ID, @@ -20,6 +21,7 @@ export { } from "./check-ids.js"; export { validationPythonAdapterName } from "./check-constants.js"; export { isPythonSourcePath } from "./source-files.js"; +export type { PythonImportAnalyzer, PythonImportEdge, PythonImportSourceFile } from "./import-analysis.js"; export { resolvePythonProjectContext, resolvePythonProjectContexts, @@ -36,7 +38,9 @@ export { createPythonValidationAdapterStatus, type PythonValidationToolchainOpti export { createSyntaxCheck, type PythonSyntaxCheckOptions } from "./syntax-check.js"; export { createTypeCheck, type PythonTypeCheckOptions } from "./type-check.js"; -export interface CreatePythonValidationChecksOptions extends PythonTypeCheckOptions, PythonSyntaxCheckOptions {} +export interface CreatePythonValidationChecksOptions extends PythonTypeCheckOptions, PythonSyntaxCheckOptions { + importAnalyzer?: PythonImportAnalyzer; +} export function createPythonValidationChecks( options: CreatePythonValidationChecksOptions = {} @@ -51,13 +55,15 @@ export function createPythonValidationChecks( ...(options.processProbe === undefined ? {} : { processProbe: options.processProbe }), ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }) }); + const resolveSources = createPythonSourceSetResolver(options.importAnalyzer, resolveContexts); + const resolveRoots = createPythonSourceRootResolver(); return [ createSyntaxCheck(options, resolveContexts), createSourceHygieneCheck(), - createTypeCheck(options, resolveContexts), - createImportGraphCheck(resolveContexts), - createDeadCodeCheck(), - createRelevantTestsCheck() + createTypeCheck(options, resolveContexts, resolveSources), + createImportGraphCheck(resolveSources), + createDeadCodeCheck(resolveRoots), + createRelevantTestsCheck(resolveRoots) ].map((check) => withPythonProjectContexts(check, resolveContexts)); } diff --git a/packages/validation-python/src/project-config-files.ts b/packages/validation-python/src/project-config-files.ts new file mode 100644 index 0000000..ed16565 --- /dev/null +++ b/packages/validation-python/src/project-config-files.ts @@ -0,0 +1,14 @@ +export const pythonBoundaryFileNames = [ + "pyproject.toml", "Pipfile", "Pipfile.lock", "poetry.lock", "pdm.lock", "uv.lock", "setup.cfg", "setup.py" +] as const; + +const pythonToolConfigFileNames = [ + "pyrightconfig.json", "ruff.toml", ".ruff.toml", "mypy.ini", ".mypy.ini", "pytest.ini", "tox.ini" +] as const; + +export function isRelevantPythonConfig(path: string): boolean { + const name = path.slice(path.lastIndexOf("/") + 1); + return pythonBoundaryFileNames.includes(name as (typeof pythonBoundaryFileNames)[number]) || + /^requirements.*\.txt$/u.test(name) || + pythonToolConfigFileNames.includes(name as (typeof pythonToolConfigFileNames)[number]); +} diff --git a/packages/validation-python/src/project-discovery.ts b/packages/validation-python/src/project-discovery.ts index 8789095..e04e840 100644 --- a/packages/validation-python/src/project-discovery.ts +++ b/packages/validation-python/src/project-discovery.ts @@ -3,7 +3,7 @@ import type { PythonProjectFileEvidence, PythonProjectLayoutEvidence } from "@the-open-engine/opcore-contracts"; -import { pythonBoundaryFileNames } from "./static-config.js"; +import { pythonBoundaryFileNames } from "./project-config-files.js"; import type { PythonProjectWorkspace } from "./project-workspace.js"; export interface PythonProjectDiscovery { diff --git a/packages/validation-python/src/project-workspace.ts b/packages/validation-python/src/project-workspace.ts index 8d69e09..7cdf1b7 100644 --- a/packages/validation-python/src/project-workspace.ts +++ b/packages/validation-python/src/project-workspace.ts @@ -3,6 +3,7 @@ import { realpathSync } from "node:fs"; import { relative, resolve, sep } from "node:path"; import { validateRepoRelativePath } from "@the-open-engine/opcore-contracts"; import type { ValidationFileView } from "@the-open-engine/opcore-validation"; +import { isRelevantPythonConfig } from "./project-config-files.js"; export interface PythonProjectWorkspaceRealpath { path: string; @@ -30,12 +31,24 @@ export function createValidationFileViewPythonWorkspace( return result.status === "found" ? result.content : undefined; }, list: async () => { - const candidates = [...new Set([ - ...await fileView.listVisibleFiles(), - ...(fullWorkspace === undefined ? [] : await fullWorkspace.list()) - ])].sort(); + const fileViewPaths = await fileView.listVisibleFiles(); + const fullWorkspacePaths = fullWorkspace === undefined ? [] : await fullWorkspace.list(); + const fileViewPathSet = new Set(fileViewPaths); + const fullWorkspacePathSet = new Set(fullWorkspacePaths); + const candidates = [...new Set([...fileViewPaths, ...fullWorkspacePaths])] + .filter(isPythonProjectWorkspaceInput) + .sort(); const visible: string[] = []; for (const path of candidates) { + if (fileView.defaultReadState === "after") { + const overlay = fileView.overlayFor(path); + if (overlay?.action === "delete") continue; + if (overlay?.action === "write" + || (fileViewPathSet.has(path) && (fullWorkspace === undefined || fullWorkspacePathSet.has(path)))) { + visible.push(path); + continue; + } + } if (await fileView.exists(path)) visible.push(path); } return visible; @@ -53,6 +66,9 @@ export function createValidationFileViewPythonWorkspace( executableExists: executableAvailable }; } +function isPythonProjectWorkspaceInput(path: string): boolean { + return /\.pyi?$/u.test(path) || isRelevantPythonConfig(path); +} export function createNodePythonProjectWorkspace(repoRoot: string): PythonProjectWorkspace { const canonicalRoot = realpathSync(resolve(repoRoot)); diff --git a/packages/validation-python/src/relevant-tests-check.ts b/packages/validation-python/src/relevant-tests-check.ts index 46da5c4..2a648a5 100644 --- a/packages/validation-python/src/relevant-tests-check.ts +++ b/packages/validation-python/src/relevant-tests-check.ts @@ -2,10 +2,10 @@ import type { GraphFactEdge, ValidationDiagnostic } from "@the-open-engine/opcor import type { ValidationCheckDefinition } from "@the-open-engine/opcore-validation"; import { PYTHON_RELEVANT_TESTS_CHECK_ID } from "./check-ids.js"; import { pythonCheckAdapter, pythonCheckOwner, supportedPythonValidationScopes } from "./check-constants.js"; -import { relevantTestsGraphRequirements } from "./graph-requirements.js"; -import { materializePythonSources, toFileNodeId } from "./source-files.js"; +import { createRelevantTestsGraphRequirements } from "./graph-requirements.js"; +import { toFileNodeId, type PythonSourceRootResolver } from "./source-files.js"; -export function createRelevantTestsCheck(): ValidationCheckDefinition { +export function createRelevantTestsCheck(resolveRoots: PythonSourceRootResolver): ValidationCheckDefinition { return { id: PYTHON_RELEVANT_TESTS_CHECK_ID, owner: pythonCheckOwner, @@ -13,10 +13,10 @@ export function createRelevantTestsCheck(): ValidationCheckDefinition { defaultSeverity: "info", supportedScopes: supportedPythonValidationScopes, requiresGraph: true, - graphRequirements: relevantTestsGraphRequirements, + graphRequirements: createRelevantTestsGraphRequirements(resolveRoots), run: async (context) => { - const [sourceSet, testedBy] = await Promise.all([materializePythonSources(context), context.graph.testedBy()]); - const diagnostics = sourceSet.rootPaths.map((path): ValidationDiagnostic => { + const [rootPaths, testedBy] = await Promise.all([resolveRoots(context), context.graph.testedBy()]); + const diagnostics = rootPaths.map((path): ValidationDiagnostic => { const evidence = testedBy.filter((edge) => edgeReferencesFile(edge, path)); if (evidence.length > 0) { return { diff --git a/packages/validation-python/src/source-files.ts b/packages/validation-python/src/source-files.ts index 48461db..344be36 100644 --- a/packages/validation-python/src/source-files.ts +++ b/packages/validation-python/src/source-files.ts @@ -1,60 +1,90 @@ import type { PythonProjectContext } from "@the-open-engine/opcore-contracts"; import type { ValidationCheckContext, ValidationCheckResult, ValidationFileView } from "@the-open-engine/opcore-validation"; import { normalizeValidationFileViewPath } from "@the-open-engine/opcore-validation"; +import { + requirePythonImportAnalyzer, + validatePythonImportEdges, + type PythonImportAnalyzer, + type PythonImportEdge, + type PythonImportSourceFile +} from "./import-analysis.js"; import { resolvePythonProjectContexts, type ResolvePythonProjectContextsOptions } from "./project-context.js"; import { createValidationFileViewPythonWorkspace, type PythonProjectWorkspace } from "./project-workspace.js"; export const pythonSourceExtensions = [".py", ".pyi"] as const; -export interface PythonMaterializedSourceFile { - path: string; - content: string; -} - -export interface PythonRepoImport { - fromPath: string; - specifier: string; - resolvedPath: string; -} +export type PythonMaterializedSourceFile = PythonImportSourceFile; export interface PythonMaterializedSourceSet { rootPaths: readonly string[]; paths: readonly string[]; files: readonly PythonMaterializedSourceFile[]; sourceFileByPath: ReadonlyMap; - repoImports: readonly PythonRepoImport[]; -} - -interface ParsedPythonImport { - specifier: string; - member?: string; + repoImports: readonly PythonImportEdge[]; } -export type PythonProjectContextResolver = (context: ValidationCheckContext) => Promise; +export type PythonProjectContextResolver = ( + context: ValidationCheckContext, + targets?: readonly string[] +) => Promise; +export type PythonSourceRootResolver = (context: ValidationCheckContext) => Promise; +export type PythonSourceSetResolver = (context: ValidationCheckContext) => Promise; export function createPythonProjectContextResolver( options: Omit & { nodeWorkspace?: PythonProjectWorkspace; } = {} ): PythonProjectContextResolver { - const cache = new WeakMap>(); - return (context) => { - const existing = cache.get(context.fileView); - if (existing !== undefined) return existing; - const targets = pythonInputSet(context); - const promise = targets.length === 0 - ? Promise.resolve([]) - : resolvePythonProjectContexts({ - repoRoot: context.request.repo.repoRoot ?? process.cwd(), - targets, - workspace: createValidationFileViewPythonWorkspace(context.fileView, undefined, options.nodeWorkspace), - ...withoutNodeWorkspace(options) - }); - cache.set(context.fileView, promise); - return promise; + const cache = new WeakMap(); + return async (context, requestedTargets) => { + let cached = cache.get(context.fileView); + if (cached === undefined) { + cached = { contexts: new Map() }; + cache.set(context.fileView, cached); + } + const targets = requestedTargets === undefined + ? await (cached.inputTargets ??= readPythonAfterSources(context).then((sources) => sources.map((source) => source.path))) + : uniqueSorted(requestedTargets.map(normalizeValidationFileViewPath).filter(isPythonSourcePath)); + while (true) { + const missing = targets.filter((target) => !cached.contexts.has(target)); + if (missing.length === 0) return targets.map((target) => requiredProjectContext(cached.contexts, target)); + if (cached.pending !== undefined) { + await cached.pending; + continue; + } + const pending = resolvePythonProjectContexts({ + repoRoot: context.request.repo.repoRoot ?? process.cwd(), + targets: missing, + workspace: createValidationFileViewPythonWorkspace(context.fileView, undefined, options.nodeWorkspace), + ...withoutNodeWorkspace(options) + }).then((contexts) => { + for (const projectContext of contexts) cached.contexts.set(projectContext.target, projectContext); + }); + cached.pending = pending; + try { + await pending; + } finally { + delete cached.pending; + } + } }; } +interface PythonProjectContextCache { + contexts: Map; + inputTargets?: Promise; + pending?: Promise; +} + +function requiredProjectContext( + contexts: ReadonlyMap, + target: string +): PythonProjectContext { + const projectContext = contexts.get(target); + if (projectContext === undefined) throw new Error(`Python project context resolution omitted target ${target}`); + return projectContext; +} + function withoutNodeWorkspace( options: Omit & { nodeWorkspace?: PythonProjectWorkspace; @@ -89,6 +119,17 @@ export async function readPythonAfterSources(context: ValidationCheckContext): P return files.sort((left, right) => left.path.localeCompare(right.path)); } +export function createPythonSourceRootResolver(): PythonSourceRootResolver { + const cache = new WeakMap>(); + return (context) => { + const existing = cache.get(context.fileView); + if (existing !== undefined) return existing; + const promise = readPythonAfterSources(context).then((sources) => sources.map((source) => source.path)); + cache.set(context.fileView, promise); + return promise; + }; +} + export function skippedPythonInputResult(context: ValidationCheckContext): ValidationCheckResult | undefined { if (pythonInputSet(context).length > 0) return undefined; return { @@ -98,186 +139,159 @@ export function skippedPythonInputResult(context: ValidationCheckContext): Valid }; } -export async function materializePythonSources( - context: ValidationCheckContext, - projectContexts: readonly PythonProjectContext[] = [] -): Promise { - return materializePythonSourcesUncached(context, projectContexts); +export function createPythonSourceSetResolver( + importAnalyzer: PythonImportAnalyzer | undefined, + resolveContexts: PythonProjectContextResolver +): PythonSourceSetResolver { + const cache = new WeakMap>(); + return (context) => { + const existing = cache.get(context.fileView); + if (existing !== undefined) return existing; + const promise = materializePythonSourcesUncached(context, importAnalyzer, resolveContexts); + cache.set(context.fileView, promise); + return promise; + }; } async function materializePythonSourcesUncached( context: ValidationCheckContext, - projectContexts: readonly PythonProjectContext[] + importAnalyzer: PythonImportAnalyzer | undefined, + resolveContexts: PythonProjectContextResolver ): Promise { - const initialPaths = pythonInputSet(context); - const rootPaths: string[] = []; - const pending = [...initialPaths]; - const visited = new Set(); - const sourceFileByPath = new Map(); - const repoImports: PythonRepoImport[] = []; - - while (pending.length > 0) { - const path = pending.shift(); - if (path === undefined || visited.has(path)) continue; - visited.add(path); - + const visiblePaths = uniqueSorted( + (await context.fileView.listVisibleFiles()) + .map(normalizeValidationFileViewPath) + .filter(isPythonSourcePath) + ); + const allSources: PythonMaterializedSourceFile[] = []; + for (const path of visiblePaths) { const result = await context.fileView.readAfter(path); - if (result.status !== "found") continue; - const sourceFile = { path, content: result.content }; - sourceFileByPath.set(path, sourceFile); - if (initialPaths.includes(path)) rootPaths.push(path); - - for (const parsedImport of parsePythonImports(result.content)) { - const resolvedPath = await resolvePythonImport(context, path, parsedImport, projectContexts); - if (resolvedPath === undefined) continue; - repoImports.push({ fromPath: path, specifier: parsedImport.specifier, resolvedPath }); - if (!visited.has(resolvedPath) && !sourceFileByPath.has(resolvedPath)) pending.push(resolvedPath); - } + if (result.status === "found") allSources.push({ path, content: result.content }); } - const files = [...sourceFileByPath.values()].sort((left, right) => left.path.localeCompare(right.path)); + const allSourceByPath = new Map(allSources.map((source) => [source.path, source])); + const rootPaths = pythonInputSet(context).filter((path) => allSourceByPath.has(path)); + if (rootPaths.length === 0) return emptySourceSet(); + + const analyzer = requirePythonImportAnalyzer(importAnalyzer); + const repoImports = validatePythonImportEdges( + await analyzer.analyze(allSources), + new Set(allSourceByPath.keys()) + ); + const selectedPaths = await expandSourceClosure(context, rootPaths, repoImports, allSourceByPath, resolveContexts); + const files = selectedPaths.map((path) => allSourceByPath.get(path)).filter(isDefined); + const sourceFileByPath = new Map(files.map((file) => [file.path, file])); + const selectedPathSet = new Set(selectedPaths); return { - rootPaths: uniqueSorted(rootPaths), - paths: files.map((file) => file.path), + rootPaths, + paths: selectedPaths, files, sourceFileByPath, - repoImports: uniqueRepoImports(repoImports) + repoImports: repoImports.filter((edge) => selectedPathSet.has(edge.fromPath) && selectedPathSet.has(edge.toPath)) }; } -function parsePythonImports(content: string): readonly ParsedPythonImport[] { - const imports: ParsedPythonImport[] = []; - for (const rawLine of content.split(/\r?\n/u)) { - const line = stripInlineComment(rawLine).trim(); - if (line.length === 0) continue; - const importMatch = /^import\s+(.+)$/u.exec(line); - if (importMatch !== null) { - for (const entry of importMatch[1].split(",")) { - const specifier = entry.trim().split(/\s+as\s+/u)[0]?.trim(); - if (isImportableModuleSpecifier(specifier)) imports.push({ specifier }); - } - continue; - } - const fromMatch = /^from\s+([.\w]+)\s+import\s+(.+)$/u.exec(line); - if (fromMatch === null) continue; - const moduleSpecifier = fromMatch[1]; - const importedNames = fromMatch[2].split(",").map((entry) => entry.trim().split(/\s+as\s+/u)[0]?.trim() ?? ""); - if (/^\.+$/u.test(moduleSpecifier)) { - for (const name of importedNames) { - if (/^[A-Za-z_]\w*$/u.test(name)) imports.push({ specifier: `${moduleSpecifier}${name}` }); - } - } else if (isImportableModuleSpecifier(moduleSpecifier)) { - imports.push({ specifier: moduleSpecifier }); - for (const name of importedNames) { - if (/^[A-Za-z_]\w*$/u.test(name)) imports.push({ specifier: `${moduleSpecifier}.${name}`, member: name }); +async function expandSourceClosure( + context: ValidationCheckContext, + rootPaths: readonly string[], + edges: readonly PythonImportEdge[], + sourceByPath: ReadonlyMap, + resolveContexts: PythonProjectContextResolver +): Promise { + const projectContexts = new Map(); + let selected = transitiveSourcePaths(rootPaths, edges); + while (true) { + const unresolvedTargets = selected.filter((path) => !projectContexts.has(path)); + if (unresolvedTargets.length > 0) { + for (const projectContext of await resolveContexts(context, unresolvedTargets)) { + projectContexts.set(projectContext.target, projectContext); } } + const expanded = transitiveSourcePaths( + includePackageInitializers(selected, sourceByPath, [...projectContexts.values()]), + edges + ); + if (expanded.length === selected.length && expanded.every((path, index) => path === selected[index])) return expanded; + selected = expanded; } - return imports; } -async function resolvePythonImport( - context: ValidationCheckContext, - fromPath: string, - parsedImport: ParsedPythonImport, +function includePackageInitializers( + selectedPaths: readonly string[], + sourceByPath: ReadonlyMap, projectContexts: readonly PythonProjectContext[] -): Promise { - for (const moduleBase of moduleBasePaths(fromPath, parsedImport.specifier, projectContexts)) { - const resolved = await resolveModulePath(context, moduleBase); - if (resolved !== undefined) return resolved; - } - return undefined; -} - -async function resolveModulePath(context: ValidationCheckContext, moduleBase: string): Promise { - for (const candidate of moduleCandidates(moduleBase)) { - let normalized: string; - try { - normalized = normalizeValidationFileViewPath(candidate); - } catch { - continue; +): readonly string[] { + const expanded = new Set(selectedPaths); + for (const path of selectedPaths) { + const sourceRoot = owningSourceRoot(path, projectContexts); + if (sourceRoot === undefined) continue; + let directory = path.slice(0, path.lastIndexOf("/")); + while (directory.length > 0 && directory !== sourceRoot && pathWithinRoot(directory, sourceRoot)) { + const initializers = [`${directory}/__init__.py`, `${directory}/__init__.pyi`] + .filter((candidate) => sourceByPath.has(candidate)); + if (initializers.length === 0) { + const separator = directory.lastIndexOf("/"); + if (separator < 0) break; + directory = directory.slice(0, separator); + continue; + } + // Package markers are structural type-checker inputs, not import expectations. + for (const initializer of initializers) expanded.add(initializer); + const separator = directory.lastIndexOf("/"); + if (separator < 0) break; + directory = directory.slice(0, separator); } - if (!isPythonSourcePath(normalized)) continue; - if (await context.fileView.exists(normalized)) return normalized; } - return undefined; + return [...expanded].sort(); } - -function moduleBasePaths( - fromPath: string, - specifier: string, - projectContexts: readonly PythonProjectContext[] -): readonly string[] { - const leadingDots = /^\.*/u.exec(specifier)?.[0].length ?? 0; - const moduleSpecifier = specifier.slice(leadingDots); - const moduleParts = moduleSpecifier.length === 0 ? [] : moduleSpecifier.split("."); - if (leadingDots === 0) { - const module = moduleParts.join("/"); - const owning = owningContext(fromPath, projectContexts); - const roots = owning?.sourceRoots ?? ["."]; - return roots.map((root) => root === "." ? module : `${root}/${module}`); - } - - const baseParts = fromPath.split("/"); - baseParts.pop(); - for (let index = 1; index < leadingDots; index += 1) { - if (baseParts.length === 0) return []; - baseParts.pop(); - } - return [[...baseParts, ...moduleParts].filter((part) => part.length > 0).join("/")]; +function owningSourceRoot(path: string, projectContexts: readonly PythonProjectContext[]): string | undefined { + return projectContexts + .flatMap((projectContext) => projectContext.sourceRoots) + .filter((sourceRoot) => pathWithinRoot(path, sourceRoot)) + .sort((left, right) => right.length - left.length || left.localeCompare(right))[0]; } -function owningContext(path: string, contexts: readonly PythonProjectContext[]): PythonProjectContext | undefined { - return [...contexts] - .filter((context) => context.target === path || context.projectRoot === "." || path.startsWith(`${context.projectRoot}/`)) - .sort((left, right) => { - const exact = Number(right.target === path) - Number(left.target === path); - return exact !== 0 ? exact : right.projectRoot.length - left.projectRoot.length; - })[0]; +function pathWithinRoot(path: string, root: string): boolean { + return root === "." || path === root || path.startsWith(`${root}/`); } -function moduleCandidates(moduleBase: string): readonly string[] { - if (moduleBase.length === 0) return []; - return [`${moduleBase}.py`, `${moduleBase}.pyi`, `${moduleBase}/__init__.py`, `${moduleBase}/__init__.pyi`]; -} -function stripInlineComment(line: string): string { - let quote: "'" | "\"" | undefined; - let escaped = false; - for (let index = 0; index < line.length; index += 1) { - const char = line[index]; - if (escaped) { - escaped = false; - continue; - } - if (quote !== undefined && char === "\\") { - escaped = true; - continue; - } - if (char === "'" || char === "\"") { - if (quote === undefined) quote = char; - else if (quote === char) quote = undefined; - continue; +function transitiveSourcePaths( + rootPaths: readonly string[], + edges: readonly PythonImportEdge[] +): readonly string[] { + const outgoing = new Map(); + for (const edge of edges) { + const targets = outgoing.get(edge.fromPath) ?? []; + targets.push(edge.toPath); + outgoing.set(edge.fromPath, targets); + } + const selected = new Set(rootPaths); + const pending = [...rootPaths]; + while (pending.length > 0) { + const path = pending.shift(); + if (path === undefined) continue; + for (const target of outgoing.get(path) ?? []) { + if (selected.has(target)) continue; + selected.add(target); + pending.push(target); } - if (quote === undefined && char === "#") return line.slice(0, index); } - return line; + return [...selected].sort(); } -function isImportableModuleSpecifier(value: string | undefined): value is string { - return value !== undefined && /^\.?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*$/u.test(value); +function emptySourceSet(): PythonMaterializedSourceSet { + return { + rootPaths: [], + paths: [], + files: [], + sourceFileByPath: new Map(), + repoImports: [] + }; } -function uniqueRepoImports(imports: readonly PythonRepoImport[]): readonly PythonRepoImport[] { - const byKey = new Map(); - for (const repoImport of imports) { - byKey.set(`${repoImport.fromPath}\0${repoImport.specifier}\0${repoImport.resolvedPath}`, repoImport); - } - return [...byKey.values()].sort((left, right) => - `${left.fromPath}\0${left.resolvedPath}\0${left.specifier}`.localeCompare( - `${right.fromPath}\0${right.resolvedPath}\0${right.specifier}` - ) - ); +function isDefined(value: T | undefined): value is T { + return value !== undefined; } export function uniqueSorted(values: readonly string[]): readonly string[] { diff --git a/packages/validation-python/src/static-config.ts b/packages/validation-python/src/static-config.ts index 8c4cb79..4d455e5 100644 --- a/packages/validation-python/src/static-config.ts +++ b/packages/validation-python/src/static-config.ts @@ -5,12 +5,10 @@ import type { PythonProjectTarget } from "@the-open-engine/opcore-contracts"; import { parse as parseToml } from "smol-toml"; +import { isRelevantPythonConfig } from "./project-config-files.js"; import type { PythonProjectWorkspace } from "./project-workspace.js"; import { pythonVersionSatisfiesConstraint } from "./version-constraint.js"; -export const pythonBoundaryFileNames = [ - "pyproject.toml", "Pipfile", "Pipfile.lock", "poetry.lock", "pdm.lock", "uv.lock", "setup.cfg", "setup.py" -] as const; export const pythonToolConfigPrecedence = { pyright: ["pyrightconfig.json", "pyproject.toml"], @@ -204,12 +202,6 @@ function selectConfig( return name === undefined ? undefined : joinRoot(projectRoot, name); } -function isRelevantPythonConfig(path: string): boolean { - const name = basename(path); - return pythonBoundaryFileNames.includes(name as (typeof pythonBoundaryFileNames)[number]) || - /^requirements.*\.txt$/u.test(name) || - ["pyrightconfig.json", "ruff.toml", ".ruff.toml", "mypy.ini", ".mypy.ini", "pytest.ini", "tox.ini"].includes(name); -} function isTomlConfig(path: string): boolean { return path.endsWith(".toml") || ["Pipfile", "poetry.lock", "pdm.lock", "uv.lock"].includes(basename(path)); diff --git a/packages/validation-python/src/syntax-check.ts b/packages/validation-python/src/syntax-check.ts index b2b5dd9..97fd67e 100644 --- a/packages/validation-python/src/syntax-check.ts +++ b/packages/validation-python/src/syntax-check.ts @@ -40,6 +40,7 @@ export function createSyntaxCheck( if (skipped !== undefined) return skipped; const sources = await readPythonAfterSources(context); + if (sources.length === 0) return { diagnostics: [] }; if (resolveContexts === undefined) return missingContextResult(sources.map((source) => source.path)); const resolvedContexts = await resolveContexts(context); const missing = sources.map((source) => source.path).filter((path) => !resolvedContexts.some((candidate) => candidate.target === path)); diff --git a/packages/validation-python/src/type-check.ts b/packages/validation-python/src/type-check.ts index 51cbf9b..0a5f5c9 100644 --- a/packages/validation-python/src/type-check.ts +++ b/packages/validation-python/src/type-check.ts @@ -18,11 +18,11 @@ import { pythonCheckAdapter, pythonCheckOwner, supportedPythonValidationScopes } import { diagnostic, sortDiagnostics } from "./diagnostics.js"; import { runTool } from "./process.js"; import { - materializePythonSources, pythonInputSet, skippedPythonInputResult, type PythonMaterializedSourceFile, - type PythonProjectContextResolver + type PythonProjectContextResolver, + type PythonSourceSetResolver } from "./source-files.js"; import type { PythonValidationToolchainOptions } from "./toolchain.js"; @@ -43,7 +43,8 @@ interface PythonProjectGroup { export function createTypeCheck( options: PythonTypeCheckOptions = {}, - resolveContexts?: PythonProjectContextResolver + resolveContexts?: PythonProjectContextResolver, + resolveSources?: PythonSourceSetResolver ): ValidationCheckDefinition { return { id: PYTHON_TYPES_CHECK_ID, @@ -55,11 +56,13 @@ export function createTypeCheck( const skipped = skippedPythonInputResult(context); if (skipped !== undefined) return skipped; if (resolveContexts === undefined) return missingContextResult(pythonInputSet(context)); + if (resolveSources === undefined) throw new Error("A shared Python source-set resolver is required for Python type validation"); + const sourceSet = await resolveSources(context); + if (sourceSet.rootPaths.length === 0) return { diagnostics: [] }; const resolvedContexts = await resolveContexts(context); - const selectedTargets = pythonInputSet(context); + const selectedTargets = sourceSet.rootPaths; const missing = selectedTargets.filter((path) => !resolvedContexts.some((candidate) => candidate.target === path)); if (resolvedContexts.length === 0 || missing.length > 0) return missingContextResult(missing); - const sourceSet = await materializePythonSources(context, resolvedContexts); if (sourceSet.files.length === 0) return { diagnostics: [] }; const unresolved = resolvedContexts.find(isUnresolvedTypeContext); if (unresolved !== undefined) return unresolvedProjectResult(unresolved); @@ -68,8 +71,7 @@ export function createTypeCheck( for (const project of projects) { const checker = selectTypeChecker(project.context); if (checker === undefined) return missingTypeChecker(project.context); - const files = sourceSet.files.filter((file) => owningProject(file.path, projects)?.context.projectKey === project.context.projectKey); - const workspace = await materializePythonTypeWorkspace(context, project, files); + const workspace = await materializePythonTypeWorkspace(context, project, sourceSet.files); try { const args = [ ...materializedCheckerPrefix(checker, project.context.projectRoot), @@ -349,11 +351,6 @@ function relativeProjectPath(path: string, projectRoot: string): string { return projectRoot === "." ? path : path.slice(`${projectRoot}/`.length); } -function owningProject(path: string, projects: readonly PythonProjectGroup[]): PythonProjectGroup | undefined { - return [...projects] - .filter((project) => project.context.projectRoot === "." || path.startsWith(`${project.context.projectRoot}/`)) - .sort((left, right) => right.context.projectRoot.length - left.context.projectRoot.length)[0]; -} function isConfigPath(path: string): boolean { return /(?:\.toml|\.ini|\.cfg|\.lock|Pipfile|requirements.*\.txt)$/u.test(path); diff --git a/tests/conformance.test.mjs b/tests/conformance.test.mjs index b5a217b..0d07ca4 100644 --- a/tests/conformance.test.mjs +++ b/tests/conformance.test.mjs @@ -366,6 +366,22 @@ describe("conformance fixture metadata", () => { assert.deepEqual(sourceExtraction.nodeKinds, ["File", "Module", "Class", "Function", "Variable"]); assert.ok(sourceExtraction.edgeKinds.includes("TESTED_BY")); assert.deepEqual(sourceExtraction.diagnostics, ["parse_error", "unresolved_import"]); + assert.deepEqual(sourceExtraction.importResolution, { + owner: "graph-core", + expectedEdgeField: "pythonImportEdges", + cases: [ + "multiline-parenthesized", + "alias", + "conditional", + "submodule", + "star", + "relative", + "package-initializer", + "stub", + "namespace", + "src-layout" + ] + }); }); it("describes Python validation fixture metadata for #22", () => { diff --git a/tests/fixtures/package-packlists.json b/tests/fixtures/package-packlists.json index 6490d9e..a946aea 100644 --- a/tests/fixtures/package-packlists.json +++ b/tests/fixtures/package-packlists.json @@ -296,6 +296,9 @@ "node_modules/@the-open-engine/opcore-graph/dist/native-targets.d.ts", "node_modules/@the-open-engine/opcore-graph/dist/native-targets.d.ts.map", "node_modules/@the-open-engine/opcore-graph/dist/native-targets.js", + "node_modules/@the-open-engine/opcore-graph/dist/python-import-analysis.d.ts", + "node_modules/@the-open-engine/opcore-graph/dist/python-import-analysis.d.ts.map", + "node_modules/@the-open-engine/opcore-graph/dist/python-import-analysis.js", "node_modules/@the-open-engine/opcore-graph/dist/serve.d.ts", "node_modules/@the-open-engine/opcore-graph/dist/serve.d.ts.map", "node_modules/@the-open-engine/opcore-graph/dist/serve.js", @@ -388,6 +391,9 @@ "node_modules/@the-open-engine/opcore-validation-python/dist/graph-requirements.d.ts", "node_modules/@the-open-engine/opcore-validation-python/dist/graph-requirements.d.ts.map", "node_modules/@the-open-engine/opcore-validation-python/dist/graph-requirements.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/import-analysis.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/import-analysis.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/import-analysis.js", "node_modules/@the-open-engine/opcore-validation-python/dist/import-graph-check.d.ts", "node_modules/@the-open-engine/opcore-validation-python/dist/import-graph-check.d.ts.map", "node_modules/@the-open-engine/opcore-validation-python/dist/import-graph-check.js", @@ -397,6 +403,9 @@ "node_modules/@the-open-engine/opcore-validation-python/dist/process.d.ts", "node_modules/@the-open-engine/opcore-validation-python/dist/process.d.ts.map", "node_modules/@the-open-engine/opcore-validation-python/dist/process.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/project-config-files.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/project-config-files.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/project-config-files.js", "node_modules/@the-open-engine/opcore-validation-python/dist/project-context.d.ts", "node_modules/@the-open-engine/opcore-validation-python/dist/project-context.d.ts.map", "node_modules/@the-open-engine/opcore-validation-python/dist/project-context.js", diff --git a/tests/graph-python-import-analysis.test.mjs b/tests/graph-python-import-analysis.test.mjs new file mode 100644 index 0000000..6a9dd15 --- /dev/null +++ b/tests/graph-python-import-analysis.test.mjs @@ -0,0 +1,112 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { dirname, join, relative } from "node:path"; +import { fileURLToPath } from "node:url"; +import { analyzePythonImports } from "../packages/graph/dist/index.js"; +import { analyzePythonImportsWithGraph } from "../packages/graph/dist/python-import-analysis.js"; + +const repoRoot = fileURLToPath(new URL("..", import.meta.url)); +const fixtureRoot = join(repoRoot, "packages/fixtures/source-extraction/python"); +const expected = JSON.parse(readFileSync(join(fixtureRoot, "python.expected.json"), "utf8")); + +describe("graph Python import analysis adapter", () => { + it("returns canonical directed graph-core edges for supplied Python files", async () => { + const edges = await analyzePythonImports([ + { path: "src/app.py", content: "from pkg import (\n dep as renamed,\n)\n" }, + { path: "src/pkg/__init__.py", content: "" }, + { path: "src/pkg/dep.py", content: "VALUE = 1\n" } + ]); + + assert.deepEqual(edges, [{ fromPath: "src/app.py", toPath: "src/pkg/dep.py" }]); + }); + + it("matches the shared Python fixture edge matrix without mutating source files", async () => { + const beforeFiles = fixtureFiles(); + const edges = await analyzePythonImports( + [...beforeFiles].map(([path, content]) => ({ path, content })) + ); + + assert.deepEqual(edges, expected.pythonImportEdges); + assert.deepEqual(fixtureFiles(), beforeFiles); + }); + + it("rejects unsafe or malformed supplied-file paths before native analysis", async () => { + await assert.rejects( + analyzePythonImports([{ path: "../outside.py", content: "VALUE = 1\n" }]), + /repo-relative|traversal|invalid/i + ); + await assert.rejects( + analyzePythonImports([{ path: "src/app.ts", content: "export const value = 1;\n" }]), + /only \.py\/\.pyi/i + ); + }); + + it("fails loudly and removes its temporary state when graph analysis fails", async () => { + let analysisRoot; + await assert.rejects( + analyzePythonImportsWithGraph( + [{ path: "src/app.py", content: "VALUE = 1\n" }], + { + build(repo) { + analysisRoot = dirname(repo.repoRoot); + assert.equal(existsSync(analysisRoot), true); + throw new Error("native build unavailable"); + }, + query() { + throw new Error("query must not run"); + } + } + ), + /native build unavailable/ + ); + assert.equal(typeof analysisRoot, "string"); + assert.equal(existsSync(analysisRoot), false); + }); + + it("rejects unavailable and malformed graph query results instead of returning empty edges", async () => { + const available = { state: "available", mode: "required", provider: "opcore-graph", schemaVersion: 1 }; + const build = () => ({ status: available }); + await assert.rejects( + analyzePythonImportsWithGraph([{ path: "src/app.py", content: "VALUE = 1\n" }], { + build, + query: () => ({ + status: { + state: "required_missing", + mode: "required", + provider: "opcore-graph", + schemaVersion: 1, + message: "native query unavailable" + } + }) + }), + /query failed.*native query unavailable/i + ); + await assert.rejects( + analyzePythonImportsWithGraph([{ path: "src/app.py", content: "VALUE = 1\n" }], { + build, + query: () => ({ status: available }) + }), + /query failed.*available/i + ); + }); +}); + +function fixtureFiles() { + return new Map( + walkFiles(fixtureRoot) + .filter((path) => path.endsWith(".py") || path.endsWith(".pyi")) + .map((path) => [relative(fixtureRoot, path).replaceAll("\\", "/"), readFileSync(path, "utf8")]) + ); +} + +function walkFiles(root) { + const files = []; + for (const entry of readdirSync(root, { withFileTypes: true })) { + const path = join(root, entry.name); + if (entry.isDirectory() && entry.name !== ".opcore") files.push(...walkFiles(path)); + else if (entry.isFile()) files.push(path); + } + return files.sort(); +} + diff --git a/tests/graph-store-conformance.test.mjs b/tests/graph-store-conformance.test.mjs index 24dab7d..4f8cd3d 100644 --- a/tests/graph-store-conformance.test.mjs +++ b/tests/graph-store-conformance.test.mjs @@ -119,21 +119,8 @@ describe("GraphProvider SQLite store conformance", () => { const status = graphProviderStatus({ repoRoot: fixtureRoot }); assert.equal(status.state, "available"); assert.equal(status.dbPath, refresh.dbPath); - assert.deepEqual(status.nodes_by_kind, { - Class: 3, - File: 9, - Function: 12, - Module: 9, - Variable: 2 - }); - assert.deepEqual(status.edges_by_kind, { - CALLS: 11, - CONTAINS: 26, - DEPENDS_ON: 7, - IMPORTS_FROM: 7, - INHERITS: 1, - TESTED_BY: 3 - }); + assert.deepEqual(status.nodes_by_kind, expectedPythonNodeCounts()); + assert.deepEqual(status.edges_by_kind, expectedPythonEdgeCounts()); const rawStatus = rawGraphStatus(fixtureRoot); assert.equal(rawStatus.state, "available"); assert.deepEqual(rawStatus.nodes_by_kind, status.nodes_by_kind); @@ -150,28 +137,15 @@ describe("GraphProvider SQLite store conformance", () => { assert.equal(db.prepare("pragma user_version").get().user_version, 1); assert.deepEqual( plainRows(db.prepare("select kind, count(*) as count from nodes group by kind order by kind").all()), - [ - { kind: "Class", count: 3 }, - { kind: "File", count: 9 }, - { kind: "Function", count: 12 }, - { kind: "Module", count: 9 }, - { kind: "Variable", count: 2 } - ] + Object.entries(expectedPythonNodeCounts()).map(([kind, count]) => ({ kind, count })) ); assert.deepEqual( plainRows(db.prepare("select kind, count(*) as count from edges group by kind order by kind").all()), - [ - { kind: "CALLS", count: 11 }, - { kind: "CONTAINS", count: 26 }, - { kind: "DEPENDS_ON", count: 7 }, - { kind: "IMPORTS_FROM", count: 7 }, - { kind: "INHERITS", count: 1 }, - { kind: "TESTED_BY", count: 3 } - ] + Object.entries(expectedPythonEdgeCounts()).map(([kind, count]) => ({ kind, count })) ); assert.deepEqual( plainRows(db.prepare("select language, count(*) as count from file_hashes group by language").all()), - [{ language: "python", count: 9 }] + [{ language: "python", count: expectedPythonNodeCounts().File }] ); assert.deepEqual( Object.fromEntries( @@ -234,7 +208,13 @@ describe("GraphProvider SQLite store conformance", () => { { queryKind: "importers_of", target: "src/pkg/models.py", maxDepth: 2, limit: 100 } ); assert.equal(importers.status.state, "available"); - assert.deepEqual(paths(importers.nodes), ["src/pkg/__init__.py", "src/pkg/models.py", "tests/test_models.py"]); + assert.deepEqual(paths(importers.nodes), [ + "src/cases/docstring_case.py", + "src/cases/package_case.py", + "src/pkg/__init__.py", + "src/pkg/models.py", + "tests/test_models.py" + ]); const tests = graphProviderNamedQuery( { repoRoot: fixtureRoot }, @@ -759,6 +739,29 @@ function assertFreshMetadataTimestamp(value, minEpochMs) { assert.equal(parsed <= Date.now() + 5000, true, value); } +function expectedPythonNodeCounts() { + const kindByPrefix = { + class: "Class", + file: "File", + function: "Function", + module: "Module", + variable: "Variable" + }; + return countBy( + pythonExpected.nodeIds.map((id) => kindByPrefix[id.slice(0, id.indexOf(":"))]) + ); +} + +function expectedPythonEdgeCounts() { + return countBy(pythonExpected.edgeTriples.map(([kind]) => kind)); +} + +function countBy(values) { + const counts = new Map(); + for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1); + return Object.fromEntries([...counts].sort(([left], [right]) => left.localeCompare(right))); +} + function edgeTriples(edges) { return edges.map((edge) => [edge.kind, edge.from, edge.to]).sort(compareTuple); } diff --git a/tests/installed-bins.test.mjs b/tests/installed-bins.test.mjs index 54247b7..f9e1429 100644 --- a/tests/installed-bins.test.mjs +++ b/tests/installed-bins.test.mjs @@ -513,7 +513,13 @@ function assertSmoke(project, args, expectedExitCode, bin = "opcore") { function assertCliJson(command, args, expectedExitCode, cwd, options = {}) { const startedAt = performance.now(); - const result = run(command, args, { cwd, env: options.env, expectedStatus: expectedExitCode }); + // Installed-artifact checks must not inherit development-only tools from the + // monorepo's node_modules/.bin through the test runner PATH. + const result = run(command, args, { + cwd, + env: options.env ?? sourceSafeOpcoreEnv(), + expectedStatus: expectedExitCode + }); const durationMs = nonNegativeDuration(performance.now() - startedAt); assert.doesNotMatch(result.stdout, onboardingForbiddenOutput); assert.doesNotMatch(result.stderr, onboardingForbiddenOutput); @@ -588,6 +594,7 @@ function assertAspProviderInitializeSmoke(project) { }; const result = spawnSync(binPath(project, "opcore-asp-provider"), ["--stdio"], { cwd: project, + env: sourceSafeOpcoreEnv(), input: `${JSON.stringify(request)}\n`, encoding: "utf8", stdio: ["pipe", "pipe", "pipe"] @@ -615,6 +622,7 @@ async function evaluateInstalledAspPythonContext(project) { const host = createInstalledAspHost(files); const child = spawn(binPath(project, "opcore-asp-provider"), ["--stdio"], { cwd: project, + env: sourceSafeOpcoreEnv(), stdio: ["pipe", "pipe", "pipe"] }); const peer = new InstalledAspPeer(child, host).start(); diff --git a/tests/python-import-composition.test.mjs b/tests/python-import-composition.test.mjs new file mode 100644 index 0000000..5a72bfd --- /dev/null +++ b/tests/python-import-composition.test.mjs @@ -0,0 +1,70 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { defaultAspProviderValidationChecks } from "../packages/asp-provider/dist/validation-composition.js"; +import { defaultValidationChecks as defaultOpcoreValidationChecks } from "../packages/opcore/dist/repo-validation-policy.js"; +import { createBuiltInValidationChecks } from "../packages/validation-policy/dist/index.js"; + +describe("Python import analyzer composition", () => { + it("forwards the structural analyzer through validation-policy", async () => { + let calls = 0; + const checks = createBuiltInValidationChecks(undefined, { + clone: false, + pythonImportAnalyzer: { + async analyze() { + calls += 1; + return [{ fromPath: "src/app.py", toPath: "src/pkg/dep.py" }]; + } + } + }); + + await importGraphRequirements(checks); + assert.equal(calls, 1); + }); + + it("injects the graph-owned analyzer into Opcore and ASP compositions", async () => { + await importGraphRequirements(defaultOpcoreValidationChecks); + await importGraphRequirements(defaultAspProviderValidationChecks); + }); +}); + +async function importGraphRequirements(checks) { + const check = checks.find((candidate) => candidate.id === "python.import-graph"); + assert.ok(check?.graphRequirements); + const requirements = await check.graphRequirements(validationContext()); + assert.deepEqual(requirements.map((requirement) => requirement.selector.kind), ["edges", "nodes"]); +} + +function validationContext() { + const files = new Map([ + ["src/app.py", "from pkg import (\n dep,\n)\n"], + ["src/pkg/dep.py", "VALUE = 1\n"] + ]); + return { + request: { + requestId: "python-import-composition", + repo: { repoId: "python-import-composition" }, + scope: { kind: "files", files: ["src/app.py"] }, + graph: { mode: "optional", provider: "opcore-graph" }, + overlays: [], + checks: ["python.import-graph"] + }, + fileView: { + overlays: [], + scopeFiles: ["src/app.py"], + defaultReadState: "after", + listVisibleFiles: async () => [...files.keys()], + readFile: async (path) => files.has(path) + ? { status: "found", content: files.get(path) } + : { status: "missing" }, + readBefore: async (path) => files.has(path) + ? { status: "found", content: files.get(path) } + : { status: "missing" }, + readAfter: async (path) => files.has(path) + ? { status: "found", content: files.get(path) } + : { status: "missing" }, + exists: async (path) => files.has(path), + hasOverlay: () => false, + overlayFor: () => undefined + } + }; +} diff --git a/tests/validation-python.test.mjs b/tests/validation-python.test.mjs index 77d6681..897254c 100644 --- a/tests/validation-python.test.mjs +++ b/tests/validation-python.test.mjs @@ -15,6 +15,7 @@ import { createPythonValidationAdapterStatus, createPythonValidationChecks as createCanonicalPythonValidationChecks, createNodePythonProjectWorkspace, + createValidationFileViewPythonWorkspace, createSyntaxCheck, createTypeCheck, isPythonSourcePath, @@ -25,6 +26,10 @@ import { const repoRoot = dirname(fileURLToPath(import.meta.url)); const validationFixtureRoot = join(repoRoot, "../packages/fixtures/validation-python"); +const sourceExtractionPythonFixtureRoot = join(repoRoot, "../packages/fixtures/source-extraction/python"); +const sourceExtractionPythonExpected = JSON.parse( + readFileSync(join(sourceExtractionPythonFixtureRoot, "python.expected.json"), "utf8") +); describe("validation-python adapter", () => { it("keeps compiler-truth fixture resources out of repository Python source discovery", () => { @@ -170,7 +175,7 @@ describe("validation-python adapter", () => { } }); - it("materializes absolute imports through the owning project's source roots", async () => { + it("materializes canonical transitive imports through the owning project's source roots", async () => { const repoRoot = mkdtempSync(join(tmpdir(), "opcore-python-types-source-root-")); try { const projectRoot = join(repoRoot, "services/api"); @@ -182,27 +187,50 @@ describe("validation-python adapter", () => { [ "#!/bin/sh", "if [ \"$1\" = \"--version\" ]; then echo 'mypy 1.8.0'; exit 0; fi", - "if [ ! -f src/acme/dep.py ]; then echo 'dependency was not materialized' >&2; exit 9; fi", + "if [ -f ../__init__.py ] || [ -f ../leak.py ]; then echo 'initializer ascent escaped the source root' >&2; exit 9; fi", + "if [ ! -f src/acme/__init__.py ]; then echo 'package initializer was not materialized' >&2; exit 9; fi", + "if [ ! -f src/acme/models.py ]; then echo 'initializer dependency was not materialized' >&2; exit 9; fi", + "if [ ! -f src/acme/mid.py ]; then echo 'direct dependency was not materialized' >&2; exit 9; fi", + "if [ ! -f src/acme/dep.py ]; then echo 'transitive dependency was not materialized' >&2; exit 9; fi", + "if [ ! -f ../../libs/shared/src/shared/__init__.py ]; then echo 'cross-project initializer was not materialized' >&2; exit 9; fi", + "if [ ! -f ../../libs/shared/src/shared/bootstrap.py ]; then echo 'cross-project initializer dependency was not materialized' >&2; exit 9; fi", "exit 0", "" ].join("\n") ); const files = { + "services/__init__.py": "from . import leak\n", + "services/leak.py": "VALUE = 'outside source root'\n", + "libs/shared/pyproject.toml": "[project]\nname='shared'\n", + "libs/shared/src/shared/__init__.py": "from . import bootstrap\n", + "libs/shared/src/shared/bootstrap.py": "READY = True\n", + "libs/shared/src/shared/models.py": "class SharedModel: pass\n", "services/api/pyproject.toml": "[project]\nname='api'\n[tool.mypy]\n", - "services/api/src/acme/app.py": "from acme import dep\nVALUE = dep.VALUE\n", - "services/api/src/acme/dep.py": "VALUE: int = 1\n" + "services/api/src/acme/__init__.py": "from .models import Model\n", + "services/api/src/acme/ns/app.py": "from acme import (\n mid,\n)\nfrom shared import models as shared_models\nVALUE = mid.VALUE\n", + "services/api/src/acme/mid.py": "from acme import dep\nVALUE = dep.VALUE\n", + "services/api/src/acme/dep.py": "VALUE: int = 1\n", + "services/api/src/acme/models.py": "class Model: pass\n" }; const result = await runner({ files, checks: createPythonValidationChecks({ repoRoot, env: { PATH: "" }, + importAnalyzer: fixedImportAnalyzer([ + { fromPath: "services/__init__.py", toPath: "services/leak.py" }, + { fromPath: "services/api/src/acme/ns/app.py", toPath: "services/api/src/acme/mid.py" }, + { fromPath: "services/api/src/acme/ns/app.py", toPath: "libs/shared/src/shared/models.py" }, + { fromPath: "services/api/src/acme/mid.py", toPath: "services/api/src/acme/dep.py" }, + { fromPath: "services/api/src/acme/__init__.py", toPath: "services/api/src/acme/models.py" }, + { fromPath: "libs/shared/src/shared/__init__.py", toPath: "libs/shared/src/shared/bootstrap.py" }, + ]), nodeWorkspace: projectWorkspace(files, () => true) }) }).runValidation(request({ repo: { repoRoot }, checks: [PYTHON_TYPES_CHECK_ID], - scope: { kind: "files", files: ["services/api/src/acme/app.py"] } + scope: { kind: "files", files: ["services/api/src/acme/ns/app.py"] } })); assert.equal(result.status, "passed", JSON.stringify(result, null, 2)); @@ -278,7 +306,11 @@ describe("validation-python adapter", () => { }; const result = await runner({ files, - checks: createPythonValidationChecks({ env: { PATH: "" }, repoRoot }) + checks: createPythonValidationChecks({ + env: { PATH: "" }, + repoRoot, + importAnalyzer: fixedImportAnalyzer([{ fromPath: "script.py", toPath: "helper.py" }]) + }) }).runValidation(request({ repo: { repoRoot }, checks: [PYTHON_TYPES_CHECK_ID], @@ -535,7 +567,7 @@ describe("validation-python adapter", () => { }) ); - assert.equal(result.status, "passed"); + assert.equal(result.status, "passed", JSON.stringify(result, null, 2)); assert.deepEqual(result.diagnostics, []); }); @@ -746,6 +778,9 @@ describe("validation-python adapter", () => { "pkg/app.py": "from .dep import value\nresult = value\n", "pkg/dep.py": "value = 1\n" }, + checks: createPythonValidationChecks({ + importAnalyzer: fixedImportAnalyzer([{ fromPath: "pkg/app.py", toPath: "pkg/dep.py" }]) + }), graphProviderClient: graphClient({ factQuery: (query) => availableFactResult(query, [], []) }) @@ -761,6 +796,230 @@ describe("validation-python adapter", () => { assert.match(result.diagnostics[0].message, /pkg\/app\.py -> pkg\/dep\.py/); }); + it("uses one canonical analyzer pass over the complete after-state for import graph validation", async () => { + const analysisCalls = []; + const importAnalyzer = { + async analyze(files) { + analysisCalls.push(files); + return [{ fromPath: "pkg/app.py", toPath: "pkg/new.py" }]; + } + }; + const graphEdges = [ + { kind: "IMPORTS_FROM", from: "file:pkg/app.py", to: "file:pkg/new.py" } + ]; + const result = await runner({ + files: { + "pkg/__init__.py": "", + "pkg/app.py": "from pkg import old\n", + "pkg/old.py": "OLD = True\n" + }, + checks: createPythonValidationChecks({ importAnalyzer }), + graphProviderClient: graphClient({ + factQuery: (query) => availableFactResult(query, [], graphEdges) + }) + }).runValidation(request({ + checks: [PYTHON_IMPORT_GRAPH_CHECK_ID], + scope: { kind: "files", files: ["pkg/app.py"] }, + overlays: [{ + path: "pkg/app.py", + action: "write", + content: "from pkg import (\n new,\n)\n" + }, { + path: "pkg/new.py", + action: "write", + content: "NEW = True\n" + }] + })); + + assert.equal(result.status, "passed", JSON.stringify(result, null, 2)); + assert.deepEqual(result.diagnostics, []); + assert.equal(analysisCalls.length, 1); + assert.deepEqual(analysisCalls[0].map((file) => file.path), [ + "pkg/__init__.py", + "pkg/app.py", + "pkg/new.py", + "pkg/old.py" + ]); + assert.equal(analysisCalls[0].find((file) => file.path === "pkg/app.py").content, "from pkg import (\n new,\n)\n"); + }); + + it("fails import graph validation closed when the canonical analyzer is missing", async () => { + const checks = createCanonicalPythonValidationChecks({ + nodeWorkspace: canonicalTestPythonWorkspace() + }); + const result = await runner({ + files: { "pkg/app.py": "VALUE = 1\n" }, + checks, + graphProviderClient: graphClient() + }).runValidation(request({ + checks: [PYTHON_IMPORT_GRAPH_CHECK_ID], + scope: { kind: "files", files: ["pkg/app.py"] } + })); + + assert.equal(result.status, "infrastructure_failure"); + assert.match(JSON.stringify(result), /python import analyzer/i); + }); + + it("runs graph-only dead-code and relevant-test checks without an import analyzer", async () => { + const checks = createCanonicalPythonValidationChecks({ + nodeWorkspace: canonicalTestPythonWorkspace() + }); + const result = await runner({ + files: { "pkg/app.py": "VALUE = 1\n" }, + checks, + graphProviderClient: graphClient() + }).runValidation(request({ + checks: [PYTHON_DEAD_CODE_CHECK_ID, PYTHON_RELEVANT_TESTS_CHECK_ID], + scope: { kind: "files", files: ["pkg/app.py"] } + })); + + assert.equal(result.status, "passed", JSON.stringify(result, null, 2)); + assert.deepEqual(result.diagnostics.map((diagnostic) => diagnostic.code).sort(), [ + "PY_DEAD_CODE_UNSUPPORTED", + "PY_RELEVANT_TESTS_ABSENT" + ]); + }); + + it("fails import-dependent validation closed for throwing and malformed analyzers", async (testContext) => { + const cases = [ + { + name: "throwing", + analyzer: { analyze: async () => { throw new Error("native analysis failed"); } }, + message: /native analysis failed/i + }, + { + name: "non-array", + analyzer: { analyze: async () => ({ edges: [] }) }, + message: /expected an array/i + }, + { + name: "outside after-state", + analyzer: fixedImportAnalyzer([{ fromPath: "pkg/app.py", toPath: "pkg/missing.py" }]), + message: /outside the supplied after-state/i + } + ]; + + for (const testCase of cases) { + await testContext.test(testCase.name, async () => { + const result = await runner({ + files: { "pkg/app.py": "VALUE = 1\n" }, + checks: createPythonValidationChecks({ importAnalyzer: testCase.analyzer }), + graphProviderClient: graphClient() + }).runValidation(request({ + checks: [PYTHON_IMPORT_GRAPH_CHECK_ID], + scope: { kind: "files", files: ["pkg/app.py"] } + })); + + assert.equal(result.status, "infrastructure_failure"); + assert.match(result.failure.cause, testCase.message); + }); + } + }); + + it("excludes deleted Python files from canonical after-state analysis", async () => { + const calls = []; + const result = await runner({ + files: { + "pkg/app.py": "from .dep import VALUE\n", + "pkg/dep.py": "VALUE = 1\n" + }, + checks: createPythonValidationChecks({ + importAnalyzer: { + async analyze(files) { + calls.push(files); + return []; + } + } + }), + graphProviderClient: graphClient() + }).runValidation(request({ + checks: [PYTHON_IMPORT_GRAPH_CHECK_ID], + scope: { kind: "files", files: ["pkg/app.py"] }, + overlays: [{ path: "pkg/dep.py", action: "delete" }] + })); + + assert.equal(result.status, "passed"); + assert.deepEqual(calls[0].map((file) => file.path), ["pkg/app.py"]); + assert.deepEqual(result.diagnostics, []); + }); + + it("skips deleted scoped Python targets before project or type-tool resolution", async () => { + const result = await runner({ + files: { "pkg/app.py": "VALUE = 1\n" }, + checks: createPythonValidationChecks({ importAnalyzer: fixedImportAnalyzer([]) }) + }).runValidation(request({ + checks: [PYTHON_SYNTAX_CHECK_ID, PYTHON_TYPES_CHECK_ID], + scope: { kind: "files", files: ["pkg/app.py"] }, + overlays: [{ path: "pkg/app.py", action: "delete" }] + })); + + assert.equal(result.status, "passed", JSON.stringify(result, null, 2)); + assert.deepEqual(result.pythonProjectContexts, []); + }); + + it("shares one canonical source resolver across graph-dependent Python checks", async () => { + let analysisCalls = 0; + const checks = createPythonValidationChecks({ + importAnalyzer: { + async analyze() { + analysisCalls += 1; + return []; + } + } + }); + const result = await runner({ + files: { "pkg/app.py": "VALUE = 1\n" }, + checks, + graphProviderClient: graphClient() + }).runValidation(request({ + checks: [PYTHON_IMPORT_GRAPH_CHECK_ID, PYTHON_DEAD_CODE_CHECK_ID, PYTHON_RELEVANT_TESTS_CHECK_ID], + scope: { kind: "files", files: ["pkg/app.py"] } + })); + + assert.equal(result.status, "passed"); + assert.equal(analysisCalls, 1); + }); + + it("consumes the shared canonical Python import edge matrix and reports only a genuinely omitted edge", async () => { + const files = fixtureFilesAt(sourceExtractionPythonFixtureRoot); + const expectedEdges = sourceExtractionPythonExpected.pythonImportEdges; + const graphEdges = expectedEdges.map(({ fromPath, toPath }) => ({ + kind: "IMPORTS_FROM", + from: `file:${fromPath}`, + to: `file:${toPath}` + })); + const scopeFiles = [...new Set(expectedEdges.map((edge) => edge.fromPath))].sort(); + const checks = createPythonValidationChecks({ importAnalyzer: fixedImportAnalyzer(expectedEdges) }); + const complete = await runner({ + files, + checks, + graphProviderClient: graphClient({ + factQuery: (query) => availableFactResult(query, [], graphEdges) + }) + }).runValidation(request({ checks: [PYTHON_IMPORT_GRAPH_CHECK_ID], scope: { kind: "files", files: scopeFiles } })); + + assert.equal(complete.status, "passed"); + assert.deepEqual(complete.diagnostics, []); + + const omitted = expectedEdges.find((edge) => edge.fromPath === "src/cases/alias_case.py"); + assert.ok(omitted); + const incomplete = await runner({ + files, + checks, + graphProviderClient: graphClient({ + factQuery: (query) => availableFactResult( + query, + [], + graphEdges.filter((edge) => edge.from !== `file:${omitted.fromPath}` || edge.to !== `file:${omitted.toPath}`) + ) + }) + }).runValidation(request({ checks: [PYTHON_IMPORT_GRAPH_CHECK_ID], scope: { kind: "files", files: scopeFiles } })); + + assert.deepEqual(incomplete.diagnostics.map((diagnostic) => diagnostic.code), ["PY_IMPORT_GRAPH_MISSING_EDGE"]); + assert.equal(incomplete.diagnostics[0].path, omitted.fromPath); + assert.match(incomplete.diagnostics[0].message, new RegExp(`${omitted.fromPath} -> ${omitted.toPath}`)); + }); + it("reports exported callable Python symbols without incoming CALLS as unused exports", async () => { const result = await runner({ files: { @@ -891,6 +1150,9 @@ describe("validation-python adapter", () => { it("reports syntax, hygiene, and import graph diagnostics from failing Python validation fixture", async () => { const result = await runner({ files: fixtureFiles("failing"), + checks: createPythonValidationChecks({ + importAnalyzer: fixedImportAnalyzer([{ fromPath: "pkg/app.py", toPath: "pkg/dep.py" }]) + }), graphProviderClient: graphClient({ factQuery: (query) => availableFactResult(query, [], []) }) @@ -925,6 +1187,72 @@ describe("validation-python adapter", () => { }); describe("Python project-context resolver", () => { + it("filters non-Python paths without reading file contents from matching after-state indexes", async () => { + const indexedPaths = ["large-native-binary", "pkg/app.py", "pyproject.toml"]; + const workspace = createValidationFileViewPythonWorkspace({ + overlays: [], + scopeFiles: ["pkg/app.py"], + defaultReadState: "after", + listVisibleFiles: async () => indexedPaths, + readFile: async () => { throw new Error("unexpected file content read"); }, + readBefore: async () => { throw new Error("unexpected file content read"); }, + readAfter: async () => { throw new Error("unexpected file content read"); }, + exists: async () => { throw new Error("unexpected file content read"); }, + hasOverlay: () => false, + overlayFor: () => undefined + }, undefined, { + read: async () => undefined, + list: async () => indexedPaths, + exists: async () => true, + realpath: async (path) => ({ path, symlink: false }), + executableExists: async () => true + }); + + assert.deepEqual(await workspace.list(), ["pkg/app.py", "pyproject.toml"]); + }); + + it("keeps before-state project boundaries independent of current files and overlays", async () => { + const before = new Map([ + ["pyproject.toml", "[project]\nname='root'\n"], + ["services/api/app.py", "VALUE = 1\n"] + ]); + const current = { + "services/api/pyproject.toml": "[project]\nname='api'\n", + "services/api/app.py": "VALUE = 2\n", + "large-native-binary": "not Python project evidence", + }; + const overlays = [ + { path: "pyproject.toml", action: "delete" }, + { path: "services/api/pyproject.toml", action: "write", content: current["services/api/pyproject.toml"] } + ]; + const readBefore = async (path) => + before.has(path) ? { status: "found", content: before.get(path) } : { status: "missing" }; + const workspace = createValidationFileViewPythonWorkspace({ + overlays, + scopeFiles: ["services/api/app.py"], + defaultReadState: "before", + listVisibleFiles: async () => [...new Set([...before.keys(), ...Object.keys(current)])].sort(), + readFile: readBefore, + readBefore, + readAfter: readBefore, + exists: async (path) => { + if (path === "large-native-binary") throw new Error("irrelevant binary existence was probed"); + return before.has(path); + }, + hasOverlay: (path) => overlays.some((overlay) => overlay.path === path), + overlayFor: (path) => overlays.find((overlay) => overlay.path === path) + }, undefined, projectWorkspace(current, () => true)); + + assert.deepEqual(await workspace.list(), [...before.keys()].sort()); + const context = await resolvePythonProjectContext({ + repoRoot: "/fixture", + target: "services/api/app.py", + workspace, + processProbe: successfulProbe() + }); + assert.equal(context.projectRoot, "."); + }); + it("resolves nearest nested projects, layouts, managers, and stable fingerprints", async () => { const files = { "pyproject.toml": "[project]\nname='root'\nrequires-python='>=3.11'\n", @@ -1953,6 +2281,7 @@ function runner(options = {}) { function createPythonValidationChecks(options = {}) { return createCanonicalPythonValidationChecks({ ...options, + importAnalyzer: options.importAnalyzer ?? { analyze: async () => [] }, nodeWorkspace: options.nodeWorkspace ?? canonicalTestPythonWorkspace() }); } @@ -1978,6 +2307,10 @@ function projectWorkspace(files, executableExists, symlinks = new Set()) { }; } +function fixedImportAnalyzer(edges) { + return { analyze: async () => edges }; +} + function successfulProbe(version = "3.12.4") { return { resolveExecutable(command) { @@ -2097,6 +2430,7 @@ function workspace(options = {}) { const files = new Map(Object.entries(options.files ?? { "pkg/app.py": "value = 1\n" })); return { readFile: (path) => (files.has(path) ? { status: "found", content: files.get(path) } : { status: "missing" }), + listFiles: () => ({ files: [...files.keys()] }), listChangedFiles: () => ({ files: [...files.keys()] }), listStagedFiles: () => ({ files: [...files.keys()] }), listRepoFiles: () => ({ files: [...files.keys()] }), @@ -2108,6 +2442,10 @@ function workspace(options = {}) { function fixtureFiles(name) { const root = join(validationFixtureRoot, name); + return fixtureFilesAt(root); +} + +function fixtureFilesAt(root) { const entries = {}; for (const path of walkFiles(root)) { const fixturePath = relative(root, path).replaceAll("\\", "/");