From 39a8d9c9715b0f43baf0707290f17288a8506beb Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Wed, 1 Jul 2026 15:58:16 -0700 Subject: [PATCH] feat: Add --allow-build flag & tool.pywrangler setting By default we pass `--no-build` because building a binary Pyodide wheel will fail in a confusing way. This makes it annoying to work with local packages though. This adds a `--allow-build` setting to override. --- packages/cli/src/pywrangler/cli.py | 14 ++- packages/cli/src/pywrangler/resolve.py | 42 +++++-- packages/cli/src/pywrangler/sync.py | 45 +++++--- packages/cli/src/pywrangler/utils.py | 10 ++ packages/cli/tests/test_cli.py | 146 +++++++++++++++++++++++++ 5 files changed, 233 insertions(+), 24 deletions(-) diff --git a/packages/cli/src/pywrangler/cli.py b/packages/cli/src/pywrangler/cli.py index 73190bf..f196f03 100644 --- a/packages/cli/src/pywrangler/cli.py +++ b/packages/cli/src/pywrangler/cli.py @@ -132,14 +132,24 @@ def types_command(outdir: str | None, config: str | None) -> Never: is_flag=True, help="Allow package upgrades, ignoring pinned versions in pylock.toml", ) -def sync_command(force: bool = False, upgrade: bool = False) -> None: +@click.option( + "--allow-build/--no-allow-build", + default=None, + help=( + "Allow building source distributions and local directory sources. " + "Defaults to the [tool.pywrangler] allow-build setting in pyproject.toml." + ), +) +def sync_command( + force: bool = False, upgrade: bool = False, allow_build: bool | None = None +) -> None: """ Installs Python packages from pyproject.toml into src/vendor. Also creates a virtual env for Workers that you can use for testing. """ - sync(force, directly_requested=True, upgrade=upgrade) + sync(force, directly_requested=True, upgrade=upgrade, allow_build=allow_build) write_success("Sync process completed successfully.") diff --git a/packages/cli/src/pywrangler/resolve.py b/packages/cli/src/pywrangler/resolve.py index 7cd9874..0de2198 100644 --- a/packages/cli/src/pywrangler/resolve.py +++ b/packages/cli/src/pywrangler/resolve.py @@ -18,21 +18,40 @@ class InstallPlan: + # Lockfile keys that indicate a package is sourced locally + _LOCAL_SOURCE_KEYS = ("directory", "sdist", "archive") + def __init__(self, lockfile: Path) -> None: self.lockfile = lockfile self.requirements: list[tuple[str, str]] = [] + # Names of packages sourced from a local path. They need refreshing when + # rebuilt. + self.local_packages: list[str] = [] with open(lockfile, "rb") as f: data = tomllib.load(f) for pkg in data.get("packages", []): name = pkg.get("name") + if name and any( + self._is_local_source(pkg, key) for key in self._LOCAL_SOURCE_KEYS + ): + self.local_packages.append(name) + version = pkg.get("version") if not name or not version: logger.warning("Skipping malformed lockfile entry: %s", pkg) continue self.requirements.append((name, version)) + @staticmethod + def _is_local_source(pkg: dict, key: str) -> bool: + source = pkg.get(key) + if not isinstance(source, dict): + return False + # A local reference has a `path` + return "path" in source + def to_requirement_strings(self) -> list[str]: return [f"{name}=={version}" for name, version in self.requirements] @@ -49,12 +68,18 @@ def _compile_lockfile( lockfile_path: Path, *, upgrade: bool = False, + allow_build: bool = False, ) -> None: """Run ``uv pip compile`` targeting Pyodide. Writes the compiled output to *lockfile_path*. When *lockfile_path* already exists, ``uv pip compile`` uses it as a constraint source so pinned versions are preserved across re-runs (no silent upgrades). + + By default ``--no-build`` is passed so only prebuilt wheels are used. This + is because building a Pyodide platformed wheel will fail. Set *allow_build* + to permit building source distributions / local directory sources. This is + useful for testing against local checkouts of pure Python packages. """ with temp_requirements_file(requirements) as req_in_path: cmd = [ @@ -68,31 +93,34 @@ def _compile_lockfile( get_pyodide_index(), "--index-strategy", "unsafe-best-match", - "--no-build", "--no-header", "-o", str(lockfile_path), ] + if not allow_build: + cmd.append("--no-build") if upgrade: cmd.append("--upgrade") run_command(cmd, cwd=get_project_root(), capture_output=True) -def resolve_requirements(*, upgrade: bool = False) -> InstallPlan: +def resolve_requirements( + *, upgrade: bool = False, allow_build: bool = False +) -> InstallPlan: """Build an InstallPlan by compiling dependencies for the Pyodide target. - Runs ``uv pip compile`` with the Pyodide interpreter and ``--no-build`` - to resolve versions that have Pyodide wheels. The compiled output is - written to ``pylock.toml``; on subsequent runs the existing file - constrains versions so they don't drift. + Runs ``uv pip compile`` with the Pyodide interpreter and ``--no-build`` to + resolve versions that have Pyodide wheels. The compiled output is written + to ``pylock.toml``; on subsequent runs the existing file constrains versions + so they don't drift. """ lockfile = get_lockfile_path() deps = parse_requirements() deps.append(MANAGED_SDK_PACKAGE) - _compile_lockfile(deps, lockfile, upgrade=upgrade) + _compile_lockfile(deps, lockfile, upgrade=upgrade, allow_build=allow_build) plan = InstallPlan(lockfile) logger.info("Resolved %d requirements from %s.", len(plan.requirements), lockfile) diff --git a/packages/cli/src/pywrangler/sync.py b/packages/cli/src/pywrangler/sync.py index ce9a14b..354bf2b 100644 --- a/packages/cli/src/pywrangler/sync.py +++ b/packages/cli/src/pywrangler/sync.py @@ -16,6 +16,7 @@ get_lockfile_path, get_project_root, get_python_version, + get_pywrangler_config, get_pywrangler_version, get_uv_pyodide_interp_name, run_command, @@ -145,9 +146,15 @@ def create_pyodide_venv() -> None: run_command(["uv", "venv", str(pyodide_venv_path), "--python", interp_name]) -def _install_requirements_to_vendor(plan: InstallPlan) -> str | None: +def _install_requirements_to_vendor( + plan: InstallPlan, allow_build: bool = False +) -> str | None: """Install packages to the Pyodide vendor directory from pylock.toml. + By default ``--no-build`` is passed so only prebuilt wheels install. When + *allow_build* is True, source distributions / local directory sources are + allowed to build. + Returns: Error message string if installation failed, None if successful. """ @@ -179,17 +186,18 @@ def _install_requirements_to_vendor(plan: InstallPlan) -> str | None: shutil.rmtree(pyodide_site_packages) pyodide_site_packages.mkdir() + install_cmd = ["uv", "pip", "install"] + if not allow_build: + install_cmd.append("--no-build") + else: + # uv caches built wheels for local sources keyed on their path, so edits + # to local checkouts wouldn't be picked up. Refresh the build cache for + # those packages so `sync` always rebuilds them. + for name in plan.local_packages: + install_cmd += ["--refresh-package", name] + install_cmd += ["-r", str(plan.lockfile), "--preview-features", "pylock"] result = run_command( - [ - "uv", - "pip", - "install", - "--no-build", - "-r", - str(plan.lockfile), - "--preview-features", - "pylock", - ], + install_cmd, capture_output=True, check=False, env=os.environ | {"VIRTUAL_ENV": str(get_pyodide_venv_path())}, @@ -290,10 +298,10 @@ def _get_vendor_package_versions() -> list[str]: return _parse_pip_freeze(result.stdout) -def install_requirements(plan: InstallPlan) -> None: +def install_requirements(plan: InstallPlan, allow_build: bool = False) -> None: # First, install to the Pyodide vendor directory. This determines the exact package # versions that will run in production. - pyodide_error = _install_requirements_to_vendor(plan) + pyodide_error = _install_requirements_to_vendor(plan, allow_build=allow_build) # Then install to .venv-workers using the pinned versions from vendor. # This ensures host packages accurately reflect what will run in production. @@ -399,10 +407,17 @@ def sync( force: bool = False, directly_requested: bool = False, upgrade: bool = False, + allow_build: bool | None = None, ) -> None: # Check if requirements.txt does not exist. check_requirements_txt() + # Resolve the build override: explicit CLI flag wins, otherwise fall back to + # `[tool.pywrangler] allow-build` in pyproject.toml so `dev`/`deploy` (which + # call sync() without flags) honor it too. + if allow_build is None: + allow_build = bool(get_pywrangler_config().get("allow-build", False)) + # Check if sync is needed based on file timestamps sync_needed = force or is_sync_needed() if not sync_needed: @@ -425,9 +440,9 @@ def sync( create_pyodide_venv() # Resolve dependencies via uv pip compile targeting Pyodide, then install into vendor folder. - plan = resolve_requirements(upgrade=upgrade) + plan = resolve_requirements(upgrade=upgrade, allow_build=allow_build) if not plan.requirements: logger.warning( "No dependencies found in [project.dependencies] section of pyproject.toml." ) - install_requirements(plan) + install_requirements(plan, allow_build=allow_build) diff --git a/packages/cli/src/pywrangler/utils.py b/packages/cli/src/pywrangler/utils.py index 79bb1ca..bffd303 100644 --- a/packages/cli/src/pywrangler/utils.py +++ b/packages/cli/src/pywrangler/utils.py @@ -228,6 +228,16 @@ def get_project_root() -> Path: return find_pyproject_toml().parent +def get_pywrangler_config() -> dict: + """Return the ``[tool.pywrangler]`` table from pyproject.toml (or ``{}``).""" + data = read_pyproject_toml() + tool = data.get("tool", {}) + if not isinstance(tool, dict): + return {} + config = tool.get("pywrangler", {}) + return config if isinstance(config, dict) else {} + + MIN_UV_VERSION = (0, 8, 10) MIN_WRANGLER_VERSION = (4, 42, 1) diff --git a/packages/cli/tests/test_cli.py b/packages/cli/tests/test_cli.py index dd65f5f..12a8852 100644 --- a/packages/cli/tests/test_cli.py +++ b/packages/cli/tests/test_cli.py @@ -30,6 +30,10 @@ def is_package_installed(site_packages_path, package_name): # Normalize package name (lowercase, remove dashes) package_name_normalized = package_name.lower().replace("-", "_") + if not site_packages_path.exists(): + print(f"{site_packages_path} does not exist") + return False + matches = list(site_packages_path.glob(f"*{package_name_normalized}*")) if matches: print(f"Found {package_name} as: {matches}") @@ -367,6 +371,148 @@ def test_sync_lockfile_lifecycle(test_dir): assert is_package_installed(vendor_path, "six") +def create_dummy_build_dep(parent_dir: Path, name: str = "dummy-build-dep") -> Path: + """Create a tiny pure-Python dependency that must be built from source. + + The package only exists as a local directory (no prebuilt wheel on any + index), so installing it requires ``uv`` to run its build backend. This is + exactly what ``--allow-build`` gates: with ``--no-build`` (the default) the + resolver refuses to build it, and with ``--allow-build`` it succeeds. + + Returns the path to the created dependency directory. + """ + module_name = name.replace("-", "_") + dep_dir = parent_dir / module_name + (dep_dir / module_name).mkdir(parents=True) + + (dep_dir / "pyproject.toml").write_text( + dedent(f""" + [build-system] + requires = ["setuptools>=61.0"] + build-backend = "setuptools.build_meta" + + [project] + name = "{name}" + version = "0.1.0" + description = "A tiny dummy dependency built from source" + requires-python = ">=3.8" + """) + ) + (dep_dir / module_name / "__init__.py").write_text( + 'MESSAGE = "hello from dummy build dep"\n' + ) + return dep_dir + + +def create_worker_pyproject_with_local_dep( + test_dir: Path, dep_dir: Path, dep_name: str, *, allow_build_config: bool = False +) -> None: + """Write a worker pyproject.toml that depends on a local directory source. + + The dependency is expressed as a PEP 508 direct reference to the local + directory (``name @ file://...``) so the resolver treats it as a + ``directory`` source that must be built from source. ``allow_build_config`` + toggles the ``[tool.pywrangler] allow-build`` key. + """ + pywrangler_table = ( + "[tool.pywrangler]\nallow-build = true\n" if allow_build_config else "" + ) + content = dedent(f""" + [build-system] + requires = ["setuptools>=61.0"] + build-backend = "setuptools.build_meta" + + [project] + name = "test-project" + version = "0.1.0" + description = "Test Project" + requires-python = ">=3.8" + dependencies = ["{dep_name} @ {dep_dir.as_uri()}"] + + {pywrangler_table} + """) + (test_dir / "pyproject.toml").write_text(content) + + +def test_sync_allow_build_local_dependency(test_dir): + """End-to-end test for --allow-build with a local source dependency. + + A tiny dummy package that only exists as a local directory (and therefore + must be built from source) is added to the worker's pyproject.toml. Syncing + without --allow-build must fail (default is --no-build), while syncing with + --allow-build must succeed and vendor the built package. + """ + dep_name = "dummy-build-dep" + dep_dir = create_dummy_build_dep(test_dir, dep_name) + create_worker_pyproject_with_local_dep(test_dir, dep_dir, dep_name) + create_test_wrangler_jsonc(test_dir, "src/worker.py") + + vendor_path = test_dir / "python_modules" + sync_cmd = ["uv", "run", "pywrangler", "sync"] + + # Without --allow-build: the default --no-build rejects the local source. + result = subprocess.run( + [*sync_cmd, "--no-allow-build"], + capture_output=True, + text=True, + cwd=test_dir, + check=False, + ) + assert result.returncode != 0, ( + "sync should fail without --allow-build because the local dependency " + "must be built from source" + ) + assert not is_package_installed(vendor_path, dep_name), ( + "dummy build dep should not be vendored when the build was rejected" + ) + + # With --allow-build: uv is allowed to build the local source. + result = subprocess.run( + [*sync_cmd, "--force", "--allow-build"], + capture_output=True, + text=True, + cwd=test_dir, + check=False, + ) + assert result.returncode == 0, ( + f"sync --allow-build failed: {result.stdout}\n{result.stderr}" + ) + assert is_package_installed(vendor_path, dep_name), ( + f"{dep_name} should be built and vendored into python_modules " + "when --allow-build is passed" + ) + + +def test_sync_allow_build_via_pyproject_config(test_dir): + """End-to-end test for the [tool.pywrangler] allow-build config fallback. + + When no CLI flag is passed, sync should honor `allow-build = true` in the + [tool.pywrangler] table of pyproject.toml. + """ + dep_name = "dummy-build-dep" + dep_dir = create_dummy_build_dep(test_dir, dep_name) + create_worker_pyproject_with_local_dep( + test_dir, dep_dir, dep_name, allow_build_config=True + ) + create_test_wrangler_jsonc(test_dir, "src/worker.py") + + vendor_path = test_dir / "python_modules" + result = subprocess.run( + ["uv", "run", "pywrangler", "sync"], + capture_output=True, + text=True, + cwd=test_dir, + check=False, + ) + assert result.returncode == 0, ( + f"sync failed with [tool.pywrangler] allow-build = true: " + f"{result.stdout}\n{result.stderr}" + ) + assert is_package_installed(vendor_path, dep_name), ( + f"{dep_name} should be vendored when allow-build is enabled via config" + ) + + def test_sync_command_handles_missing_pyproject(): """Test that the sync command correctly handles a missing pyproject.toml file.""" import tempfile