feat: Extending cPython builds to match all the interpreter envs

Author: jakub-kocka

.github/workflows/build-wheels-python-dependent.yml

@@ -18,6 +18,8 @@ jobs:
     runs-on: ${{ matrix.runner }}
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      # PyO3 (cryptography, etc.): allow building against CPython newer than PyO3's declared max when using stable ABI
+      PYO3_USE_ABI3_FORWARD_COMPATIBILITY: "1"
     strategy:
       fail-fast: false
       matrix:
@@ -148,7 +150,7 @@ jobs:
               bash os_dependencies/linux_arm.sh
               # Source Rust environment after installation
               . \$HOME/.cargo/env
-              python build_wheels_from_file.py dependent_requirements_${{ matrix.arch }}
+              python build_wheels_from_file.py --force-interpreter-binary dependent_requirements_${{ matrix.arch }}
             "
 
       - name: Build Python dependent wheels - ARMv7 Legacy (in Docker)
@@ -171,7 +173,7 @@ jobs:
               bash os_dependencies/linux_arm.sh
               # Source Rust environment after installation
               . \$HOME/.cargo/env
-              python build_wheels_from_file.py dependent_requirements_${{ matrix.arch }}
+              python build_wheels_from_file.py --force-interpreter-binary dependent_requirements_${{ matrix.arch }}
             "
 
       - name: Build Python dependent wheels - Linux/macOS
@@ -184,11 +186,11 @@ jobs:
               export ARCHFLAGS="-arch x86_64"
             fi
 
-            python build_wheels_from_file.py dependent_requirements_${{ matrix.arch }}
+            python build_wheels_from_file.py --force-interpreter-binary dependent_requirements_${{ matrix.arch }}
 
       - name: Build Python dependent wheels for ${{ matrix.python-version }} - Windows
         if: matrix.os == 'Windows'
-        run: python build_wheels_from_file.py dependent_requirements_${{ matrix.arch }}
+        run: python build_wheels_from_file.py --force-interpreter-binary dependent_requirements_${{ matrix.arch }}
 
 
       - name: Upload artifacts

_helper_functions.py

@@ -107,11 +107,23 @@ def get_no_binary_args(requirement_name: str) -> list:
     return []
 
 
+def _safe_text_for_stdout(text: str) -> str:
+    """Avoid UnicodeEncodeError when printing pip/tool output on Windows (e.g. cp1252 console)."""
+    encoding = getattr(sys.stdout, "encoding", None) or "utf-8"
+    if encoding.lower() in ("utf-8", "utf8"):
+        return text
+    try:
+        text.encode(encoding)
+        return text
+    except UnicodeEncodeError:
+        return text.encode(encoding, errors="replace").decode(encoding, errors="replace")
+
+
 def print_color(text: str, color: str = Fore.BLUE):
     """Print colored text specified by color argument based on colorama
     - default color BLUE
     """
-    print(f"{color}", f"{text}", Style.RESET_ALL)
+    print(f"{color}", f"{_safe_text_for_stdout(text)}", Style.RESET_ALL)
 
 
 def merge_requirements(requirement: Requirement, another_req: Requirement) -> Requirement:

build_wheels_from_file.py

@@ -3,16 +3,65 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
+from __future__ import annotations
+
 import argparse
 import os
+import platform
 import subprocess
 import sys
 
 from colorama import Fore
+from packaging.requirements import InvalidRequirement
+from packaging.requirements import Requirement
+from packaging.utils import canonicalize_name
 
 from _helper_functions import get_no_binary_args
 from _helper_functions import print_color
 
+# Do not pass --no-binary for these in --force-interpreter-binary mode:
+# - sdists whose legacy setup breaks under PEP 517 isolation (pkg_resources in isolated env).
+# - sdists that fail to compile on CI when a usable wheel exists (e.g. ruamel.yaml.clib + clang).
+# - PyObjC: all pyobjc / pyobjc-framework-* use pyobjc_setup.py + pkg_resources (macOS).
+# - cryptography: abi3 wheels; avoid PyO3 max-Python / heavy Rust rebuilds in dependent jobs.
+# - pydantic-core: maturin + jiter + PyO3 can fail from sdist on some CI combos (e.g. ARM64 3.9:
+#   jiter vs pyo3-ffi PyUnicode_* / extract API). Prefer compatible wheels from find-links or PyPI.
+_FORCE_INTERPRETER_BINARY_SKIP_EXACT = frozenset(
+    {
+        canonicalize_name("cryptography"),
+        canonicalize_name("pydantic-core"),
+        canonicalize_name("protobuf"),
+        canonicalize_name("ruamel.yaml.clib"),
+    }
+)
+
+
+def _force_interpreter_skip_package(canonical_dist_name: str) -> bool:
+    if canonical_dist_name in _FORCE_INTERPRETER_BINARY_SKIP_EXACT:
+        return True
+    # PyObjC meta and framework bindings (pyobjc-framework-corebluetooth, etc.)
+    return canonical_dist_name == "pyobjc" or canonical_dist_name.startswith("pyobjc-")
+
+
+def _force_interpreter_no_binary_args(requirement_line: str) -> list[str]:
+    """Return pip --no-binary for this package so pip cannot reuse e.g. cp311-abi3 wheels on 3.13."""
+    line = requirement_line.strip()
+    if not line:
+        return []
+    try:
+        req = Requirement(line)
+    except InvalidRequirement:
+        return []
+    if _force_interpreter_skip_package(canonicalize_name(req.name)):
+        return []
+    return ["--no-binary", req.name]
+
+
+def _apply_force_interpreter_binary(cli_flag: bool) -> bool:
+    """Linux/macOS only: forcing sdist builds for cryptography etc. is unreliable on Windows CI."""
+    return cli_flag and platform.system() != "Windows"
+
+
 parser = argparse.ArgumentParser(description="Process build arguments.")
 parser.add_argument(
     "requirements_path",
@@ -36,6 +85,16 @@
     action="store_true",
     help="CI exclude-tests mode: fail if all wheels succeed (expect some to fail, e.g. excluded packages)",
 )
+parser.add_argument(
+    "--force-interpreter-binary",
+    action="store_true",
+    help=(
+        "For each requirement, pass --no-binary <pkg> so pip builds a wheel for the current "
+        "interpreter instead of reusing a compatible abi3 / older cpXY wheel from --find-links. "
+        "Ignored on Windows (source builds for e.g. cryptography are not used in CI there). "
+        "Some packages are always skipped (e.g. cryptography, pydantic-core, protobuf, PyObjC, ruamel.yaml.clib)."
+    ),
+)
 
 args = parser.parse_args()
 
@@ -55,22 +114,31 @@
         raise SystemExit(f"Python version dependent requirements directory or file not found ({e})")
 
     for requirement in requirements:
+        requirement = requirement.strip()
+        if not requirement or requirement.startswith("#"):
+            continue
         # Get no-binary args for packages that should be built from source
         no_binary_args = get_no_binary_args(requirement)
+        force_interpreter_args = (
+            _force_interpreter_no_binary_args(requirement)
+            if _apply_force_interpreter_binary(args.force_interpreter_binary)
+            else []
+        )
 
         out = subprocess.run(
             [
                 f"{sys.executable}",
                 "-m",
                 "pip",
                 "wheel",
-                f"{requirement}",
+                requirement,
                 "--find-links",
                 "downloaded_wheels",
                 "--wheel-dir",
                 "downloaded_wheels",
             ]
-            + no_binary_args,
+            + no_binary_args
+            + force_interpreter_args,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )
@@ -100,6 +168,11 @@
     for requirement in in_requirements:
         # Get no-binary args for packages that should be built from source
         no_binary_args = get_no_binary_args(requirement)
+        force_interpreter_args = (
+            _force_interpreter_no_binary_args(requirement)
+            if _apply_force_interpreter_binary(args.force_interpreter_binary)
+            else []
+        )
 
         out = subprocess.run(
             [
@@ -113,7 +186,8 @@
                 "--wheel-dir",
                 "downloaded_wheels",
             ]
-            + no_binary_args,
+            + no_binary_args
+            + force_interpreter_args,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )