gh-122931 CI for Linux multiarch co-installability

.github/CODEOWNERS

@@ -150,6 +150,8 @@ Tools/build/generate_sbom.py  @sethmlarson
 # ABI check
 Misc/libabigail.abignore      @encukou
 
+# Multiarch
+Tools/coinstall-check/        @stefanor
 
 # ----------------------------------------------------------------------------
 # Platform Support

.github/workflows/build.yml

@@ -260,6 +260,7 @@ jobs:
       free-threading: ${{ matrix.free-threading }}
       os: ${{ matrix.os }}
       test-opts: ${{ matrix.test-opts || '' }}
+      upload-install-hashes: ${{ !matrix.bolt }}
 
   build-ubuntu-ssltests:
     name: 'Ubuntu SSL tests'
@@ -608,6 +609,26 @@ jobs:
         run: |
           "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed
 
+  linux-install-compare:
+    name: Ubuntu Co-install comparison
+    runs-on: ubuntu-latest
+    needs: build-ubuntu
+    steps:
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.x'
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - name: Download install hashes
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: install-hashes
+          pattern: install-hashes-*
+          merge-multiple: true
+      - name: Compare install hashes
+        run: python3 Tools/coinstall-check/compare.py install-hashes
+
   cifuzz:
     # ${{ '' } is a hack to nest jobs under the same sidebar category.
     name: CIFuzz${{ '' }}  # zizmor: ignore[obfuscation]
@@ -675,6 +696,7 @@ jobs:
     - build-san
     - cross-build-linux
     - cifuzz
+    - linux-install-compare
     if: always()
 
     steps:
@@ -721,6 +743,7 @@ jobs:
             build-asan,
             build-san,
             cross-build-linux,
+            linux-install-compare,
             '
             || ''
           }}

.github/workflows/reusable-ubuntu.yml

@@ -8,20 +8,30 @@ on:
         required: false
         type: boolean
         default: false
+      debug:
+        description: Whether to create a Debug Build
+        required: false
+        type: boolean
+        default: true
       free-threading:
         description: Whether to use free-threaded mode
         required: false
         type: boolean
         default: false
       os:
-         description: OS to run the job
-         required: true
-         type: string
+        description: OS to run the job
+        required: true
+        type: string
       test-opts:
-         description: Extra options to pass to the test runner via TESTOPTS
-         required: false
-         type: string
-         default: ''
+        description: Extra options to pass to the test runner via TESTOPTS
+        required: false
+        type: string
+        default: ''
+      upload-install-hashes:
+        description: Install Python and upload the result artifact
+        required: false
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -35,6 +45,7 @@ jobs:
     runs-on: ${{ inputs.os }}
     timeout-minutes: 60
     env:
+      INSTALL_HASHES_FILE: install-hashes-${{ inputs.os }}-${{ case(inputs.free-threading, 't', '')}}${{ case(inputs.debug, 'd', '') }}.json.gz
       OPENSSL_VER: 3.5.7
       PYTHONSTRICTEXTENSIONBUILD: 1
       TERM: linux
@@ -47,7 +58,7 @@ jobs:
     - name: Install dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: Install Clang and BOLT
-      if: ${{ fromJSON(inputs.bolt-optimizations) }}
+      if: fromJSON(inputs.bolt-optimizations)
       run: |
         sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 19
         sudo apt-get install --no-install-recommends bolt-19
@@ -84,14 +95,15 @@ jobs:
         PROFILE_TASK='-m test --pgo --ignore test_unpickle_module_race'
         ../cpython-ro-srcdir/configure
         --config-cache
-        --with-pydebug
         --enable-slower-safety
         --enable-safety
         --with-openssl="$OPENSSL_DIR"
+        ${{ fromJSON(inputs.debug) && '--with-pydebug' || '' }}
         ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}
         ${{ fromJSON(inputs.bolt-optimizations) && '--enable-bolt' || '' }}
+        ${{ fromJSON(inputs.upload-install-hashes) && '--prefix=/usr --libdir=/usr/lib/$(gcc --print-multiarch) --with-build-details-suffix' || '' }}
     - name: Build CPython out-of-tree
-      if: ${{ inputs.free-threading }}
+      if: inputs.free-threading
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: make -j
     - name: Build CPython out-of-tree (for compiler warning check)
@@ -119,3 +131,42 @@ jobs:
       run: xvfb-run make ci EXTRATESTOPTS="${TEST_OPTS}"
       env:
         TEST_OPTS: ${{ inputs.test-opts }}
+    - name: Install Python
+      if: inputs.upload-install-hashes
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: make install DESTDIR=install
+    - name: Install test C extension
+      if: inputs.upload-install-hashes
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      env:
+        CPYTHON_TEST_EXT_NAME: c_mod
+      run: install/usr/bin/python3 -m pip install ../cpython-ro-srcdir/Lib/test/test_cext
+    - name: Install test stable ABI extension
+      if: inputs.upload-install-hashes && !inputs.free-threading
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      env:
+        CPYTHON_TEST_EXT_NAME: abi3_mod
+        CPYTHON_TEST_LIMITED: 1
+      run: install/usr/bin/python3 -m pip install ../cpython-ro-srcdir/Lib/test/test_cext
+    - name: Install test free-threaded stable ABI extension
+      if: inputs.upload-install-hashes
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      env:
+        CPYTHON_TEST_EXT_NAME: abi3t_mod
+        CPYTHON_TEST_ABI3T: 1
+      run: install/usr/bin/python3 -m pip install ../cpython-ro-srcdir/Lib/test/test_cext
+    - name: Hash the installed Python
+      if: inputs.upload-install-hashes
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: >-
+        install/usr/bin/python3
+        ../cpython-ro-srcdir/Tools/coinstall-check/hash-r.py
+        install -o "$INSTALL_HASHES_FILE"
+    - name: Upload the installed Python hashes
+      if: inputs.upload-install-hashes
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+      with:
+        name: ${{ env.INSTALL_HASHES_FILE }}
+        path: ${{ env.CPYTHON_BUILDDIR }}/${{ env.INSTALL_HASHES_FILE }}
+        archive: false
+        retention-days: 1

Lib/test/test_build_details.py

@@ -130,7 +130,8 @@ def location(self):
                 dirname = os.path.join(projectdir, f.read())
         else:
             dirname = sysconfig.get_path('stdlib')
-        return os.path.join(dirname, 'build-details.json')
+        filename = sysconfig.get_config_var('BUILD_DETAILS')
+        return os.path.join(dirname, filename)
 
     @property
     def contents(self):

Lib/test/test_cext/setup.py

@@ -107,6 +107,8 @@ def main():
     if internal:
         cflags.append('-DTEST_INTERNAL_C_API=1')
 
+    py_limited_api = limited or abi3t
+
     # Add additional include and library directories, typically for in-tree
     # testing where not all directories are inferred
     include_dirs = []
@@ -131,7 +133,9 @@ def main():
         sources=sources,
         extra_compile_args=cflags,
         include_dirs=include_dirs,
-        library_dirs=library_dirs)
+        library_dirs=library_dirs,
+        py_limited_api=py_limited_api,
+    )
     setup(name=f'internal_{module_name}',
           version='0.0',
           ext_modules=[ext])

Misc/NEWS.d/next/Tests/2026-07-01-18-14-25.gh-issue-122931.OeRDeo.rst

@@ -0,0 +1,3 @@
+CI Tests to ensure Debian `multi-arch co-installability
+<https://www.debian.org/doc/debian-policy/ch-controlfields.html#multi-arch>`_
+of Python.

Tools/README

@@ -14,6 +14,9 @@ clinic          A preprocessor for CPython C files in order to automate
                 the boilerplate involved with writing argument parsing
                 code for "builtins".
 
+coinstall-check A tool to ensure that multiple CPython builds can be
+                co-installed on Linux.
+
 freeze          Create a stand-alone executable from a Python program.
 
 ftscalingbench  Benchmarks for free-threading and finding bottlenecks.

Tools/coinstall-check/compare.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+# Compare that multiple installs of Python don't have conflicting files
+#
+# This is a requirement for Debian's Multi-Arch installs of Python
+# https://www.debian.org/doc/debian-policy/ch-controlfields.html#multi-arch
+
+from argparse import ArgumentParser
+from typing import Any
+from pathlib import Path
+import gzip
+import json
+
+
+def compare_trees(base: Path) -> bool:
+    seen: dict[str, str] = {}
+    success: bool = True
+    for tree in base.iterdir():
+        if not tree.is_file():
+            continue
+
+        hashes: dict[str, str] = {}
+        print(f"Examining {tree}")
+        with gzip.open(tree) as f:
+            data = json.load(f)
+        build_details = data["build_details"]
+        hashes = data["hashes"]
+
+        for path, digest in hashes.items():
+            if is_ignored(path, build_details):
+                continue
+            if path not in seen:
+                seen[path] = digest
+                continue
+            if digest != seen[path]:
+                print(f"Mismatch found in {tree}: {path}")
+                print(f"{digest} != {seen[path]}")
+                success = False
+    return success
+
+
+def is_ignored(pathname: str, build_details: dict[str, Any]) -> bool:
+    """Is this a path that we should ignore?"""
+
+    path = Path(pathname)
+
+    if path.parent.name == "__pycache__":
+        # Includes a timestamp, we expect a mismatch
+        return True
+
+    if path.is_relative_to("usr/bin"):
+        # Only libraries are multi-arch co-installed, only one arch can
+        # have binaries in /usr/bin at a time.
+        return True
+
+    in_usr_include = path.is_relative_to("usr/include")
+    if in_usr_include and path.name == "pyconfig.h":
+        # Varies according to config, installed into a tag-specific
+        # include directory
+        return True
+
+    in_usr_lib = path.is_relative_to("usr/lib")
+    in_pkgconfig = in_usr_lib and path.parent.name == "pkgconfig"
+    if in_pkgconfig and path.name in ("python3.pc", "python3-embed.pc"):
+        # Only the tag-suffixed .pc files are co-installable
+        return True
+
+    version = build_details["language"]["version"]
+    if (
+        in_pkgconfig
+        and build_details["abi"]["flags"]  # non-default install
+        and path.name in (f"python-{version}.pc", f"python-{version}-embed.pc")
+    ):
+        # Only the tag-suffixed .pc files are co-installable
+        return True
+
+    in_dist_info = path.parent.name.endswith(".dist-info")
+    if in_dist_info and path.name in ("RECORD", "WHEEL"):
+        # RECORD: Contains hashes, not co-installable.
+        # WHEEL: Contains arch and version tags. Tags can be merged but
+        # not architectures.
+        return True
+
+    in_site_packages = path.parent.name == "site-packages"
+    if in_site_packages and path.name.endswith((".abi3.so", ".abi3t.so")):
+        # abi3 and abi3t are not current co-installable (#122931)
+        return True
+
+    return False
+
+
+def main() -> None:
+    p = ArgumentParser("Compare multiple hash-r files")
+    p.add_argument(
+        "base_directory",
+        type=Path,
+        help="Directory containing hashes of Python installs.",
+    )
+    args = p.parse_args()
+    if not compare_trees(args.base_directory):
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()

Tools/coinstall-check/hash-r.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+# Export a SHA512 manifest of installed files, so that we can ensure that
+# multiple installs of Python don't have conflicting files
+#
+# This is a requirement for Debian's Multi-Arch installs of Python
+# https://www.debian.org/doc/debian-policy/ch-controlfields.html#multi-arch
+
+from argparse import ArgumentParser
+from hashlib import file_digest
+from pathlib import Path
+from typing import Any, cast
+import gzip
+import json
+
+
+def load_build_details(base: Path) -> dict[str, Any]:
+    for path in base.glob("usr/lib/python*/build-details*.json"):
+        details = json.loads(path.read_bytes())
+        return cast(dict[str, Any], details)
+    raise AssertionError(f"build-details.json not found in {base}")
+
+
+def hash_tree(base: Path, algorithm: str = "sha512") -> dict[str, str]:
+    hashes: dict[str, str] = {}
+    for dirpath, dirnames, filenames in base.walk():
+        for file in filenames:
+            filepath = dirpath / file
+            with filepath.open("rb") as f:
+                digest = file_digest(f, algorithm)
+            hashes[str(filepath.relative_to(base))] = digest.hexdigest()
+    return hashes
+
+
+def write_json(destdir: Path, output: Path) -> None:
+    """Hash the Python install at destdir, write gzipped JSON to output."""
+    data = {
+        "build_details": load_build_details(destdir),
+        "hashes": hash_tree(destdir),
+    }
+    with gzip.open(output, "wt") as f:
+        f.write(json.dumps(data))
+
+
+def main() -> None:
+    p = ArgumentParser("Hash a python install for comparison later")
+    p.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        help="Output file (gzipped)",
+    )
+    p.add_argument(
+        "destdir",
+        type=Path,
+        help="Directory below which Python is installed",
+    )
+    args = p.parse_args()
+    write_json(args.destdir, args.output)
+
+
+if __name__ == "__main__":
+    main()

Tools/freeze/test/freeze.py

@@ -128,6 +128,7 @@ def prepare(script=None, outdir=None):
     # Run configure.
     print(f'configuring python in {builddir}...')
     config_args = shlex.split(sysconfig.get_config_var('CONFIG_ARGS') or '')
+    config_args = [arg for arg in config_args if not arg.startswith("--libdir=/")]
     cmd = [os.path.join(srcdir, 'configure'), *config_args]
     ensure_opt(cmd, 'cache-file', os.path.join(outdir, 'python-config.cache'))
     prefix = os.path.join(outdir, 'python-installation')