Skip to content

Commit 564629c

Browse files
committed
Compare multiple install results
1 parent 6a81046 commit 564629c

5 files changed

Lines changed: 124 additions & 0 deletions

File tree

.github/CODEOWNERS

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,8 @@ Tools/build/generate_sbom.py @sethmlarson
150150
# ABI check
151151
Misc/libabigail.abignore @encukou
152152

153+
# Multiarch
154+
Tools/coinstall-check/ @stefanor
153155

154156
# ----------------------------------------------------------------------------
155157
# Platform Support

.github/workflows/build.yml

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -638,6 +638,37 @@ jobs:
638638
test: false
639639
upload-install: true
640640

641+
linux-install-compare:
642+
name: Ubuntu Co-install comparison
643+
runs-on: ubuntu-latest
644+
needs: linux-install-build
645+
steps:
646+
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
647+
with:
648+
python-version: '3.x'
649+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
650+
with:
651+
persist-credentials: false
652+
- name: Download installs
653+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
654+
with:
655+
path: installs
656+
pattern: install-tree-*
657+
- name: Remove files we don't expect to match
658+
# .pyc files include timestamps
659+
# /usr/bin/ is not expected to be co-installable
660+
# The generic python3.pc and python3.X.pc files are not expected to be co-installable
661+
# pyconfig.h is not expected to be co-installable
662+
# dist-info RECORD and WHEEL are not co-installable
663+
run: |
664+
find installs -type d -name __pycache__ | xargs rm -r
665+
rm -r installs/*/usr/bin/
666+
rm installs/install-tree-*-*[td]/usr/lib/*/pkgconfig/python*[0-9]{,-embed}.pc
667+
rm installs/*/usr/include/python*/pyconfig.h
668+
rm installs/*/usr/lib/python*/site-packages/*.dist-info/{RECORD,WHEEL}
669+
- name: Compare installs
670+
run: python3 Tools/coinstall-check/compare.py installs/
671+
641672
cifuzz:
642673
# ${{ '' } is a hack to nest jobs under the same sidebar category.
643674
name: CIFuzz${{ '' }} # zizmor: ignore[obfuscation]
@@ -705,6 +736,7 @@ jobs:
705736
- build-san
706737
- cross-build-linux
707738
- cifuzz
739+
- linux-install-compare
708740
if: always()
709741

710742
steps:

.github/workflows/reusable-ubuntu.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ jobs:
105105
${{ fromJSON(inputs.debug) && '--with-pydebug' || '' }}
106106
${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}
107107
${{ fromJSON(inputs.bolt-optimizations) && '--enable-bolt' || '' }}
108+
${{ fromJSON(inputs.upload-install) && '--prefix=/usr --libdir=/usr/lib/$(gcc --print-multiarch) --with-build-details-suffix' || '' }}
108109
- name: Build CPython out-of-tree
109110
if: ${{ inputs.free-threading }}
110111
working-directory: ${{ env.CPYTHON_BUILDDIR }}
@@ -139,6 +140,21 @@ jobs:
139140
if: ${{ inputs.upload-install }}
140141
working-directory: ${{ env.CPYTHON_BUILDDIR }}
141142
run: make install DESTDIR=build/install
143+
- name: Install some test packages in the Python
144+
if: ${{ inputs.upload-install }}
145+
working-directory: ${{ env.CPYTHON_BUILDDIR }}
146+
run: |
147+
CPYTHON_TEST_EXT_NAME=c_mod \
148+
build/install/usr/bin/python3 -m pip install ../cpython-ro-srcdir/Lib/test/test_cext
149+
CPYTHON_TEST_LIMITED=1 CPYTHON_TEST_EXT_NAME=abi3_mod \
150+
build/install/usr/bin/python3 -m pip install ../cpython-ro-srcdir/Lib/test/test_cext
151+
CPYTHON_TEST_ABI3T=1 CPYTHON_TEST_EXT_NAME=abi3t_mod \
152+
build/install/usr/bin/python3 -m pip install ../cpython-ro-srcdir/Lib/test/test_cext
153+
# Until https://github.com/pypa/setuptools/pull/5193 is released
154+
if [ ${{ fromJSON(inputs.free-threading) && '1' || '0' }} = '0' ]; then \
155+
(cd build/install/usr/lib/python*/site-packages && \
156+
mv abi3t_mod.abi3.so abi3t_mod.abi3t.so); \
157+
fi
142158
- name: Upload Installed Python
143159
if: ${{ inputs.upload-install }}
144160
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0

Tools/README

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ clinic A preprocessor for CPython C files in order to automate
1414
the boilerplate involved with writing argument parsing
1515
code for "builtins".
1616

17+
coinstall-check A tool to ensure that multiple CPython builds can be
18+
co-installed on Linux.
19+
1720
freeze Create a stand-alone executable from a Python program.
1821

1922
ftscalingbench Benchmarks for free-threading and finding bottlenecks.

Tools/coinstall-check/compare.py

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/usr/bin/env python3
2+
# Compare that multiple installs of Python don't have conflicting files
3+
#
4+
# This is a requirement for Debian's Multi-Arch installs of Python
5+
# https://www.debian.org/doc/debian-policy/ch-controlfields.html#multi-arch
6+
#
7+
# Excluded from this should be:
8+
# * /usr/bin/*: only libraries are multi-arch co-installed, one arch's binaries
9+
# are installed at a time
10+
# * pyconfig.h: Varies according to config, installed into a tag-specific
11+
# directory.
12+
# * Non-tag suffixed .pc files: Only the suffixed versions are co-installable
13+
# * .dist-info/RECORD: Contains hashes, not co-installable.
14+
# * .dist-info/WHEEL: Contains arch and version tags. Can be merged in some
15+
# cases, but not typically co-installable.
16+
17+
from argparse import ArgumentParser
18+
from hashlib import file_digest
19+
from pathlib import Path
20+
21+
22+
def hash_tree(base: Path, algorithm: str = "sha512") -> dict[str, str]:
23+
print(f"Hashing {base}")
24+
seen: dict[str, str] = {}
25+
for dirpath, dirnames, filenames in base.walk():
26+
if dirpath.name == "__pycache__":
27+
# Includes a timestamp, we expect a mismatch
28+
continue
29+
for file in filenames:
30+
filepath = dirpath / file
31+
with filepath.open("rb") as f:
32+
digest = file_digest(f, algorithm)
33+
seen[str(filepath.relative_to(base))] = digest.hexdigest()
34+
return seen
35+
36+
37+
def compare_trees(base: Path) -> bool:
38+
seen: dict[str, str] = {}
39+
success: bool = True
40+
for tree in base.iterdir():
41+
if not tree.is_dir():
42+
continue
43+
hashes = hash_tree(tree)
44+
for path, digest in hashes.items():
45+
if path not in seen:
46+
seen[path] = digest
47+
continue
48+
if digest != seen[path]:
49+
print(f"Mismatch found in {tree}: {path}")
50+
print(f"{digest} != {seen[path]}")
51+
success = False
52+
return success
53+
54+
55+
def main() -> None:
56+
p = ArgumentParser("Compare multiple installs of Python")
57+
p.add_argument(
58+
"base_directory",
59+
type=Path,
60+
help=(
61+
"Directory below which multiple Pythons are installed, "
62+
"each inside their own directory."
63+
),
64+
)
65+
args = p.parse_args()
66+
if not compare_trees(args.base_directory):
67+
raise SystemExit(1)
68+
69+
70+
if __name__ == "__main__":
71+
main()

0 commit comments

Comments
 (0)