SG-42304 Replace OS-specific binary scripts with install_binary.py

install_binary.py is a single cross-platform Python script that replaces
install_binary_linux.sh, install_binary_mac.sh and install_binary_windows.ps1.
Platform detection is done via sys.platform at runtime.

pipelines.yml already calls python install_binary.py - this commit adds
the script and removes the three old OS-specific scripts.

SG-42304 Clean up pipeline naming and move git add to pipeline steps

Author: julien-lang

.snyk

@@ -8,6 +8,7 @@ ignore:
     - resources/python/src/3.10/explicit_requirements.txt
     - resources/python/src/3.11/explicit_requirements.txt
     - resources/python/src/3.13/explicit_requirements.txt
+    # WHY &&&&??????? that's not what we want to do. Quite the opposite....
 exclude:
   global:
     # Exclude Python 3.7 deps because Snyk only scans with supported versions

azure-pipelines.yml

@@ -35,10 +35,10 @@ jobs:
       - name: tk-framework-desktopclient
       - name: tk-shotgun
     extra_test_dependencies:
-      # Required when binary dependencies are not bundled
-      - attrs==22.2.0    # Fix version. Otherwise tk-ci-tools will install latest
-      - Twisted==22.10.0 # Last version supporting Python 3.7
-      - websocket-client==1.6.1 # Last version supporting Python 3.7
+      # All runtime dependencies come from the unified requirements file.
+      # This guarantees CI-installed versions match what pkgs.zip bundles,
+      # preventing sys.modules version conflicts (see SG-42304). TODO
+      - --requirement=azure-pipelines/requirements.txt
     post_tests_steps:
       - task: Bash@3
         displayName: Run interpreter integration tests

azure-pipelines/requirements.txt

@@ -0,0 +1,16 @@
+# Required when binary dependencies are not bundled
+attrs==22.2.0    # Fix version. Otherwise tk-ci-tools will install latest
+## TODO will see later if we can update this ..!.!.!.!
+
+# Twisted must match the version bundled in pkgs.zip/src/ for the corresponding
+# Python version. If the test environment installs a different version, it gets
+# loaded first and causes a sys.modules version conflict at runtime (see SG-42304).
+Twisted==22.10.0; python_version < "3.9"
+# Twisted==24.10.0; python_version >= "3.9" and python_version < "3.13"
+# Twisted~=24.11.0; python_version >= "3.13"
+Twisted~=25.5.0; python_version >= "3.9"
+
+# websocket-client is a test-only dependency (not bundled in pkgs.zip).
+# 1.6.2+ dropped Python 3.7 support; our minimum is Python 3.9.
+websocket-client==1.6.1; python_version < "3.9" # Last version supporting Python 3.7
+websocket-client~=1.9.0; python_version >= "3.9"

resources/python/install_binary.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2017 Shotgun Software Inc.
+#
+# CONFIDENTIAL AND PROPRIETARY
+#
+# This work is provided "AS IS" and subject to the Shotgun Pipeline Toolkit
+# Source Code License included in this distribution package. See LICENSE.
+# By accessing, using, copying or modifying this work you indicate your
+# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
+# not expressly granted therein are reserved by Shotgun Software Inc.
+"""Cross-platform binary requirements installer.
+
+Platform directory mapping:
+  linux  -> bin/<version>/linux
+  darwin -> bin/<version>/mac
+  win32  -> bin/<version>/win
+
+Usage: python install_binary.py
+"""
+
+import os
+import pathlib
+import shutil
+import sys
+
+from pip._internal.cli.main import main as pip_main
+
+_PLATFORM_DIR = {
+    "linux": "linux",
+    "darwin": "mac",
+    "win32": "win",
+}
+
+platform_dir = _PLATFORM_DIR[sys.platform]
+
+python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
+bin_dir = pathlib.Path("bin") / python_version / platform_dir
+requirements = pathlib.Path("bin") / python_version / "explicit_requirements.txt"
+
+# Delete and recreate the bin directory
+shutil.rmtree(bin_dir, ignore_errors=True)
+bin_dir.mkdir(parents=True)
+
+# Build pip install arguments
+pip_args = [
+    "install",
+    "--target",
+    os.fspath(bin_dir),
+    "--no-deps",
+    "--requirement",
+    os.fspath(requirements),
+]
+
+# Linux requires explicit manylinux platform tags so that pip downloads the
+# correct pre-built wheels regardless of the host platform.
+# See: https://deepwiki.com/pypa/manylinux#policy-and-platform-support-matrix
+if sys.platform == "linux":
+    pip_args += [
+        "--platform",
+        "manylinux_2_28_x86_64",
+        "--platform",
+        "manylinux2014_x86_64",
+        "--platform",
+        "manylinux2010_x86_64",
+        "--prefer-binary",
+    ]
+
+if pip_main(pip_args) != 0:
+    raise SystemExit("pip install failed")
+
+# For some reason zope is missing a top-level __init__.py when installed
+# with pip, so we add it manually.
+(bin_dir / "zope" / "__init__.py").touch()
+
+# Remove test directories to reduce package size
+for pattern in [
+    "Crypto/SelfTest",
+    "zope/interface/tests",
+    "zope/interface/*/tests",
+]:
+    for match in bin_dir.glob(pattern):
+        shutil.rmtree(match, ignore_errors=True)
+
+

resources/python/install_binary_linux.sh

@@ -8,12 +8,14 @@
 # agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
 # not expressly granted therein are reserved by Shotgun Software Inc.
 
+set -o errexit
+
 echo "----------------------------------------------------"
 echo "Get python version"
 
-python_major_version=$(python -c "import sys; print(sys.version_info.major)")
-python_minor_version=$(python -c "import sys; print(sys.version_info.minor)")
-python_version="$python_major_version.$python_minor_version"
+python_major_version="$(python -c "import sys; print(sys.version_info.major)")"
+python_minor_version="$(python -c "import sys; print(sys.version_info.minor)")"
+python_version="${python_major_version}.${python_minor_version}"
 
 echo "Python version is $python_version"
 
@@ -22,51 +24,58 @@ echo "Set base paths"
 
 requirements_filename="explicit_requirements.txt"
 package_filename="pkgs.zip"
-source_dir="src/$python_version"
-source_requirements="$source_dir/$requirements_filename"
+source_dir="src/${python_version}"
+source_requirements="${source_dir}/${requirements_filename}"
 
 echo "Source Dir: $source_dir"
 echo "Source Requirements: $source_requirements"
 
 echo "----------------------------------------------------"
 echo "Remove current packages"
 
-find $source_dir/* ! -name $package_filename ! -name $requirements_filename -maxdepth 1 -exec rm -rf {} +
+find "${source_dir}"/* \
+    ! -name "${package_filename}" \
+    ! -name "${requirements_filename}" \
+    -maxdepth 1 \
+    -exec rm --recursive --force {} +
 
 echo "----------------------------------------------------"
 echo "Install new packages"
 
 pip install --upgrade pip setuptools wheel
 
 pip install \
-  --target $source_dir \
+  --target "${source_dir}" \
   --no-deps \
-  -r $source_requirements
+  --requirement "${source_requirements}"
 
 echo "----------------------------------------------------"
 echo "Remove unnecessary files"
 
-rm -Rf $source_dir/autobahn/test
-rm -Rf $source_dir/autobahn/*/test
-rm -Rf $source_dir/twisted/test
-rm -Rf $source_dir/twisted/*/test
-rm -Rf $source_dir/twisted/*/*/test
-rm -Rf $source_dir/automat/_test
-rm -Rf $source_dir/hyperlink/test
-rm -Rf $source_dir/incremental/tests
+rm --recursive --force "${source_dir}/autobahn/test"
+rm --recursive --force "${source_dir}/autobahn/*/test"
+rm --recursive --force "${source_dir}/twisted/test"
+rm --recursive --force "${source_dir}/twisted/*/test"
+rm --recursive --force "${source_dir}/twisted/*/*/test"
+rm --recursive --force "${source_dir}/automat/_test"
+rm --recursive --force "${source_dir}/hyperlink/test"
+rm --recursive --force "${source_dir}/incremental/tests"
 
-# In twisted.internet.unix, there is a mixin which we don't use that allows to copy file descriptors
-# into other processes, which we don't require. That module is compiled, so we'll delete it.
-rm -Rf $source_dir/twisted/python/_sendmsg.so
+# In twisted.internet.unix, there is a mixin which we don't use that allows to
+# copy file descriptors into other processes, which we don't require. That module
+# is compiled, so we'll delete it.
+rm --recursive --force "${source_dir}/twisted/python/_sendmsg.so"
 
 # Compress all files
-pushd $source_dir
-zip -q -r pkgs.zip ./*
+pushd "${source_dir}"
+zip --quiet --recurse-paths pkgs.zip ./*
 popd
 
 # Remove files
-find $source_dir/* ! -name $package_filename ! -name $requirements_filename -maxdepth 1 -exec rm -rf {} +
+find "${source_dir}"/* \
+    ! -name "${package_filename}" \
+    ! -name "${requirements_filename}" \
+    -maxdepth 1 \
+    -exec rm --recursive --force {} +
+
 
-echo "----------------------------------------------------"
-echo "Adding new files to git"
-git add $source_dir