From 6a63956a8ed15915ab7fe97104ac176ebd1088c6 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Thu, 2 Apr 2026 15:02:03 -0500 Subject: [PATCH 1/4] add pip authenticate for all pipelines --- eng/ci/emulator-tests.yml | 3 ++- eng/ci/official-build.yml | 6 +++++- eng/ci/public-build.yml | 7 ++++++- eng/pack/templates/macos_64_env_gen.yml | 4 ++++ eng/pack/templates/nix_arm64_env_gen.yml | 4 ++++ eng/pack/templates/nix_env_gen.yml | 4 ++++ eng/pack/templates/win_env_gen.yml | 4 ++++ eng/templates/jobs/build.yml | 3 ++- eng/templates/jobs/ci-dependency-check.yml | 7 +++++++ eng/templates/jobs/ci-emulator-tests.yml | 5 +++++ eng/templates/jobs/ci-library-unit-tests.yml | 5 +++++ eng/templates/jobs/ci-unit-tests.yml | 5 +++++ eng/templates/official/jobs/build-library.yml | 4 ++++ eng/templates/official/jobs/ci-core-tools-tests.yml | 4 ++++ eng/templates/official/jobs/ci-custom-image-tests.yml | 4 ++++ .../official/jobs/ci-docker-consumption-tests.yml | 4 ++++ eng/templates/official/jobs/ci-docker-dedicated-tests.yml | 4 ++++ eng/templates/official/jobs/ci-e2e-tests.yml | 4 ++++ eng/templates/official/jobs/ci-lc-tests.yml | 4 ++++ eng/templates/official/jobs/publish-library-release.yml | 4 ++++ eng/templates/shared/build-steps.yml | 5 +++++ 21 files changed, 90 insertions(+), 4 deletions(-) diff --git a/eng/ci/emulator-tests.yml b/eng/ci/emulator-tests.yml index c7792ea94..a9eabcff5 100644 --- a/eng/ci/emulator-tests.yml +++ b/eng/ci/emulator-tests.yml @@ -49,4 +49,5 @@ extends: jobs: - template: /eng/templates/jobs/ci-emulator-tests.yml@self parameters: - PoolName: 1es-pool-azfunc \ No newline at end of file + PoolName: 1es-pool-azfunc + ArtifactFeed: 'internal/PythonWorker_Internal_PublicPackages' \ No newline at end of file diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index 40095352a..818656ba0 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -63,12 +63,14 @@ extends: - template: /eng/templates/jobs/ci-emulator-tests.yml@self parameters: PoolName: 1es-pool-azfunc + ArtifactFeed: 'internal/PythonWorker_Internal_PublicPackages' - stage: RunWorkerUnitTests dependsOn: BuildPythonWorker jobs: - template: /eng/templates/jobs/ci-unit-tests.yml@self parameters: PoolName: 1es-pool-azfunc + ArtifactFeed: 'internal/PythonWorker_Internal_PublicPackages' - stage: RunWorkerDockerConsumptionTests dependsOn: BuildPythonWorker jobs: @@ -99,6 +101,7 @@ extends: PROJECT_NAME: 'Python V2 Library' PROJECT_DIRECTORY: 'runtimes/v2' PoolName: 1es-pool-azfunc + ArtifactFeed: 'internal/PythonWorker_Internal_PublicPackages' # Python V1 Library Build and Test Stages - stage: BuildV1Library @@ -116,4 +119,5 @@ extends: parameters: PROJECT_NAME: 'Python V1 Library' PROJECT_DIRECTORY: 'runtimes/v1' - PoolName: 1es-pool-azfunc \ No newline at end of file + PoolName: 1es-pool-azfunc + ArtifactFeed: 'internal/PythonWorker_Internal_PublicPackages' \ No newline at end of file diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index 5605fa025..aa9dee25b 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -79,6 +79,7 @@ extends: - template: /eng/templates/jobs/ci-dependency-check.yml@self parameters: PoolName: 1es-pool-azfunc-public + ArtifactFeed: 'public/PythonWorker_PublicPackages' - stage: RunWorkerUnitTests dependsOn: BuildPythonWorker jobs: @@ -86,12 +87,14 @@ extends: parameters: PROJECT_DIRECTORY: 'workers' PoolName: 1es-pool-azfunc-public + ArtifactFeed: 'public/PythonWorker_PublicPackages' - stage: RunWorkerEmulatorTests dependsOn: BuildPythonWorker jobs: - template: /eng/templates/jobs/ci-emulator-tests.yml@self parameters: PoolName: 1es-pool-azfunc-public + ArtifactFeed: 'public/PythonWorker_PublicPackages' # Python V2 Library Build and Test Stages - stage: BuildV2Library @@ -110,6 +113,7 @@ extends: PROJECT_NAME: 'V2 Library' PROJECT_DIRECTORY: 'runtimes/v2' PoolName: 1es-pool-azfunc-public + ArtifactFeed: 'public/PythonWorker_PublicPackages' # Python V1 Library Build and Test Stages - stage: BuildV1Library @@ -127,4 +131,5 @@ extends: parameters: PROJECT_NAME: 'V1 Library' PROJECT_DIRECTORY: 'runtimes/v1' - PoolName: 1es-pool-azfunc-public \ No newline at end of file + PoolName: 1es-pool-azfunc-public + ArtifactFeed: 'public/PythonWorker_PublicPackages' \ No newline at end of file diff --git a/eng/pack/templates/macos_64_env_gen.yml b/eng/pack/templates/macos_64_env_gen.yml index 2b1c831a9..4168d2727 100644 --- a/eng/pack/templates/macos_64_env_gen.yml +++ b/eng/pack/templates/macos_64_env_gen.yml @@ -9,6 +9,10 @@ steps: versionSpec: ${{ parameters.pythonVersion }} allowUnstable: true addToPath: true +- task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'internal/PythonWorker_Internal_PublicPackages' - powershell: | # Parse the Python minor version $PY_VER = "$(pythonVersion)" diff --git a/eng/pack/templates/nix_arm64_env_gen.yml b/eng/pack/templates/nix_arm64_env_gen.yml index 528722e09..fd3d77a55 100644 --- a/eng/pack/templates/nix_arm64_env_gen.yml +++ b/eng/pack/templates/nix_arm64_env_gen.yml @@ -9,6 +9,10 @@ steps: versionSpec: ${{ parameters.pythonVersion }} allowUnstable: true addToPath: true +- task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'internal/PythonWorker_Internal_PublicPackages' - powershell: | # Parse the Python minor version $PY_VER = "$(pythonVersion)" diff --git a/eng/pack/templates/nix_env_gen.yml b/eng/pack/templates/nix_env_gen.yml index 079dbe21b..744f6fa6c 100644 --- a/eng/pack/templates/nix_env_gen.yml +++ b/eng/pack/templates/nix_env_gen.yml @@ -9,6 +9,10 @@ steps: versionSpec: ${{ parameters.pythonVersion }} allowUnstable: true addToPath: true +- task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'internal/PythonWorker_Internal_PublicPackages' - powershell: | # Parse the Python minor version $PY_VER = "$(pythonVersion)" diff --git a/eng/pack/templates/win_env_gen.yml b/eng/pack/templates/win_env_gen.yml index 1760193ea..4df08ee49 100644 --- a/eng/pack/templates/win_env_gen.yml +++ b/eng/pack/templates/win_env_gen.yml @@ -10,6 +10,10 @@ steps: allowUnstable: true architecture: ${{ parameters.architecture }} addToPath: true +- task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'internal/PythonWorker_Internal_PublicPackages' - powershell: | # Parse the Python minor version $PY_VER = "$(pythonVersion)" diff --git a/eng/templates/jobs/build.yml b/eng/templates/jobs/build.yml index 1ceb033db..7a6825008 100644 --- a/eng/templates/jobs/build.yml +++ b/eng/templates/jobs/build.yml @@ -18,4 +18,5 @@ jobs: parameters: PYTHON_VERSION: ${{ parameters.PYTHON_VERSION }} PROJECT_NAME: ${{ parameters.PROJECT_NAME }} - PROJECT_DIRECTORY: ${{ parameters.PROJECT_DIRECTORY }} \ No newline at end of file + PROJECT_DIRECTORY: ${{ parameters.PROJECT_DIRECTORY }} + ArtifactFeed: 'public/PythonWorker_PublicPackages' \ No newline at end of file diff --git a/eng/templates/jobs/ci-dependency-check.yml b/eng/templates/jobs/ci-dependency-check.yml index 1da4982f6..2334eb147 100644 --- a/eng/templates/jobs/ci-dependency-check.yml +++ b/eng/templates/jobs/ci-dependency-check.yml @@ -1,3 +1,6 @@ +parameters: + ArtifactFeed: '' + jobs: - job: "TestPython" displayName: "Run Dependency Checks" @@ -25,6 +28,10 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: $(PYTHON_VERSION) + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: ${{ parameters.ArtifactFeed }} - powershell: | $PY_VER = "$(PYTHON_VERSION)" Write-Host "Python version: $PY_VER" diff --git a/eng/templates/jobs/ci-emulator-tests.yml b/eng/templates/jobs/ci-emulator-tests.yml index fe721cfb1..eddb6a32d 100644 --- a/eng/templates/jobs/ci-emulator-tests.yml +++ b/eng/templates/jobs/ci-emulator-tests.yml @@ -1,5 +1,6 @@ parameters: PROJECT_DIRECTORY: 'workers' + ArtifactFeed: '' jobs: - job: "TestPython" @@ -46,6 +47,10 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: $(PYTHON_VERSION) + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: ${{ parameters.ArtifactFeed }} - task: UseDotNet@2 displayName: 'Install .NET 8' inputs: diff --git a/eng/templates/jobs/ci-library-unit-tests.yml b/eng/templates/jobs/ci-library-unit-tests.yml index f82cf5850..b5eaefea4 100644 --- a/eng/templates/jobs/ci-library-unit-tests.yml +++ b/eng/templates/jobs/ci-library-unit-tests.yml @@ -1,6 +1,7 @@ parameters: PROJECT_NAME: '' PROJECT_DIRECTORY: '' + ArtifactFeed: '' jobs: - job: "TestPython" @@ -20,6 +21,10 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: $(PYTHON_VERSION) + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: ${{ parameters.ArtifactFeed }} - task: UseDotNet@2 displayName: 'Install .NET 8' inputs: diff --git a/eng/templates/jobs/ci-unit-tests.yml b/eng/templates/jobs/ci-unit-tests.yml index 45d98855d..45b0efa96 100644 --- a/eng/templates/jobs/ci-unit-tests.yml +++ b/eng/templates/jobs/ci-unit-tests.yml @@ -1,5 +1,6 @@ parameters: PROJECT_DIRECTORY: 'workers' + ArtifactFeed: '' jobs: - job: "TestPython" @@ -46,6 +47,10 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: $(PYTHON_VERSION) + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: ${{ parameters.ArtifactFeed }} - task: UseDotNet@2 displayName: 'Install .NET 8' inputs: diff --git a/eng/templates/official/jobs/build-library.yml b/eng/templates/official/jobs/build-library.yml index f68f8985b..c5bb02925 100644 --- a/eng/templates/official/jobs/build-library.yml +++ b/eng/templates/official/jobs/build-library.yml @@ -23,6 +23,10 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: "3.13" + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'public/PythonWorker_PublicPackages' - bash: | python --version displayName: 'Check python version' diff --git a/eng/templates/official/jobs/ci-core-tools-tests.yml b/eng/templates/official/jobs/ci-core-tools-tests.yml index 3e8a9b622..903630b49 100644 --- a/eng/templates/official/jobs/ci-core-tools-tests.yml +++ b/eng/templates/official/jobs/ci-core-tools-tests.yml @@ -13,6 +13,10 @@ jobs: inputs: versionSpec: "3.10" addToPath: true + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'internal/PythonWorker_Internal_PublicPackages' - task: UseDotNet@2 displayName: 'Install DotNet 3' inputs: diff --git a/eng/templates/official/jobs/ci-custom-image-tests.yml b/eng/templates/official/jobs/ci-custom-image-tests.yml index 581a3a7ce..8bf854858 100644 --- a/eng/templates/official/jobs/ci-custom-image-tests.yml +++ b/eng/templates/official/jobs/ci-custom-image-tests.yml @@ -14,6 +14,10 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: $(CUSTOM_PYTHON_VERSION) + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'internal/PythonWorker_Internal_PublicPackages' - bash: | chmod +x eng/scripts/install-dependencies.sh diff --git a/eng/templates/official/jobs/ci-docker-consumption-tests.yml b/eng/templates/official/jobs/ci-docker-consumption-tests.yml index c4af5c014..d8ee29cbe 100644 --- a/eng/templates/official/jobs/ci-docker-consumption-tests.yml +++ b/eng/templates/official/jobs/ci-docker-consumption-tests.yml @@ -41,6 +41,10 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: $(PYTHON_VERSION) + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'internal/PythonWorker_Internal_PublicPackages' - bash: | chmod +x eng/scripts/install-dependencies.sh diff --git a/eng/templates/official/jobs/ci-docker-dedicated-tests.yml b/eng/templates/official/jobs/ci-docker-dedicated-tests.yml index dff9f5d1f..711c21d06 100644 --- a/eng/templates/official/jobs/ci-docker-dedicated-tests.yml +++ b/eng/templates/official/jobs/ci-docker-dedicated-tests.yml @@ -41,6 +41,10 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: $(PYTHON_VERSION) + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'internal/PythonWorker_Internal_PublicPackages' - bash: | chmod +x eng/scripts/install-dependencies.sh diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml index dce8f8af2..2d319df8e 100644 --- a/eng/templates/official/jobs/ci-e2e-tests.yml +++ b/eng/templates/official/jobs/ci-e2e-tests.yml @@ -70,6 +70,10 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: $(PYTHON_VERSION) + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'internal/PythonWorker_Internal_PublicPackages' - task: UseDotNet@2 displayName: 'Install .NET 8' inputs: diff --git a/eng/templates/official/jobs/ci-lc-tests.yml b/eng/templates/official/jobs/ci-lc-tests.yml index 0bb4d365c..e8df65d25 100644 --- a/eng/templates/official/jobs/ci-lc-tests.yml +++ b/eng/templates/official/jobs/ci-lc-tests.yml @@ -42,6 +42,10 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: $(PYTHON_VERSION) + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'internal/PythonWorker_Internal_PublicPackages' - bash: | # Start Azurite storage emulator in the background diff --git a/eng/templates/official/jobs/publish-library-release.yml b/eng/templates/official/jobs/publish-library-release.yml index 341870568..345ec2fe4 100644 --- a/eng/templates/official/jobs/publish-library-release.yml +++ b/eng/templates/official/jobs/publish-library-release.yml @@ -79,6 +79,10 @@ jobs: displayName: 'Use Python 3.13' inputs: versionSpec: 3.13 + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'internal/PythonWorker_Internal_PublicPackages' - pwsh: | $newLibraryVersion = "$(NewWorkerVersion)" $pypiToken = "$(PypiToken)" diff --git a/eng/templates/shared/build-steps.yml b/eng/templates/shared/build-steps.yml index 4ecd148c3..2ca962aa7 100644 --- a/eng/templates/shared/build-steps.yml +++ b/eng/templates/shared/build-steps.yml @@ -2,11 +2,16 @@ parameters: PYTHON_VERSION: '' PROJECT_NAME: '' PROJECT_DIRECTORY: '' + ArtifactFeed: '' steps: - task: UsePythonVersion@0 inputs: versionSpec: ${{ parameters.PYTHON_VERSION }} + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: ${{ parameters.ArtifactFeed }} - bash: | python --version displayName: 'Check python version' From 0505f717c819ba55beda7a93af162bb2a7fbf0da Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 7 Apr 2026 12:11:57 -0500 Subject: [PATCH 2/4] linux-arm64 build changes --- eng/pack/scripts/nix_arm64_deps.sh | 129 ++++++++++++----------- eng/pack/scripts/rc_nix_arm64_deps.sh | 129 ++++++++++++----------- eng/pack/templates/nix_arm64_env_gen.yml | 6 -- 3 files changed, 131 insertions(+), 133 deletions(-) diff --git a/eng/pack/scripts/nix_arm64_deps.sh b/eng/pack/scripts/nix_arm64_deps.sh index ff86f9c51..a3284567b 100644 --- a/eng/pack/scripts/nix_arm64_deps.sh +++ b/eng/pack/scripts/nix_arm64_deps.sh @@ -1,80 +1,81 @@ #!/bin/bash +# This script installs ARM64 Python dependencies without Docker +# Uses pip's --platform flag to download ARM64-specific wheels +set -e + +PYTHON_VERSION=$1 +version_minor=$(echo $PYTHON_VERSION | cut -d '.' -f 2) + +echo "Building ARM64 dependencies for Python version: $PYTHON_VERSION (minor: $version_minor)" + +# Create deps directory +mkdir -p $BUILD_SOURCESDIRECTORY/deps + +# Setup Python environment python -m venv .env source .env/bin/activate python -m pip install --upgrade pip python -m pip install setuptools==81.0 -version_minor=$(echo $1 | cut -d '.' -f 2) -mkdir -p $BUILD_SOURCESDIRECTORY/deps - -# Targeting: grpcio manylinux_2_17_aarch64.whl build - -# Starts a docker container using the linux/arm64 platform -# Inside the container, we perform the same steps as our typical builds -# However, since we're running them on the linux/arm64 platform, we ensure -# that we pull in the correct grpc, etc. builds -docker run --privileged --rm tonistiigi/binfmt --install all -docker run --name my-arm64-container --platform linux/arm64 \ - -v ./:/src \ - -w /src \ - python:3.$version_minor bash -c " - ls -la /src # debug: see what files exist - apt-get update && apt-get install -y git curl && \ - pip install --upgrade pip && \ - pip install setuptools==81.0 && \ - cd workers && \ - pip install . && \ - pip install . --target /src && \ - pip install invoke && \ - cd tests && \ - python -m invoke -c test_setup build-protos && \ - ls -la /src - " +echo "Installing build tools (which must be platform-compatible)..." +pip install grpcio-tools invoke +# Navigate to workers directory cd workers -# This copies over the build files from the docker container to the local pipeline -docker cp my-arm64-container:/src/. $BUILD_SOURCESDIRECTORY/all/ -docker rm my-arm64-container - -# From the container, we have many unnecessary files. Here, we only -# copy over the relevant files to the 'deps/' directory. -copy_list=( - "azure" - "azure_functions_worker" - "azure_functions_runtime" - "azure_functions_runtime_v1" - "azurefunctions" - "dateutil" - "google" - "grpc" - "markupsafe" - "proxy_worker" - "six.py" - "werkzeug" -) - -for dir in "${copy_list[@]}"; do - src="$BUILD_SOURCESDIRECTORY/all/$dir" - dest="$BUILD_SOURCESDIRECTORY/deps" - - if [ -e $src ]; then - echo "Copying $dir..." - cp -r $src $dest - else - echo "Directory $dir not found in deps — skipping" - fi - done - -cp .artifactignore "$BUILD_SOURCESDIRECTORY/deps" - -version_minor=$(echo $1 | cut -d '.' -f 2) +# Create temp directory for downloading ARM64 wheels +mkdir -p /tmp/arm64_wheels + +# Download ARM64 wheels for the specific Python version +echo "Downloading ARM64 wheels for Python ${PYTHON_VERSION}..." +pip download \ + --platform manylinux_2_17_aarch64 \ + --only-binary=:all: \ + --dest /tmp/arm64_wheels \ + . + +# List downloaded wheels for debugging +echo "Downloaded wheels:" +ls -la /tmp/arm64_wheels/ + +# Install ARM64 wheels from the downloaded files +echo "Installing ARM64 dependencies from downloaded wheels..." +# Extract wheel files manually to bypass platform compatibility checks +# Wheels are just ZIP archives, so we can extract them directly +for wheel in /tmp/arm64_wheels/*.whl; do + echo "Extracting $wheel..." + unzip -o -q "$wheel" -d $BUILD_SOURCESDIRECTORY/deps +done + +# Remove .dist-info directories to avoid conflicts +rm -rf $BUILD_SOURCESDIRECTORY/deps/*.dist-info + +# Build protos +cd tests +python -m invoke -c test_setup build-protos +cd .. + +echo "Current working directory after building protos & cd-ing one level back: $(pwd)" +ls -la + + +# Copy .artifactignore +if [ -f .artifactignore ]; then + cp .artifactignore $BUILD_SOURCESDIRECTORY/deps +fi + +# Copy protos based on Python version if [[ $version_minor -lt 13 ]]; then - cp -r azure_functions_worker/protos "$BUILD_SOURCESDIRECTORY/deps/azure_functions_worker" + echo "Copying azure_functions_worker protos..." + mkdir -p $BUILD_SOURCESDIRECTORY/deps/azure_functions_worker + cp -r azure_functions_worker/protos $BUILD_SOURCESDIRECTORY/deps/azure_functions_worker else - cp -r proxy_worker/protos "$BUILD_SOURCESDIRECTORY/deps/proxy_worker" + echo "Copying proxy_worker protos..." + mkdir -p $BUILD_SOURCESDIRECTORY/deps/proxy_worker + cp -r proxy_worker/protos $BUILD_SOURCESDIRECTORY/deps/proxy_worker fi +echo "ARM64 dependencies installed successfully!" echo "Listing contents of deps directory:" ls -la $BUILD_SOURCESDIRECTORY/deps \ No newline at end of file diff --git a/eng/pack/scripts/rc_nix_arm64_deps.sh b/eng/pack/scripts/rc_nix_arm64_deps.sh index 20da01183..a3284567b 100644 --- a/eng/pack/scripts/rc_nix_arm64_deps.sh +++ b/eng/pack/scripts/rc_nix_arm64_deps.sh @@ -1,78 +1,81 @@ #!/bin/bash +# This script installs ARM64 Python dependencies without Docker +# Uses pip's --platform flag to download ARM64-specific wheels +set -e + +PYTHON_VERSION=$1 +version_minor=$(echo $PYTHON_VERSION | cut -d '.' -f 2) + +echo "Building ARM64 dependencies for Python version: $PYTHON_VERSION (minor: $version_minor)" + +# Create deps directory +mkdir -p $BUILD_SOURCESDIRECTORY/deps + +# Setup Python environment python -m venv .env source .env/bin/activate python -m pip install --upgrade pip +python -m pip install setuptools==81.0 -version_minor=$(echo $1 | cut -d '.' -f 2) -mkdir -p $BUILD_SOURCESDIRECTORY/deps - -# Targeting: grpcio manylinux_2_17_aarch64.whl build - -# Starts a docker container using the linux/arm64 platform -# Inside the container, we perform the same steps as our typical builds -# However, since we're running them on the linux/arm64 platform, we ensure -# that we pull in the correct grpc, etc. builds -docker run --privileged --rm tonistiigi/binfmt --install all -docker run --name my-arm64-container --platform linux/arm64 \ - -v ./:/src \ - -w /src \ - python:3.14.0rc3-alpine3.22 sh -c " - ls -la /src # debug: see what files exist - apk update && apk add --no-cache git curl build-base && \ - pip install --upgrade pip && \ - cd workers && \ - pip install . && \ - pip install . --target /src && \ - pip install invoke && \ - cd tests && \ - python -m invoke -c test_setup build-protos && \ - ls -la /src - " +echo "Installing build tools (which must be platform-compatible)..." +pip install grpcio-tools invoke +# Navigate to workers directory cd workers -# This copies over the build files from the docker container to the local pipeline -docker cp my-arm64-container:/src/. $BUILD_SOURCESDIRECTORY/all/ -docker rm my-arm64-container - -# From the container, we have many unnecessary files. Here, we only -# copy over the relevant files to the 'deps/' directory. -copy_list=( - "azure" - "azure_functions_worker" - "azure_functions_runtime" - "azure_functions_runtime_v1" - "azurefunctions" - "dateutil" - "google" - "grpc" - "markupsafe" - "proxy_worker" - "six.py" - "werkzeug" -) - -for dir in "${copy_list[@]}"; do - src="$BUILD_SOURCESDIRECTORY/all/$dir" - dest="$BUILD_SOURCESDIRECTORY/deps" - - if [ -e $src ]; then - echo "Copying $dir..." - cp -r $src $dest - else - echo "Directory $dir not found in deps — skipping" - fi - done - -cp .artifactignore "$BUILD_SOURCESDIRECTORY/deps" - -version_minor=$(echo $1 | cut -d '.' -f 2) +# Create temp directory for downloading ARM64 wheels +mkdir -p /tmp/arm64_wheels + +# Download ARM64 wheels for the specific Python version +echo "Downloading ARM64 wheels for Python ${PYTHON_VERSION}..." +pip download \ + --platform manylinux_2_17_aarch64 \ + --only-binary=:all: \ + --dest /tmp/arm64_wheels \ + . + +# List downloaded wheels for debugging +echo "Downloaded wheels:" +ls -la /tmp/arm64_wheels/ + +# Install ARM64 wheels from the downloaded files +echo "Installing ARM64 dependencies from downloaded wheels..." +# Extract wheel files manually to bypass platform compatibility checks +# Wheels are just ZIP archives, so we can extract them directly +for wheel in /tmp/arm64_wheels/*.whl; do + echo "Extracting $wheel..." + unzip -o -q "$wheel" -d $BUILD_SOURCESDIRECTORY/deps +done + +# Remove .dist-info directories to avoid conflicts +rm -rf $BUILD_SOURCESDIRECTORY/deps/*.dist-info + +# Build protos +cd tests +python -m invoke -c test_setup build-protos +cd .. + +echo "Current working directory after building protos & cd-ing one level back: $(pwd)" +ls -la + + +# Copy .artifactignore +if [ -f .artifactignore ]; then + cp .artifactignore $BUILD_SOURCESDIRECTORY/deps +fi + +# Copy protos based on Python version if [[ $version_minor -lt 13 ]]; then - cp -r azure_functions_worker/protos "$BUILD_SOURCESDIRECTORY/deps/azure_functions_worker" + echo "Copying azure_functions_worker protos..." + mkdir -p $BUILD_SOURCESDIRECTORY/deps/azure_functions_worker + cp -r azure_functions_worker/protos $BUILD_SOURCESDIRECTORY/deps/azure_functions_worker else - cp -r proxy_worker/protos "$BUILD_SOURCESDIRECTORY/deps/proxy_worker" + echo "Copying proxy_worker protos..." + mkdir -p $BUILD_SOURCESDIRECTORY/deps/proxy_worker + cp -r proxy_worker/protos $BUILD_SOURCESDIRECTORY/deps/proxy_worker fi +echo "ARM64 dependencies installed successfully!" echo "Listing contents of deps directory:" ls -la $BUILD_SOURCESDIRECTORY/deps \ No newline at end of file diff --git a/eng/pack/templates/nix_arm64_env_gen.yml b/eng/pack/templates/nix_arm64_env_gen.yml index fd3d77a55..247bcd979 100644 --- a/eng/pack/templates/nix_arm64_env_gen.yml +++ b/eng/pack/templates/nix_arm64_env_gen.yml @@ -55,12 +55,6 @@ steps: Write-Host "workerPath: $(workerPath)" Write-Host "proxyWorker: $(proxyWorker)" displayName: 'Display variables' -- script: | - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - displayName: 'Docker login to avoid rate limits' - env: - DOCKER_USERNAME: $(DOCKER_USERNAME) - DOCKER_PASSWORD: $(DOCKER_PASSWORD) - task: ShellScript@2 inputs: disableAutoCwd: true From 6bb8b6ea9b860a4ea524f5533bde7ec392866909 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 7 Apr 2026 13:15:29 -0500 Subject: [PATCH 3/4] missed installation step --- eng/pack/scripts/nix_arm64_deps.sh | 8 +++++++- eng/pack/scripts/rc_nix_arm64_deps.sh | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/eng/pack/scripts/nix_arm64_deps.sh b/eng/pack/scripts/nix_arm64_deps.sh index a3284567b..736e0a004 100644 --- a/eng/pack/scripts/nix_arm64_deps.sh +++ b/eng/pack/scripts/nix_arm64_deps.sh @@ -39,6 +39,12 @@ pip download \ echo "Downloaded wheels:" ls -la /tmp/arm64_wheels/ +# Install the main worker package (no deps) +echo "Installing azure-functions-worker..." +pip install . \ + --target $BUILD_SOURCESDIRECTORY/deps \ + --no-deps + # Install ARM64 wheels from the downloaded files echo "Installing ARM64 dependencies from downloaded wheels..." # Extract wheel files manually to bypass platform compatibility checks @@ -78,4 +84,4 @@ fi echo "ARM64 dependencies installed successfully!" echo "Listing contents of deps directory:" -ls -la $BUILD_SOURCESDIRECTORY/deps \ No newline at end of file +ls -la $BUILD_SOURCESDIRECTORY/deps diff --git a/eng/pack/scripts/rc_nix_arm64_deps.sh b/eng/pack/scripts/rc_nix_arm64_deps.sh index a3284567b..736e0a004 100644 --- a/eng/pack/scripts/rc_nix_arm64_deps.sh +++ b/eng/pack/scripts/rc_nix_arm64_deps.sh @@ -39,6 +39,12 @@ pip download \ echo "Downloaded wheels:" ls -la /tmp/arm64_wheels/ +# Install the main worker package (no deps) +echo "Installing azure-functions-worker..." +pip install . \ + --target $BUILD_SOURCESDIRECTORY/deps \ + --no-deps + # Install ARM64 wheels from the downloaded files echo "Installing ARM64 dependencies from downloaded wheels..." # Extract wheel files manually to bypass platform compatibility checks @@ -78,4 +84,4 @@ fi echo "ARM64 dependencies installed successfully!" echo "Listing contents of deps directory:" -ls -la $BUILD_SOURCESDIRECTORY/deps \ No newline at end of file +ls -la $BUILD_SOURCESDIRECTORY/deps From f76fc350576da1a86d8223dfc96e3813662a7bdd Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 7 Apr 2026 13:42:13 -0500 Subject: [PATCH 4/4] it was actually okay --- eng/pack/scripts/nix_arm64_deps.sh | 8 +------- eng/pack/scripts/rc_nix_arm64_deps.sh | 8 +------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/eng/pack/scripts/nix_arm64_deps.sh b/eng/pack/scripts/nix_arm64_deps.sh index 736e0a004..a3284567b 100644 --- a/eng/pack/scripts/nix_arm64_deps.sh +++ b/eng/pack/scripts/nix_arm64_deps.sh @@ -39,12 +39,6 @@ pip download \ echo "Downloaded wheels:" ls -la /tmp/arm64_wheels/ -# Install the main worker package (no deps) -echo "Installing azure-functions-worker..." -pip install . \ - --target $BUILD_SOURCESDIRECTORY/deps \ - --no-deps - # Install ARM64 wheels from the downloaded files echo "Installing ARM64 dependencies from downloaded wheels..." # Extract wheel files manually to bypass platform compatibility checks @@ -84,4 +78,4 @@ fi echo "ARM64 dependencies installed successfully!" echo "Listing contents of deps directory:" -ls -la $BUILD_SOURCESDIRECTORY/deps +ls -la $BUILD_SOURCESDIRECTORY/deps \ No newline at end of file diff --git a/eng/pack/scripts/rc_nix_arm64_deps.sh b/eng/pack/scripts/rc_nix_arm64_deps.sh index 736e0a004..a3284567b 100644 --- a/eng/pack/scripts/rc_nix_arm64_deps.sh +++ b/eng/pack/scripts/rc_nix_arm64_deps.sh @@ -39,12 +39,6 @@ pip download \ echo "Downloaded wheels:" ls -la /tmp/arm64_wheels/ -# Install the main worker package (no deps) -echo "Installing azure-functions-worker..." -pip install . \ - --target $BUILD_SOURCESDIRECTORY/deps \ - --no-deps - # Install ARM64 wheels from the downloaded files echo "Installing ARM64 dependencies from downloaded wheels..." # Extract wheel files manually to bypass platform compatibility checks @@ -84,4 +78,4 @@ fi echo "ARM64 dependencies installed successfully!" echo "Listing contents of deps directory:" -ls -la $BUILD_SOURCESDIRECTORY/deps +ls -la $BUILD_SOURCESDIRECTORY/deps \ No newline at end of file