Skip to content

Commit 0505f71

Browse files
hallvictoriahallvictoria
committed
linux-arm64 build changes
1 parent 6a63956 commit 0505f71

3 files changed

Lines changed: 131 additions & 133 deletions

File tree

‎eng/pack/scripts/nix_arm64_deps.sh‎

Lines changed: 65 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,80 +1,81 @@
11
#!/bin/bash
2+
# This script installs ARM64 Python dependencies without Docker
3+
# Uses pip's --platform flag to download ARM64-specific wheels
24

5+
set -e
6+
7+
PYTHON_VERSION=$1
8+
version_minor=$(echo $PYTHON_VERSION | cut -d '.' -f 2)
9+
10+
echo "Building ARM64 dependencies for Python version: $PYTHON_VERSION (minor: $version_minor)"
11+
12+
# Create deps directory
13+
mkdir -p $BUILD_SOURCESDIRECTORY/deps
14+
15+
# Setup Python environment
316
python -m venv .env
417
source .env/bin/activate
518
python -m pip install --upgrade pip
619
python -m pip install setuptools==81.0
720

8-
version_minor=$(echo $1 | cut -d '.' -f 2)
9-
mkdir -p $BUILD_SOURCESDIRECTORY/deps
10-
11-
# Targeting: grpcio manylinux_2_17_aarch64.whl build
12-
13-
# Starts a docker container using the linux/arm64 platform
14-
# Inside the container, we perform the same steps as our typical builds
15-
# However, since we're running them on the linux/arm64 platform, we ensure
16-
# that we pull in the correct grpc, etc. builds
17-
docker run --privileged --rm tonistiigi/binfmt --install all
18-
docker run --name my-arm64-container --platform linux/arm64 \
19-
-v ./:/src \
20-
-w /src \
21-
python:3.$version_minor bash -c "
22-
ls -la /src # debug: see what files exist
23-
apt-get update && apt-get install -y git curl && \
24-
pip install --upgrade pip && \
25-
pip install setuptools==81.0 && \
26-
cd workers && \
27-
pip install . && \
28-
pip install . --target /src && \
29-
pip install invoke && \
30-
cd tests && \
31-
python -m invoke -c test_setup build-protos && \
32-
ls -la /src
33-
"
21+
echo "Installing build tools (which must be platform-compatible)..."
22+
pip install grpcio-tools invoke
3423

24+
# Navigate to workers directory
3525
cd workers
3626

37-
# This copies over the build files from the docker container to the local pipeline
38-
docker cp my-arm64-container:/src/. $BUILD_SOURCESDIRECTORY/all/
39-
docker rm my-arm64-container
40-
41-
# From the container, we have many unnecessary files. Here, we only
42-
# copy over the relevant files to the 'deps/' directory.
43-
copy_list=(
44-
"azure"
45-
"azure_functions_worker"
46-
"azure_functions_runtime"
47-
"azure_functions_runtime_v1"
48-
"azurefunctions"
49-
"dateutil"
50-
"google"
51-
"grpc"
52-
"markupsafe"
53-
"proxy_worker"
54-
"six.py"
55-
"werkzeug"
56-
)
57-
58-
for dir in "${copy_list[@]}"; do
59-
src="$BUILD_SOURCESDIRECTORY/all/$dir"
60-
dest="$BUILD_SOURCESDIRECTORY/deps"
61-
62-
if [ -e $src ]; then
63-
echo "Copying $dir..."
64-
cp -r $src $dest
65-
else
66-
echo "Directory $dir not found in deps — skipping"
67-
fi
68-
done
69-
70-
cp .artifactignore "$BUILD_SOURCESDIRECTORY/deps"
71-
72-
version_minor=$(echo $1 | cut -d '.' -f 2)
27+
# Create temp directory for downloading ARM64 wheels
28+
mkdir -p /tmp/arm64_wheels
29+
30+
# Download ARM64 wheels for the specific Python version
31+
echo "Downloading ARM64 wheels for Python ${PYTHON_VERSION}..."
32+
pip download \
33+
--platform manylinux_2_17_aarch64 \
34+
--only-binary=:all: \
35+
--dest /tmp/arm64_wheels \
36+
.
37+
38+
# List downloaded wheels for debugging
39+
echo "Downloaded wheels:"
40+
ls -la /tmp/arm64_wheels/
41+
42+
# Install ARM64 wheels from the downloaded files
43+
echo "Installing ARM64 dependencies from downloaded wheels..."
44+
# Extract wheel files manually to bypass platform compatibility checks
45+
# Wheels are just ZIP archives, so we can extract them directly
46+
for wheel in /tmp/arm64_wheels/*.whl; do
47+
echo "Extracting $wheel..."
48+
unzip -o -q "$wheel" -d $BUILD_SOURCESDIRECTORY/deps
49+
done
50+
51+
# Remove .dist-info directories to avoid conflicts
52+
rm -rf $BUILD_SOURCESDIRECTORY/deps/*.dist-info
53+
54+
# Build protos
55+
cd tests
56+
python -m invoke -c test_setup build-protos
57+
cd ..
58+
59+
echo "Current working directory after building protos & cd-ing one level back: $(pwd)"
60+
ls -la
61+
62+
63+
# Copy .artifactignore
64+
if [ -f .artifactignore ]; then
65+
cp .artifactignore $BUILD_SOURCESDIRECTORY/deps
66+
fi
67+
68+
# Copy protos based on Python version
7369
if [[ $version_minor -lt 13 ]]; then
74-
cp -r azure_functions_worker/protos "$BUILD_SOURCESDIRECTORY/deps/azure_functions_worker"
70+
echo "Copying azure_functions_worker protos..."
71+
mkdir -p $BUILD_SOURCESDIRECTORY/deps/azure_functions_worker
72+
cp -r azure_functions_worker/protos $BUILD_SOURCESDIRECTORY/deps/azure_functions_worker
7573
else
76-
cp -r proxy_worker/protos "$BUILD_SOURCESDIRECTORY/deps/proxy_worker"
74+
echo "Copying proxy_worker protos..."
75+
mkdir -p $BUILD_SOURCESDIRECTORY/deps/proxy_worker
76+
cp -r proxy_worker/protos $BUILD_SOURCESDIRECTORY/deps/proxy_worker
7777
fi
7878

79+
echo "ARM64 dependencies installed successfully!"
7980
echo "Listing contents of deps directory:"
8081
ls -la $BUILD_SOURCESDIRECTORY/deps

‎eng/pack/scripts/rc_nix_arm64_deps.sh‎

Lines changed: 66 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,78 +1,81 @@
11
#!/bin/bash
2+
# This script installs ARM64 Python dependencies without Docker
3+
# Uses pip's --platform flag to download ARM64-specific wheels
24

5+
set -e
6+
7+
PYTHON_VERSION=$1
8+
version_minor=$(echo $PYTHON_VERSION | cut -d '.' -f 2)
9+
10+
echo "Building ARM64 dependencies for Python version: $PYTHON_VERSION (minor: $version_minor)"
11+
12+
# Create deps directory
13+
mkdir -p $BUILD_SOURCESDIRECTORY/deps
14+
15+
# Setup Python environment
316
python -m venv .env
417
source .env/bin/activate
518
python -m pip install --upgrade pip
19+
python -m pip install setuptools==81.0
620

7-
version_minor=$(echo $1 | cut -d '.' -f 2)
8-
mkdir -p $BUILD_SOURCESDIRECTORY/deps
9-
10-
# Targeting: grpcio manylinux_2_17_aarch64.whl build
11-
12-
# Starts a docker container using the linux/arm64 platform
13-
# Inside the container, we perform the same steps as our typical builds
14-
# However, since we're running them on the linux/arm64 platform, we ensure
15-
# that we pull in the correct grpc, etc. builds
16-
docker run --privileged --rm tonistiigi/binfmt --install all
17-
docker run --name my-arm64-container --platform linux/arm64 \
18-
-v ./:/src \
19-
-w /src \
20-
python:3.14.0rc3-alpine3.22 sh -c "
21-
ls -la /src # debug: see what files exist
22-
apk update && apk add --no-cache git curl build-base && \
23-
pip install --upgrade pip && \
24-
cd workers && \
25-
pip install . && \
26-
pip install . --target /src && \
27-
pip install invoke && \
28-
cd tests && \
29-
python -m invoke -c test_setup build-protos && \
30-
ls -la /src
31-
"
21+
echo "Installing build tools (which must be platform-compatible)..."
22+
pip install grpcio-tools invoke
3223

24+
# Navigate to workers directory
3325
cd workers
3426

35-
# This copies over the build files from the docker container to the local pipeline
36-
docker cp my-arm64-container:/src/. $BUILD_SOURCESDIRECTORY/all/
37-
docker rm my-arm64-container
38-
39-
# From the container, we have many unnecessary files. Here, we only
40-
# copy over the relevant files to the 'deps/' directory.
41-
copy_list=(
42-
"azure"
43-
"azure_functions_worker"
44-
"azure_functions_runtime"
45-
"azure_functions_runtime_v1"
46-
"azurefunctions"
47-
"dateutil"
48-
"google"
49-
"grpc"
50-
"markupsafe"
51-
"proxy_worker"
52-
"six.py"
53-
"werkzeug"
54-
)
55-
56-
for dir in "${copy_list[@]}"; do
57-
src="$BUILD_SOURCESDIRECTORY/all/$dir"
58-
dest="$BUILD_SOURCESDIRECTORY/deps"
59-
60-
if [ -e $src ]; then
61-
echo "Copying $dir..."
62-
cp -r $src $dest
63-
else
64-
echo "Directory $dir not found in deps — skipping"
65-
fi
66-
done
67-
68-
cp .artifactignore "$BUILD_SOURCESDIRECTORY/deps"
69-
70-
version_minor=$(echo $1 | cut -d '.' -f 2)
27+
# Create temp directory for downloading ARM64 wheels
28+
mkdir -p /tmp/arm64_wheels
29+
30+
# Download ARM64 wheels for the specific Python version
31+
echo "Downloading ARM64 wheels for Python ${PYTHON_VERSION}..."
32+
pip download \
33+
--platform manylinux_2_17_aarch64 \
34+
--only-binary=:all: \
35+
--dest /tmp/arm64_wheels \
36+
.
37+
38+
# List downloaded wheels for debugging
39+
echo "Downloaded wheels:"
40+
ls -la /tmp/arm64_wheels/
41+
42+
# Install ARM64 wheels from the downloaded files
43+
echo "Installing ARM64 dependencies from downloaded wheels..."
44+
# Extract wheel files manually to bypass platform compatibility checks
45+
# Wheels are just ZIP archives, so we can extract them directly
46+
for wheel in /tmp/arm64_wheels/*.whl; do
47+
echo "Extracting $wheel..."
48+
unzip -o -q "$wheel" -d $BUILD_SOURCESDIRECTORY/deps
49+
done
50+
51+
# Remove .dist-info directories to avoid conflicts
52+
rm -rf $BUILD_SOURCESDIRECTORY/deps/*.dist-info
53+
54+
# Build protos
55+
cd tests
56+
python -m invoke -c test_setup build-protos
57+
cd ..
58+
59+
echo "Current working directory after building protos & cd-ing one level back: $(pwd)"
60+
ls -la
61+
62+
63+
# Copy .artifactignore
64+
if [ -f .artifactignore ]; then
65+
cp .artifactignore $BUILD_SOURCESDIRECTORY/deps
66+
fi
67+
68+
# Copy protos based on Python version
7169
if [[ $version_minor -lt 13 ]]; then
72-
cp -r azure_functions_worker/protos "$BUILD_SOURCESDIRECTORY/deps/azure_functions_worker"
70+
echo "Copying azure_functions_worker protos..."
71+
mkdir -p $BUILD_SOURCESDIRECTORY/deps/azure_functions_worker
72+
cp -r azure_functions_worker/protos $BUILD_SOURCESDIRECTORY/deps/azure_functions_worker
7373
else
74-
cp -r proxy_worker/protos "$BUILD_SOURCESDIRECTORY/deps/proxy_worker"
74+
echo "Copying proxy_worker protos..."
75+
mkdir -p $BUILD_SOURCESDIRECTORY/deps/proxy_worker
76+
cp -r proxy_worker/protos $BUILD_SOURCESDIRECTORY/deps/proxy_worker
7577
fi
7678

79+
echo "ARM64 dependencies installed successfully!"
7780
echo "Listing contents of deps directory:"
7881
ls -la $BUILD_SOURCESDIRECTORY/deps

‎eng/pack/templates/nix_arm64_env_gen.yml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,6 @@ steps:
5555
Write-Host "workerPath: $(workerPath)"
5656
Write-Host "proxyWorker: $(proxyWorker)"
5757
displayName: 'Display variables'
58-
- script: |
59-
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
60-
displayName: 'Docker login to avoid rate limits'
61-
env:
62-
DOCKER_USERNAME: $(DOCKER_USERNAME)
63-
DOCKER_PASSWORD: $(DOCKER_PASSWORD)
6458
- task: ShellScript@2
6559
inputs:
6660
disableAutoCwd: true

0 commit comments

Comments
 (0)