Merge pull request #408 from puneetmatharu/bump-hashes-for-r25.12

Bump hashes for r25.12 and patch PyTorch build

Author: puneetmatharu

ML-Frameworks/pytorch-aarch64/CHANGELOG.md

@@ -10,6 +10,12 @@ where `YY` is the year, and `MM` the month of the increment.
 ### Added
 
 ### Changed
+ - Updates hashes for:
+   - `PYTORCH_HASH` to `93fef4bd1dd265588863929e35d9ac89328d5695`, 2.10.0.dev20251124 from viable/strict, Nov 24th.
+   - `IDEEP_HASH` to `3724bec97a77ce990e8c6dc5e595bb3beee75257`, from ideep_pytorch, Nov 24th.
+   - `ONEDNN_HASH` to `0b8a866c009b03f322e6526d7c33cfec84a4a97a`, from main, Nov 25th.
+   - `TORCH_AO_HASH` to `ab6bc89512d912c17a79ed8d4d709612d3e32884`, from main, Nov 25th.
+   - `KLEIDIAI_HASH` to `94d6cc40689f44d308dbd57cb842e335fdd958f1`, v1.17.0 from main, Nov 17th.
 
 ### Removed
 

ML-Frameworks/pytorch-aarch64/build-torch-ao-wheel.sh

@@ -1,4 +1,4 @@
-# #!/bin/bash
+#!/bin/bash
 
 # *******************************************************************************
 # Copyright 2024-2025 Arm Limited and affiliates.

ML-Frameworks/pytorch-aarch64/build-wheel.sh

@@ -1,4 +1,4 @@
-# #!/bin/bash
+#!/bin/bash
 
 # *******************************************************************************
 # Copyright 2024-2025 Arm Limited and affiliates.
@@ -23,16 +23,20 @@
 # The logic in this script should mirror the upstream build pipelines as closely
 # as possible, along with Tool-Solutions specific changes we want to test (e.g.
 # installing tbb) or improving local development in a way that doesn't affect
-# the result (e,.g. unsetting MAX_JOBS). Currently the upstream logic is defined
-# in pytorch/.github/workflows/_binary-build-linux.yml and
-# pytorch/.github/workflows/generated-linux-aarch64-binary-manywheel-nightly.yml
+# the result. Currently the upstream logic is defined in 
+#               pytorch/.github/workflows/_binary-build-linux.yml 
+# and
+#   pytorch/.github/workflows/generated-linux-aarch64-binary-manywheel-nightly.yml
 
 set -eux -o pipefail
 
 PYTHON_VERSION="3.10"
 OPENBLAS_VERSION="v0.3.30"
 ACL_VERSION="v52.6.0"
 
+PYTHON_TAG="cp$(echo "$PYTHON_VERSION" | tr -d .)-cp$(echo "$PYTHON_VERSION" | tr -d .)"
+PYTHON_BIN="/opt/python/${PYTHON_TAG}/bin"
+
 # Specify DOCKER_IMAGE_MIRROR if you want to use a mirror of hub.docker.com
 IMAGE_NAME="${DOCKER_IMAGE_MIRROR:-}pytorch/manylinux2_28_aarch64-builder:cpu-aarch64-d8be0384e085f551506bd739678109fa0f5ee7ac"
 TORCH_BUILD_CONTAINER_ID_FILE="${PWD}/.torch_build_container_id"
@@ -42,22 +46,21 @@ OUTPUT_DIR=${OUTPUT_DIR:-"${PWD}/results"}
 
 # Where folders sit locally
 PYTORCH_HOST_DIR="${PWD}/pytorch"
-ACL_HOST_DIR="${PWD}/ComputeLibrary"
 OPENSSL_HOST_DIR="/opt/openssl"
-PYTORCH_FINAL_PACKAGE_DIR="${OUTPUT_DIR}"
+PYTORCH_FINAL_PACKAGE_HOST_DIR="${OUTPUT_DIR}"
 
 # Where folders sit mounted in the container
 PYTORCH_ROOT=/pytorch
-ACL_ROOT=/acl
 UTILS=/utils
 COMMON_UTILS=/common_utils
+PYTORCH_FINAL_PACKAGE_DIR=/artifacts
 
 # Want a CPU build
 DESIRED_CUDA=cpu
 GPU_ARCH_TYPE=cpu-aarch64
 
 # Affects the number of jobs used in install_acl.sh and install_openblas.sh
-NPROC=${NPROC:-$(nproc --ignore=2)}
+MAX_JOBS=${MAX_JOBS:-$(nproc --ignore=2)}
 
 if [ -f "$TORCH_BUILD_CONTAINER_ID_FILE" ]; then
     TORCH_BUILD_CONTAINER=$(cat $TORCH_BUILD_CONTAINER_ID_FILE)
@@ -71,7 +74,7 @@ if ! docker container inspect $TORCH_BUILD_CONTAINER >/dev/null 2>&1 ; then
     # Based on environment used in pytorch/.github/workflows/_binary-build-linux.yml
     # and pytorch/.github/workflows/generated-linux-aarch64-binary-manywheel-nightly.yml
     TORCH_BUILD_CONTAINER=$(docker run -t -d \
-        -e NPROC=${NPROC} \
+        -e MAX_JOBS=${MAX_JOBS} \
         -e OPENBLAS_VERSION=${OPENBLAS_VERSION} \
         -e ACL_VERSION=${ACL_VERSION} \
         -e BINARY_ENV_FILE=/tmp/env \
@@ -84,12 +87,10 @@ if ! docker container inspect $TORCH_BUILD_CONTAINER >/dev/null 2>&1 ; then
         -e PYTORCH_FINAL_PACKAGE_DIR="${PYTORCH_FINAL_PACKAGE_DIR}" \
         -e PYTORCH_ROOT="${PYTORCH_ROOT}" \
         -e SKIP_ALL_TESTS=1 \
-        -e PYTORCH_EXTRA_INSTALL_REQUIREMENTS="nvidia-cuda-nvrtc-cu12==12.1.105; platform_system == 'Linux' and platform_machine == 'x86_64' | nvidia-cuda-runtime-cu12==12.1.105; platform_system == 'Linux' and platform_machine == 'x86_64' | nvidia-cuda-cupti-cu12==12.1.105; platform_system == 'Linux' and platform_machine == 'x86_64' | nvidia-cudnn-cu12==8.9.2.26; platform_system == 'Linux' and platform_machine == 'x86_64' | nvidia-cublas-cu12==12.1.3.1; platform_system == 'Linux' and platform_machine == 'x86_64' | nvidia-cufft-cu12==11.0.2.54; platform_system == 'Linux' and platform_machine == 'x86_64' | nvidia-curand-cu12==10.3.2.106; platform_system == 'Linux' and platform_machine == 'x86_64' | nvidia-cusolver-cu12==11.4.5.107; platform_system == 'Linux' and platform_machine == 'x86_64' | nvidia-cusparse-cu12==12.1.0.106; platform_system == 'Linux' and platform_machine == 'x86_64' | nvidia-nccl-cu12==2.19.3; platform_system == 'Linux' and platform_machine == 'x86_64' | nvidia-nvtx-cu12==12.1.105; platform_system == 'Linux' and platform_machine == 'x86_64'" \
         -e OPENSSL_ROOT_DIR="${OPENSSL_HOST_DIR}" \
         -e CMAKE_INCLUDE_PATH="${OPENSSL_HOST_DIR}/include" \
         -v "${PYTORCH_HOST_DIR}:${PYTORCH_ROOT}" \
-        -v "${PYTORCH_FINAL_PACKAGE_DIR}:/artifacts" \
-        -v "${ACL_HOST_DIR}:${ACL_ROOT}" \
+        -v "${PYTORCH_FINAL_PACKAGE_HOST_DIR}:${PYTORCH_FINAL_PACKAGE_DIR}" \
         -v "${PWD}/utils:${UTILS}" \
         -v "${PWD}/../utils:${COMMON_UTILS}" \
         -w / \
@@ -98,36 +99,44 @@ if ! docker container inspect $TORCH_BUILD_CONTAINER >/dev/null 2>&1 ; then
     # Currently changes in these scripts will not be applied without a clean
     # build, which is not ideal for dev work. But we have to balance this with
     # extra time/network traffic when rebuilding many times.
-    docker exec -t $TORCH_BUILD_CONTAINER bash -c "$PYTORCH_ROOT/.circleci/scripts/binary_populate_env.sh"
-    docker exec -t $TORCH_BUILD_CONTAINER bash -c "$PYTORCH_ROOT/.ci/aarch64_linux/aarch64_ci_setup.sh"
+    docker exec $TORCH_BUILD_CONTAINER bash "${PYTORCH_ROOT}/.circleci/scripts/binary_populate_env.sh"
+
+    # Install scons for ACL build
+    docker exec $TORCH_BUILD_CONTAINER ${PYTHON_BIN}/python3 -m pip install scons==4.7.0
+    docker exec $TORCH_BUILD_CONTAINER ln -sf ${PYTHON_BIN}/scons /usr/local/bin
 
-    # This must be in this if block because it cannot handle being called twice
-    docker exec -t $TORCH_BUILD_CONTAINER bash -c "bash $PYTORCH_ROOT/.ci/docker/common/install_acl.sh"
-    docker exec -t $TORCH_BUILD_CONTAINER bash -c "bash $PYTORCH_ROOT/.ci/docker/common/install_openblas.sh"
+    # Affected by ACL_VERSION set as an environment variable above
+    echo "Overriding Arm Compute Library version: ${ACL_VERSION}"
+    docker exec "$TORCH_BUILD_CONTAINER" "${PYTORCH_ROOT}/.ci/docker/common/install_acl.sh"
 
-    echo "Storing torch build container id in $TORCH_BUILD_CONTAINER_ID_FILE for reuse: $TORCH_BUILD_CONTAINER"
-    echo $TORCH_BUILD_CONTAINER > "$TORCH_BUILD_CONTAINER_ID_FILE"
+    # Affected by OPENBLAS_VERSION set as an environment variable above
+    echo "Installing OpenBLAS version: ${OPENBLAS_VERSION}"
+    docker exec "$TORCH_BUILD_CONTAINER" "${PYTORCH_ROOT}/.ci/docker/common/install_openblas.sh"
+
+    echo "Storing torch build container ID in ${TORCH_BUILD_CONTAINER_ID_FILE} for reuse: ${TORCH_BUILD_CONTAINER}"
+    echo $TORCH_BUILD_CONTAINER > "${TORCH_BUILD_CONTAINER_ID_FILE}"
 else
     docker restart $TORCH_BUILD_CONTAINER
 fi
 
 # If there are multiple wheels in the dist directory, an old wheel can be
 # erroneously copied to results, so we clear the directory to be sure
-docker exec -t $TORCH_BUILD_CONTAINER bash -c "rm -rf $PYTORCH_ROOT/dist"
+docker exec $TORCH_BUILD_CONTAINER rm -rf "${PYTORCH_ROOT}/dist"
 
 # We set OVERRIDE_PACKAGE_VERSION to be based on the date of the latest torch
 # commit, this allows us to also install the matching torch* packages, set in
 # the Dockerfile. This is what PyTorch does in its nightly pipeline, see
 # pytorch/.ci/aarch64_linux/aarch64_wheel_ci_build.py for this logic.
 build_date=$(cd $PYTORCH_HOST_DIR && git log --pretty=format:%cs -1 | tr -d '-')
-version=$(cat $PYTORCH_HOST_DIR/version.txt| tr -d [:space:] )
+version=$(cat $PYTORCH_HOST_DIR/version.txt| tr -d "[:space:]" )
 OVERRIDE_PACKAGE_VERSION="${version%??}.dev${build_date}${TORCH_RELEASE_ID:+"+$TORCH_RELEASE_ID"}"
-# We unset MAX_JOBS from 12 (written to /tmp/env by populate_binary_env.sh) to
-# let any downstream functions to decide. Currently nproc when ninja is used,
-# but ideally we would let ninja make the right choice, but nproc is good enough
-docker exec -t $TORCH_BUILD_CONTAINER \
-    bash -c "source /tmp/env && unset MAX_JOBS && OVERRIDE_PACKAGE_VERSION=$OVERRIDE_PACKAGE_VERSION bash $PYTORCH_ROOT/.ci/aarch64_linux/aarch64_ci_build.sh"
+
+docker exec $TORCH_BUILD_CONTAINER bash -lc "
+  source /tmp/env &&
+  BUILD_TEST=0 \
+  OVERRIDE_PACKAGE_VERSION=$OVERRIDE_PACKAGE_VERSION \
+  bash ${PYTORCH_ROOT}/.ci/manywheel/build.sh
+"
 
 # directories generated by the docker container are owned by root, so transfer ownership to user
-docker exec $TORCH_BUILD_CONTAINER chown -R $(id -u):$(id -g) $PYTORCH_ROOT
-docker exec $TORCH_BUILD_CONTAINER chown -R $(id -u):$(id -g) /artifacts
+docker exec $TORCH_BUILD_CONTAINER chown -R "$(id -u)":"$(id -g)" "${PYTORCH_ROOT}" /artifacts

ML-Frameworks/pytorch-aarch64/get-source.sh

@@ -21,36 +21,31 @@ source ../utils/git-utils.sh
 
 set -eux -o pipefail
 
-PYTORCH_HASH=5ce4a8b49f9986b050a9f6fcc7dd4cf999baa509  # 2.10.0.dev20251112 from viable/strict, Nov 12th
-IDEEP_HASH=927570638b237b0e39fb0626a868adffdbf70bbb    # From ideep_pytorch, October 20th
-ONEDNN_HASH=80886d0559482dfe2019c2ae83eebd6d0d3a17d4   # From main, Nov 9th
-TORCH_AO_HASH=17867e6788e4889b294449770f0275045384eab2 # From main, Nov 8th
-KLEIDIAI_HASH=7bf4de9a56106f0fb0d57dfabeb4c7a2668deaf6 # v1.16.0 from main, Nov 10th
+PYTORCH_HASH=93fef4bd1dd265588863929e35d9ac89328d5695  # 2.10.0.dev20251124 from viable/strict, Nov 24th
+IDEEP_HASH=3724bec97a77ce990e8c6dc5e595bb3beee75257    # From ideep_pytorch, Nov 24th
+ONEDNN_HASH=0b8a866c009b03f322e6526d7c33cfec84a4a97a   # From main, Nov 25th
+TORCH_AO_HASH=ab6bc89512d912c17a79ed8d4d709612d3e32884 # From main, Nov 25th
+KLEIDIAI_HASH=94d6cc40689f44d308dbd57cb842e335fdd958f1 # v1.17.0 from main, Nov 17th
 
 git-shallow-clone https://github.com/pytorch/pytorch.git $PYTORCH_HASH
 (
     # Apply patches to PyTorch build
     cd pytorch
 
-    # https://github.com/pytorch/pytorch/pull/160184 - Draft: separate reqs for manywheel build and pin
+    # https://github.com/pytorch/pytorch/pull/167829 - Refactor ACL and OpenBLAS install scripts on AArch64
     # Note: as part of this patch, setuptools is pinned to ~= 78.1.1 which is not affected by
     # CVE-2025-47273 and CVE-2024-6345
-    apply-github-patch pytorch/pytorch 4d344570e5a114fa522e3370c5d59161e2ed8619
+    apply-github-patch pytorch/pytorch 69db12b465887df96d27fe2bb93746ac334577f1
+    apply-github-patch pytorch/pytorch 5184c373a8bc77809b6e59361e191d4e78d6a824
+
+    # FIXME: Temporarily disabled; to be updated in a later PR
+    # # https://github.com/pytorch/pytorch/pull/160184 - Draft: separate reqs for manywheel build and pin
+    # # Note: as part of this patch, setuptools is pinned to ~= 78.1.1 which is not affected by
+    # # CVE-2025-47273 and CVE-2024-6345
+    # apply-github-patch pytorch/pytorch 4d344570e5a114fa522e3370c5d59161e2ed8619
 
     # https://github.com/pytorch/pytorch/pull/167720 - Allow missing cutlass file if CUDA disabled
-    apply-github-patch pytorch/pytorch 18f9ef2fe29b10b385f25eb6c98e3ac06227d2d9
-
-    # https://github.com/pytorch/pytorch/pull/158250 - Integrate INT4→BF16 via KleidiAI, with fallback
-    apply-github-patch pytorch/pytorch a9ec9d509167bfd33cbcd168cb40d183acf9c13a
-    apply-github-patch pytorch/pytorch 67f1076366b88c6617256236020b58da00665ed4
-    apply-github-patch pytorch/pytorch 99c57644d5d8a9359b6b98ac7bb96787ac594606
-    apply-github-patch pytorch/pytorch a770fb9a9786d7ce39a3b066809fa8c0de7d47d5
-    apply-github-patch pytorch/pytorch 30dd7406155c51b033b5e8a9c5a453fa59599db8
-    apply-github-patch pytorch/pytorch 00b919af8e7bb50f52ec45fdad09304d4104464a
-    apply-github-patch pytorch/pytorch fe40a60d7ad506aab016e66b53fdf0fc4f83b7a1
-    apply-github-patch pytorch/pytorch 89fc01183127da738fc3723747f7bf0721fe9e09
-    apply-github-patch pytorch/pytorch 23b4c39348426914cf3e6770dfaff0745245976c
-    apply-github-patch pytorch/pytorch c5e778f5d4cac56b9d96f666c3082aab244e662f
+    apply-github-patch pytorch/pytorch 6fbbfe712d89895824e466a4e3ae6a0f35626078
 
     # https://github.com/pytorch/pytorch/pull/159859 - PoC LUT optimisation for GELU bf16 operators
     apply-github-patch pytorch/pytorch ebcc874e317f9563ab770fc5c27df969e0438a5e

ML-Frameworks/tensorflow-aarch64/CHANGELOG.md

@@ -10,6 +10,7 @@ where `YY` is the year, and `MM` the month of the increment.
 ### Added
 
 ### Changed
+ - Updates TensorFlow hash to 5d46b65af45d5694cb1676bc872d24a4a64a6b57, from nightly, Nov 25th
 
 ### Removed
 

ML-Frameworks/tensorflow-aarch64/get-source.sh

@@ -21,7 +21,7 @@ source ../utils/git-utils.sh
 
 set -eux -o pipefail
 
-TENSORFLOW_HASH=f40a10e064561ed66c03359c7751ccc5ae03dc4d # from nightly, Nov 10th
+TENSORFLOW_HASH=5d46b65af45d5694cb1676bc872d24a4a64a6b57 # from nightly, Nov 25th
 
 git-shallow-clone https://github.com/tensorflow/tensorflow.git $TENSORFLOW_HASH