Merge pull request #329 from nSircombe/feature/may_updates

Add option for auth. GitHub access and update CHANGELOGS for 25.05

Author: jondea

ML-Frameworks/pytorch-aarch64/CHANGELOG.md

@@ -7,9 +7,21 @@ where `YY` is the year, and `MM` the month of the increment.
 
 ## [unreleased]
 
+### Added
+
+### Changed
+
+### Removed
+
+### Fixed
+
+### [r25.04] 2025-05-18
+https://github.com/ARM-software/Tool-Solutions/tree/r25.05
+
 ### Added
  - WIP patch to build a newer version of libgomp from source - https://github.com/pytorch/pytorch/pull/152361
    Improve scaling for >16 threads.
+ - Support for authenticated GitHub access in apply-github-patch
 
 ### Changed
  - Updates hashes for:

ML-Frameworks/pytorch-aarch64/README.md

@@ -54,6 +54,10 @@ You can then test your changes by installing the wheel in a virtual environment
 of your choice or use `./dockerize.sh <name-of-wheel>` to launch a container
 with the wheel installed (along with examples).
 
+Note: if the environment variable `GITHUB_TOKEN` is set then the build will
+attemmpt to use `GITHUB_TOKEN` for authenticated access when downloading WIP patches.
+This can avoid issues with rate-limiting on annonymous access.
+
 ### Flags useful for development
 - `--use-existing-sources` skips `get-source.sh` and just builds
 - `--force` overwrites sources

ML-Frameworks/pytorch-aarch64/get-source.sh

@@ -30,11 +30,15 @@ git-shallow-clone https://github.com/pytorch/pytorch.git $PYTORCH_HASH
 (
     cd pytorch
 
-    apply-github-patch pytorch/pytorch 143190 6e5628b1f648d862e8fdd150ad277120b236ed15 # Enable AArch64 CI scripts to be used for local dev
-    apply-github-patch pytorch/pytorch 140159 ca4a718be80eb88ca6804b91201e4f98a3e236c8 # cpu: enable gemm-bf16f32 for SDPA BF16
-    apply-github-patch pytorch/pytorch 140159 406fe1fbd066401774c104d125a7ac0b3d6eb52b
-    apply-github-patch pytorch/pytorch 152361 7c54b6b07558c330ee2f95b4793edb3bfbb814c9 # Build libgomp (gcc-11) from source
-    apply-github-patch pytorch/pytorch 150833 02987a7c2e9b249a669723224c8d3cd80c6cb64e # Pin all root requirements to major versions
+    # https://github.com/pytorch/pytorch/pull/143190 - Enable AArch64 CI scripts to be used for local dev
+    apply-github-patch pytorch/pytorch 6e5628b1f648d862e8fdd150ad277120b236ed15
+    # https://github.com/pytorch/pytorch/pull/140159 - cpu: enable gemm-bf16f32 for SDPA BF16
+    apply-github-patch pytorch/pytorch ca4a718be80eb88ca6804b91201e4f98a3e236c8
+    apply-github-patch pytorch/pytorch 406fe1fbd066401774c104d125a7ac0b3d6eb52b
+    # https://github.com/pytorch/pytorch/pull/152361 - Build libgomp (gcc-11) from source
+    apply-github-patch pytorch/pytorch 7c54b6b07558c330ee2f95b4793edb3bfbb814c9
+    # https://github.com/pytorch/pytorch/pull/150833 - Pin all root requirements to major versions
+    apply-github-patch pytorch/pytorch 02987a7c2e9b249a669723224c8d3cd80c6cb64e
 
     git submodule sync
     git submodule update --init --checkout --force --recursive --jobs=$(nproc)
@@ -45,19 +49,22 @@ git-shallow-clone https://github.com/pytorch/pytorch.git $PYTORCH_HASH
         (
             cd mkl-dnn
             git fetch origin $ONEDNN_HASH && git clean -f && git checkout -f FETCH_HEAD
-            apply-github-patch uxlfoundation/oneDNN 3022 4a00e92b995388192e666ee332554e4ef65b484a # cpu: aarch64: enable jit conv for 128
+            # https://github.com/uxlfoundation/oneDNN/pull/3922 - cpu: aarch64: enable jit conv for 128
+            apply-github-patch uxlfoundation/oneDNN 4a00e92b995388192e666ee332554e4ef65b484a
         )
     )
 )
 
 git-shallow-clone https://review.mlplatform.org/ml/ComputeLibrary $ACL_HASH
 (
     cd ComputeLibrary
-    apply-gerrit-patch https://review.mlplatform.org/c/ml/ComputeLibrary/+/12818/1 # perf: Improve gemm_interleaved 2D vs 1D blocking heuristic
+    # Improve gemm_interleaved 2D vs 1D blocking heuristic
+    apply-gerrit-patch https://review.mlplatform.org/c/ml/ComputeLibrary/+/12818/1
 )
 
 git-shallow-clone https://github.com/pytorch/ao.git $TORCH_AO_HASH
 (
     cd ao
-    apply-github-patch pytorch/ao 1447 738d7f2c5a48367822f2bf9d538160d19f02341e # [Feat]: Add support for kleidiai quantization schemes
+    # https://github.com/pytorch/ao/pull/1447 - Add support for kleidiai quantization schemes
+    apply-github-patch pytorch/ao 738d7f2c5a48367822f2bf9d538160d19f02341e
 )

ML-Frameworks/tensorflow-aarch64/CHANGELOG.md

@@ -15,6 +15,18 @@ where `YY` is the year, and `MM` the month of the increment.
 
 ### Fixed
 
+#### [r25.04] 2025-05-18
+https://github.com/ARM-software/Tool-Solutions/tree/r25.05
+
+### Added
+ - Support for authenticated GitHub access in apply-github-patch
+
+### Changed
+
+### Removed
+
+### Fixed
+
 ## [r25.04] 2025-04-16
 https://github.com/ARM-software/Tool-Solutions/tree/r25.04
 

ML-Frameworks/tensorflow-aarch64/README.md

@@ -54,6 +54,10 @@ You can then test your changes by installing the wheel in a virtual environment
 of your choice or use `./dockerize.sh <name-of-wheel>` to launch a container
 with the wheel installed (along with examples).
 
+Note: if the environment variable `GITHUB_TOKEN` is set then the build will
+attemmpt to use `GITHUB_TOKEN` for authenticated access when downloading WIP patches.
+This can avoid issues with rate-limiting on annonymous access.
+
 ## Motivation
 TensorFlow + oneDNN + ComputeLibrary is a deep stack. The purpose of
 `tensorflow-aarch64` is to let us see the future of that stack, so that we can:

ML-Frameworks/tensorflow-aarch64/get-source.sh

@@ -37,7 +37,8 @@ git-shallow-clone https://github.com/tensorflow/tensorflow.git $TENSORFLOW_HASH
     cd tensorflow
 
     # Apply TensorFlow WIP patches here
-    apply-github-patch tensorflow/tensorflow 84975 1ca7978322313cd62733075ea354f2af5d1e54a0 # build(aarch64): Update to oneDNN-3.7 + ACL-24.12
+    # https://github.com/tensorflow/tensorflow/pull/84975 - build(aarch64): Update to oneDNN-3.7 + ACL-24.12
+    apply-github-patch tensorflow/tensorflow 1ca7978322313cd62733075ea354f2af5d1e54a0
 
     cd tensorflow
 
@@ -68,7 +69,8 @@ git-shallow-clone https://github.com/tensorflow/tensorflow.git $TENSORFLOW_HASH
 
         # Apply WIP patches here
         cd oneDNN
-        apply-github-patch uxlfoundation/oneDNN 2958 ce72a428594c58e925de38c5eb6fea725fe9d0ff # cpu: aarch64: default num_threads to max for acl_threadpool
+        # https://github.com/uxlfoundation/oneDNN/pull/2958 - set default num_threads to max for acl_threadpool
+        apply-github-patch uxlfoundation/oneDNN ce72a428594c58e925de38c5eb6fea725fe9d0ff
     )
 
     # ACL patches

ML-Frameworks/utils/git-utils.sh

@@ -35,26 +35,25 @@ function git-shallow-clone {
 }
 
 function apply-github-patch {
-    # Apply a specific commit from a specific GitHub PR
-    # $1 is 'organisation/repo', $2 is the PR number, and $3 is commit hash
-    set -u
+    # Apply a specific GitHub commit.
+    # $1 is 'organisation/repo', $2 is the commit hash
+    # To use an API token, which may avoid rate limits, set the environment variable GITHUB_TOKEN
 
-	local github_url='https://github.com'
+    set -u
 
-    # Look in the PR first
-    curl --silent -L $github_url/$1/pull/$2/commits/$3.patch -o $3.patch
+    local github_api_url='https://api.github.com/repos'
+    local github_url='https://github.com'
 
-    # If the PR has been updated, the commit may no longer be there and the .patch will be empty.
-    # Look in the full repo instead.
-    # If it can't be found, this time curl will error
-    if [[ ! -s $3.patch ]]; then
-       >&2 echo "Commit $3 not found in $1/pull/$2. Checking the full repository..."
-       curl --silent --fail -L $github_url/$1/commit/$3.patch -o $3.patch
+    # Download the .patch file.
+    if [ -n "$GITHUB_TOKEN" ]; then
+        curl --silent -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.patch" -L $github_api_url/$1/commits/$2 -o $2.patch
+    else
+        curl --silent -L $github_url/$1/commit/$2.patch -o $2.patch
     fi
 
     # Apply the patch and tidy up.
-    patch -p1 < $3.patch
-    rm $3.patch
+    patch -p1 < $2.patch
+    rm $2.patch
     return 0
 }