Merge pull request #364 from jondea/reduce-dependecies

Author: nSircombe

ML-Frameworks/pytorch-aarch64/CHANGELOG.md

@@ -14,6 +14,10 @@ where `YY` is the year, and `MM` the month of the increment.
 
 ### Removed
 - Removes WIP ComputeLibrary patch https://review.mlplatform.org/c/ml/ComputeLibrary/+/12818/1.
+- Removed unused dependencies:
+  - Delete unused submodules to avoid fetching.
+  - Replace google-perftool bundle with specific libtcmalloc-minimal4 and add symlink so that you can still use it with `LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libtcmalloc.so.4`.
+  - Remove unused git apt package.
 
 ### Fixed
 

ML-Frameworks/pytorch-aarch64/Dockerfile

@@ -39,11 +39,12 @@ RUN apt-get update && apt-get install -y \
     python-is-python3 \
     # To allow users to install new things if they want
     sudo \
-    # includes tcmalloc, which can speed up some models, see README.md for more details
-    google-perftools \
-    # Git
-    git \
-    && rm -rf /var/lib/apt/lists/*
+    # tcmalloc can speed up some models, see README.md for more details
+    # we use minimal package instead of gperftools to reduce dependencies
+    libtcmalloc-minimal4 \
+    && rm -rf /var/lib/apt/lists/* \
+    # Make libtcmalloc_minimal accessible from the usual libtcmalloc location
+    && sudo ln -s /usr/lib/aarch64-linux-gnu/libtcmalloc_minimal.so.4 /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
 
 # DOCKER_USER for the Docker user
 ENV DOCKER_USER=${USERNAME}

ML-Frameworks/pytorch-aarch64/dockerize.sh

@@ -17,6 +17,8 @@
 # limitations under the License.
 # *******************************************************************************
 
+set -eux -o pipefail
+
 help_str="dockerize.sh takes a PyTorch wheel as the first argument and an ao wheel
 as the second argument. It installs the wheel inside a Docker container with examples
 and requirements. The docker image will then be run unless you pass in the optional

ML-Frameworks/pytorch-aarch64/get-source.sh

@@ -72,6 +72,18 @@ git-shallow-clone https://github.com/pytorch/pytorch.git $PYTORCH_HASH
     # https://github.com/pytorch/pytorch/pull/159859 - PoC LUT optimisation for GELU bf16 operators
     apply-github-patch pytorch/pytorch ebcc874e317f9563ab770fc5c27df969e0438a5e
 
+    # Remove deps that we don't need for manylinux AArch64 CPU builds before fetching.
+    # Only used when jni.h is present (see .ci/pytorch/build.sh:116), which is not the case for manylinux
+    git rm android/libs/fbjni
+    # Only needed if USE_ROCM=ON, which is OFF for AArch64
+    git rm third_party/composable_kernel
+    # Not used for CPU only builds
+    git rm third_party/cudnn_frontend
+    git rm third_party/cutlass
+    git rm third_party/NVTX
+    # Only used in WIN32 builds (see CMakeLists.txt:394)
+    git rm third_party/mimalloc
+
     # Update submodules
     git submodule sync
     git submodule update --init --checkout --force --recursive --jobs=$(nproc)

ML-Frameworks/tensorflow-aarch64/dockerize.sh

@@ -17,6 +17,8 @@
 # limitations under the License.
 # *******************************************************************************
 
+set -eux -o pipefail
+
 help_str="dockerize.sh takes a TensorFlow wheel as argument. It \
 installs the wheel inside a Docker container with examples and requirements."
 if [ "$#" -ne 1 ]; then