Merge pull request #375 from puneetmatharu/patch-cleanup-for-pytorch-and-tensorflow

Patch clean-up for PyTorch and TensorFlow

Author: nSircombe

ML-Frameworks/pytorch-aarch64/build.sh

@@ -45,8 +45,34 @@ if [ -f .torch_build_container_id ] || [ -f .torch_ao_build_container_id ] || \
 
     # Wipe old build artefacts
     if [[ $* == *--fresh* ]]; then
-        if [ -f .torch_build_container_id ]; then rm -f .torch_build_container_id; fi
-        if [ -f .torch_ao_build_container_id ]; then rm -f .torch_ao_build_container_id; fi
+        # Make sure we can wipe directories created with root privileges in Docker
+        if [ -f .torch_build_container_id ]; then
+            TORCH_BUILD_CONTAINER=$(cat .torch_build_container_id)
+            if [ ! -z "$(docker ps -a --no-trunc | grep $TORCH_BUILD_CONTAINER)" ]; then
+                # Change permissions from root
+                docker exec $TORCH_BUILD_CONTAINER chown -R $(id -u):$(id -g) /artifacts 2>/dev/null || true
+                docker exec $TORCH_BUILD_CONTAINER chown -R $(id -u):$(id -g) /ComputeLibrary 2>/dev/null || true
+                docker exec $TORCH_BUILD_CONTAINER chown -R $(id -u):$(id -g) /pytorch 2>/dev/null || true
+
+                # Wipe old container
+                docker rm -f $TORCH_BUILD_CONTAINER 2>/dev/null || true
+            fi
+            rm -f .torch_build_container_id
+        fi
+
+        # Make sure we can wipe directories created with root privileges in Docker
+        if [ -f .torch_ao_build_container_id ]; then
+            TORCH_AO_BUILD_CONTAINER=$(cat .torch_ao_build_container_id)
+            if [ ! -z "$(docker ps -a --no-trunc | grep $TORCH_AO_BUILD_CONTAINER)" ]; then
+                docker exec $TORCH_AO_BUILD_CONTAINER chown -R $(id -u):$(id -g) /ao
+
+                # Wipe old container
+                docker rm -f $TORCH_AO_BUILD_CONTAINER 2>/dev/null || true
+            fi
+            rm -f .torch_ao_build_container_id
+        fi
+
+        # Wipe the other directories; we should have the privileges now
         if [ -d ao ]; then rm -rf ao; fi
         if [ -d ComputeLibrary ]; then rm -rf ComputeLibrary; fi
         if [ -d pytorch ]; then rm -rf pytorch; fi

ML-Frameworks/tensorflow-aarch64/CHANGELOG.md

@@ -25,6 +25,7 @@ https://github.com/ARM-software/Tool-Solutions/tree/r25.09
 ### Changed
  - Updates TensorFLow hash to 6aa8fd07270293b918255af5988aef45b844c5b4 # from nightly, September 22nd
  - Updates `dockerize.sh` build to use BuildKit (to add extra build contexts)
+ - Change of flag name in `./build.sh` from `--force` to `--fresh`
 
 ### Removed
  - Removed previous patch updating oneDNN and Compute Library that has been merged upstream

ML-Frameworks/tensorflow-aarch64/build.sh

@@ -25,13 +25,41 @@ exec &> >(tee -a $build_log)
 
 # # Bail out if sources are already there
 if [ -d tensorflow ] ; then
-    echo "You appear to have sources already" \
+    printf "\n\n%s\n%s\n%s\n\n\n" \
+        "You appear to have artefacts from a previous build lying around." \
+        "Check for the following:" \
+        "  - tensorflow"
 
-    if ! ([[ $* == *--force* ]] || [[ $* == *--use-existing-sources* ]]) ; then
-        echo "rerun with --force to overwrite sources or with" \
-                 "--use-existing-sources to build your existing sources." 1>&2
+    if ! ([[ $* == *--fresh* ]] || [[ $* == *--use-existing-sources* ]]) ; then
+        printf "\n\n%s\n%s\n%s\n\n\n" \
+                "Rerun with one of the following options:" \
+                "  - '--fresh': wipe the pre-existing sources and do a fresh build" \
+                "  - '--use-existing-sources': reuse the sources as is" 1>&2
         exit 1
     fi
+
+    # Wipe old build artefacts
+    if [[ $* == *--fresh* ]]; then
+        if [ -d tensorflow ]; then
+            # Change permissions for folders created as root in docker
+            if [ -d tensorflow/build_output ]; then
+                if [ ! -z "$(docker ps -a --no-trunc | grep tf)" ]; then
+                    docker exec tf chown -R $(id -u):$(id -g) build_output 2>/dev/null || true
+                else
+                    printf "\n\n%s\n%s\n%s\n\n\n" \
+                        "Unable to locate docker container 'tf'. You may need to" \
+                        "rerun this script with sudo privileges to make sure everything" \
+                        "has been properly wiped."
+                fi
+            fi
+
+            # Wipe the container. Adapted from: tensorflow/ci/official/utilities/cleanup_docker.sh
+            docker rm -f tf 2>/dev/null || true
+
+            # Wipe the folder
+            rm -rf tensorflow
+        fi
+    fi
 fi
 
 if ! [[ $* == *--use-existing-sources* ]]; then