build: speed up license extraction

Apparently licensecheck has a high startup cost, so merging invocations
promises a neat speedup. However, currently some "\x{....} cannot be
represented as ascii" error messages appear in our logs. If that happens
licensecheck exits with 255, which omits all following files of the
same invocation and also prompts `find -exec .. {} +` or `xargs` to not
spwan any more invocations. If we were to simply merge invocations by
one of those means it would result in an incomplete COPYRIGHT file.

Those encoding errors appear to be due to an ASCII locale being set in
the container, so override LC_ALL to an UTF-8 one. To further make this
bit more resilient ensure licensecheck errors are no longer ignored by
capturing and processing its exit code.

Author: TheOneric

.github/workflows/emscripten.yml

@@ -63,7 +63,7 @@ jobs:
     - name: Build Binaries
       run: |
         mkdir -p /tmp/emcc_lto
-        docker run --rm -v "${PWD}":/code -v "/tmp/emcc_lto:/emsdk/upstream/emscripten/cache/sysroot/lib/wasm32-emscripten/lto" libass/jso:latest
+        docker run --rm --env LC_ALL=C.UTF-8 -v "${PWD}":/code -v "/tmp/emcc_lto:/emsdk/upstream/emscripten/cache/sysroot/lib/wasm32-emscripten/lto" libass/jso:latest
 
     - name: Upload Nightly Build
       uses: actions/upload-artifact@v3

build/license_extract.sh

@@ -41,9 +41,16 @@ else
 fi
 
 
-find "$base_dir" $FINDOPTS -type f -regextype egrep -regex '.*\.(c|h|cpp|hpp|js)$' -exec \
-    licensecheck --machine --copyright --deb-fmt '{}' \; \
-    | awk -F"$tabulator" -v base_dir="$base_dir" \
+FIFO="$base_dir"/__LICENSE_EXTRACT_QUEUE.tmp
+mkfifo "$FIFO"
+# We want to be able to clean up the named pipe on error
+# and will check the exit codes ourselves
+set +e
+
+find "$base_dir" $FINDOPTS -type f -regextype egrep -regex '.*\.(c|h|cpp|hpp|js)$' -print0 \
+    | xargs -0 -P1 licensecheck --machine --copyright --deb-fmt --encoding UTF-8 > "$FIFO"\
+	& scan_pid="$!"
+awk -F"$tabulator" -v base_dir="$base_dir" \
         -v def_license="$def_license" -v def_copy="$def_copy" '
             BEGIN {
                 split("", lcfiles)     # Clear array with only pre-Issue 8 POSIX
@@ -100,4 +107,10 @@ find "$base_dir" $FINDOPTS -type f -regextype egrep -regex '.*\.(c|h|cpp|hpp|js)
                     printf "\n"
                 }
             }
-        '
+        ' \
+	< "$FIFO"
+fret="$?"
+wait "$scan_pid"
+sret="$?"
+rm "$FIFO"
+exit "$((fret | sret))"

run-buildah-build.sh

@@ -15,4 +15,4 @@ if [ "$FAST" -eq 0 ] ; then
     buildah rm "$CONTAINER" >/dev/null 2>&1 || :
     buildah from --name "$CONTAINER" "$IMAGE":latest
 fi
-buildah run -t -v "${PWD}":/code "$CONTAINER" $cmd "$@"
+buildah run -t --env LC_ALL=C.UTF-8 -v "${PWD}":/code "$CONTAINER" $cmd "$@"

run-docker-build.sh

@@ -8,7 +8,7 @@ if [ "$FAST" -eq 0 ] ; then
     docker build -t "$IMAGE" .
 fi
 if [ "$#" -eq 0 ] ; then
-    docker run --rm -v "${PWD}":/code --name "$CONTAINER" "$IMAGE":latest
+    docker run --rm --env LC_ALL=C.UTF-8 -v "${PWD}":/code --name "$CONTAINER" "$IMAGE":latest
 else
-    docker run --rm -v "${PWD}":/code --name "$CONTAINER" "$IMAGE":latest "$@"
+    docker run --rm --env LC_ALL=C.UTF-8 -v "${PWD}":/code --name "$CONTAINER" "$IMAGE":latest "$@"
 fi