Alexeyk/gitlab cache metadata recovery

.gitlab-ci.yml

@@ -249,6 +249,59 @@ default:
     # GitLab's cache helper restores .gradle as root, but we run as non-root-user (uid 1001),
     # and Gradle does `chmod 700 .gradle` on startup which requires user ownership.
     - sudo chown -R 1001:1001 .gradle
+    # A partial/aborted GitLab cache extraction (the runner logs "FATAL: unexpected EOF" /
+    # "Failed to extract cache" then continues anyway) can leave a Gradle immutable-workspace
+    # (dependencies-accessors, groovy-dsl, kotlin-dsl, transforms) whose metadata.bin is either
+    # TRUNCATED (EOFException) or entirely MISSING (FileNotFoundException). Either way Gradle
+    # hard-fails during configuration with "Could not read workspace metadata" and does NOT
+    # self-heal. Capture evidence (good + damaged) for diagnosis, then use Gradle's own metadata
+    # reader to identify and drop only the damaged workspace dirs so Gradle regenerates them (only
+    # the entries that would fail anyway are removed). See gradle/gradle#28974.
+    - |
+      GRADLE_METADATA_EVIDENCE_DIR="gradle-cache-metadata"
+      mkdir -p "$GRADLE_METADATA_EVIDENCE_DIR"
+      # Manifest of every metadata.bin with its byte size (damaged ones sort to the top at 0/near-0).
+      find .gradle/caches -type f -name metadata.bin -printf '%10s  %p\n' 2>/dev/null \
+        | sort -n > "$GRADLE_METADATA_EVIDENCE_DIR/metadata-manifest.txt" || true
+      damaged=""
+      # ValidateGradleMetadata enumerates the immutable-workspace dirs, skips Gradle temporary
+      # workspaces, and prints damaged ones as "<size>\t<reason>\t<workspace>" (exit 1 via wrapper).
+      validator_output="$GRADLE_METADATA_EVIDENCE_DIR/validator-output.txt"
+      validator_error="$GRADLE_METADATA_EVIDENCE_DIR/validator-error.txt"
+      validator_status=0
+      .gitlab/validate_gradle_metadata.sh "$GRADLE_VERSION" \
+        > "$validator_output" 2> "$validator_error" || validator_status=$?
+      if [ "$validator_status" -eq 1 ]; then
+        while IFS=$'\t' read -r size reason ws; do
+          [ -n "$ws" ] || continue
+          [ -d "$ws" ] || continue
+          meta="${ws}/metadata.bin"
+          if [ -z "$damaged" ]; then
+            echo -e "${TEXT_BOLD}${TEXT_YELLOW}[WARNING] Damaged Gradle metadata found, fixed:${TEXT_CLEAR}"
+            damaged="yes"
+          fi
+          dest="$GRADLE_METADATA_EVIDENCE_DIR/corrupt/${ws}"
+          mkdir -p "$dest" || true
+          if [ -f "$meta" ]; then cp -p "$meta" "$dest/metadata.bin" 2>/dev/null || true; fi
+          ls -la "$ws" > "$dest/dir-listing.txt" 2>/dev/null || true
+          echo "  - ${ws} (metadata.bin size=${size}; ${reason})"
+          rm -rf "$ws" || true
+        done < "$validator_output"
+      elif [ "$validator_status" -ne 0 ]; then
+        echo -e "${TEXT_BOLD}${TEXT_YELLOW}[WARNING] Gradle metadata validator unavailable; leaving cache unchanged.${TEXT_CLEAR}"
+        cat "$validator_error" || true
+      fi
+      if [ -z "$damaged" ]; then echo "No damaged Gradle immutable-workspace metadata detected."; fi
+      # Keep a few intact metadata.bin files for byte-level comparison with the damaged ones.
+      # `find | head` makes find die with SIGPIPE once head closes the pipe after 10 lines; under
+      # `set -o pipefail` (GitLab's bash default) that non-zero would propagate as this block's exit
+      # code and, being the shared default before_script, fail every build job. This is best-effort
+      # diagnostics and must never fail the job, so guard the pipeline with `|| true` like the rest.
+      find .gradle/caches -type f -name metadata.bin -size +32c 2>/dev/null | head -n 10 | while IFS= read -r f; do
+        dest="$GRADLE_METADATA_EVIDENCE_DIR/good/$(dirname "$f")"
+        mkdir -p "$dest" || true
+        cp -p "$f" "$dest/metadata.bin" 2>/dev/null || true
+      done || true
   after_script:
     - *cgroup_info
     - *container_info

.gitlab/collect_reports.sh

@@ -42,6 +42,12 @@ cp /tmp/*.trc $REPORTS_DIR 2>/dev/null || true
 cp /tmp/*.dmp $REPORTS_DIR 2>/dev/null || true
 cp /tmp/dd-profiler/*.jfr $REPORTS_DIR 2>/dev/null || true
 
+# Gradle immutable-workspace metadata evidence (good + corrupt metadata.bin) captured in the
+# gradle_build before_script before corrupt entries are cleared. See .gitlab-ci.yml.
+if [ -d ./gradle-cache-metadata ]; then
+  cp -r ./gradle-cache-metadata "$REPORTS_DIR/" 2>/dev/null || true
+fi
+
 function process_reports () {
     project_to_save=$1
     report_path=$REPORTS_DIR/$project_to_save

.gitlab/gradle-cache/ValidateGradleMetadata.java

@@ -0,0 +1,163 @@
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * Validates Gradle immutable-workspace metadata files before CI decides whether to clear them.
+ *
+ * <p>Given a Gradle version, enumerates the immutable-workspace dirs under the project-local cache
+ * and, for each, asks Gradle's own reader to deserialize its {@code metadata.bin}. Damaged
+ * (truncated or missing) entries are printed to stdout as {@code <size>\t<reason>\t<workspace>} and
+ * the process exits {@code 65} (EX_DATAERR). Any other failure means the validator could not run,
+ * so it exits {@code 2} and CI leaves the cache untouched rather than mistaking an unavailable
+ * validator for cache corruption.
+ */
+class ValidateGradleMetadata {
+  private static final int EXIT_VALID = 0;
+  private static final int EXIT_DAMAGED = 65;
+  private static final int EXIT_UNAVAILABLE = 2;
+
+  // CI always runs from the repository root, where the project-local Gradle cache lives.
+  private static final Path CACHES_DIR = Path.of(".gradle/caches");
+
+  // Folders with the directory depth to check.
+  private static final Map<String, Integer> WORKSPACE_CATEGORIES =
+      Map.of(
+          "dependencies-accessors", 1,
+          "groovy-dsl", 1,
+          "kotlin-dsl", 2,
+          "transforms", 1);
+
+  private static final Pattern TEMPORARY_WORKSPACE =
+      Pattern.compile(".*-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}");
+
+  // Gradle's own metadata reader, resolved reflectively.
+  private static Object metadataStore;
+  private static Method metadataLoadMethod;
+
+  public static void main(String[] args) {
+    try {
+      loadMetadataReader();
+
+      var damaged = false;
+      for (var workspace : enumerateWorkspaces(args[0])) {
+        var result = validate(workspace);
+        if (result != null) {
+          damaged = true;
+          System.out.printf("%s\t%s\t%s%n", result.size(), result.reason(), result.workspace());
+        }
+      }
+      System.exit(damaged ? EXIT_DAMAGED : EXIT_VALID);
+    } catch (Exception e) {
+      System.err.println("Gradle metadata validator unavailable: " + summarize(e));
+      System.exit(EXIT_UNAVAILABLE);
+    }
+  }
+
+  private static List<Path> enumerateWorkspaces(String gradleVersion) throws IOException {
+    var versionDir = CACHES_DIR.resolve(gradleVersion);
+    var workspaces = new ArrayList<Path>();
+    for (var category : WORKSPACE_CATEGORIES.entrySet()) {
+      collectAtDepth(versionDir.resolve(category.getKey()), category.getValue(), workspaces);
+    }
+    return workspaces;
+  }
+
+  private static void collectAtDepth(Path dir, int depth, List<Path> out) throws IOException {
+    if (!Files.isDirectory(dir)) {
+      return;
+    }
+    if (depth == 0) {
+      out.add(dir);
+      return;
+    }
+    for (var child : childDirectories(dir)) {
+      collectAtDepth(child, depth - 1, out);
+    }
+  }
+
+  private static List<Path> childDirectories(Path dir) throws IOException {
+    try (var entries = Files.list(dir)) {
+      return entries.filter(Files::isDirectory).collect(Collectors.toList());
+    }
+  }
+
+  private static void loadMetadataReader() {
+    try {
+      var storeClass =
+          Class.forName("org.gradle.internal.execution.history.impl.DefaultImmutableWorkspaceMetadataStore");
+      metadataStore = storeClass.getDeclaredConstructor().newInstance();
+      metadataLoadMethod = storeClass.getMethod("loadWorkspaceMetadata", java.io.File.class);
+    } catch (ReflectiveOperationException | LinkageError e) {
+      throw new IllegalStateException("could not load Gradle metadata reader: " + summarize(e));
+    }
+  }
+
+  private static DamagedWorkspace validate(Path workspace) {
+    // Gradle temporary workspaces are named <immutable-workspace>-<uuid> and intentionally may not
+    // have metadata.bin until Gradle moves them into their immutable location.
+    if (TEMPORARY_WORKSPACE.matcher(workspace.getFileName().toString()).matches()) {
+      return null;
+    }
+
+    var metadata = workspace.resolve("metadata.bin");
+    if (!Files.isRegularFile(metadata)) {
+      return new DamagedWorkspace(workspace, "missing", "metadata.bin is missing");
+    }
+
+    String size;
+    try {
+      size = Long.toString(Files.size(metadata));
+    } catch (IOException e) {
+      return new DamagedWorkspace(workspace, "unknown", "metadata.bin size check failed");
+    }
+
+    // A successful return means Gradle's own reader fully deserialized metadata.bin;
+    // a truncated file throws mid-read and surfaces here as an InvocationTargetException.
+    try {
+      metadataLoadMethod.invoke(metadataStore, workspace.toFile());
+      return null;
+    } catch (InvocationTargetException e) {
+      var cause = e.getCause();
+      if (cause instanceof LinkageError) {
+        throw new IllegalStateException(
+            "could not execute Gradle metadata reader: " + summarize(cause));
+      }
+      return new DamagedWorkspace(workspace, size, summarize(cause));
+    } catch (ReflectiveOperationException | RuntimeException e) {
+      throw new IllegalStateException("could not execute Gradle metadata reader: " + summarize(e));
+    }
+  }
+
+  private record DamagedWorkspace(Path workspace, String size, String reason) {
+  }
+
+  private static String summarize(Throwable throwable) {
+    if (throwable == null) {
+      return "unknown failure";
+    }
+
+    var root = throwable;
+    while ((root instanceof InvocationTargetException
+        || root instanceof UncheckedIOException
+        || root.getClass().getName().equals("org.gradle.internal.UncheckedException"))
+        && root.getCause() != null) {
+      root = root.getCause();
+    }
+
+    var message = root.getMessage();
+    if (message == null || message.isBlank()) {
+      return root.getClass().getSimpleName();
+    }
+
+    return root.getClass().getSimpleName() + ": " + message.replace('\t', ' ');
+  }
+}

.gitlab/validate_gradle_metadata.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+java_home="${JAVA_25_HOME:-}"
+if [[ -n "$java_home" ]]; then
+  java_bin="$java_home/bin/java"
+  javac_bin="$java_home/bin/javac"
+else
+  java_bin="java"
+  javac_bin="javac"
+fi
+
+if [[ "$java_bin" != "java" && ! -x "$java_bin" ]]; then
+  echo "Gradle metadata validator could not find Java executable: $java_bin" >&2
+  exit 2
+fi
+
+script_dir="$(cd "$(dirname "$0")" && pwd)"
+gradle_version="${GRADLE_VERSION:-}"
+if [[ -z "$gradle_version" && -f gradle/wrapper/gradle-wrapper.properties ]]; then
+  gradle_version="$(
+    sed -nE 's#^distributionUrl=.*gradle-([0-9][^-]*)-(bin|all)\.zip$#\1#p' \
+      gradle/wrapper/gradle-wrapper.properties \
+      | head -n 1
+  )"
+fi
+
+if [[ -z "$gradle_version" ]]; then
+  echo "Gradle metadata validator could not determine Gradle version" >&2
+  exit 2
+fi
+
+gradle_lib=""
+for gradle_home in "${GRADLE_USER_HOME:-$PWD/.gradle}" "$HOME/.gradle"; do
+  [[ -d "$gradle_home/wrapper/dists" ]] || continue
+  gradle_lib="$(
+    find "$gradle_home/wrapper/dists" \
+      -path "*/gradle-${gradle_version}/lib" \
+      -type d \
+      -print \
+      -quit 2>/dev/null
+  )"
+  if [[ -n "$gradle_lib" ]]; then
+    break
+  fi
+done
+
+if [[ -z "$gradle_lib" ]]; then
+  echo \
+    "Gradle metadata validator could not find Gradle $gradle_version distribution lib directory" \
+    >&2
+  exit 2
+fi
+
+# Pre-compile with -proc:none rather than using the `java <file>.java` source launcher: the launcher
+# would otherwise discover and run annotation processors bundled in the Gradle jars on the classpath.
+build_dir="$(mktemp -d)"
+trap 'rm -rf "$build_dir"' EXIT
+
+if ! "$javac_bin" -proc:none -classpath "$gradle_lib/*" -d "$build_dir" \
+    "$script_dir/gradle-cache/ValidateGradleMetadata.java"; then
+  echo "Gradle metadata validator could not compile ValidateGradleMetadata" >&2
+  exit 2
+fi
+
+set +e
+"$java_bin" -classpath "$build_dir:$gradle_lib/*" ValidateGradleMetadata "$@"
+status=$?
+set -e
+
+# ValidateGradleMetadata exits 65 (EX_DATAERR) for damaged metadata so that a JVM that exits 1
+# before main() runs is treated as validator-unavailable instead of as cache corruption.
+case "$status" in
+  0) exit 0 ;;
+  65) exit 1 ;;
+  *) exit 2 ;;
+esac