From 888f1a2cd68147ad5af8ccfec5fd4e1e628b0f67 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Sat, 30 May 2026 14:00:58 -0400 Subject: [PATCH 1/6] Band-Aid in case of corrupted cache. --- .gitlab-ci.yml | 42 ++++++++++++++++++++++++++++++++++++++ .gitlab/collect_reports.sh | 6 ++++++ 2 files changed, 48 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8177e7bc54..a0feed61789 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -249,6 +249,48 @@ default: # GitLab's cache helper restores .gradle as root, but we run as non-root-user (uid 1001), # and Gradle does `chmod 700 .gradle` on startup which requires user ownership. - sudo chown -R 1001:1001 .gradle + # A partial/aborted GitLab cache extraction (the runner logs "FATAL: unexpected EOF" / + # "Failed to extract cache" then continues anyway) can leave a Gradle immutable-workspace + # (dependencies-accessors, groovy-dsl, kotlin-dsl, transforms) whose metadata.bin is either + # TRUNCATED (EOFException) or entirely MISSING (FileNotFoundException). Either way Gradle + # hard-fails during configuration with "Could not read workspace metadata" and does NOT + # self-heal. A valid metadata.bin is ~110 bytes. Capture evidence (good + damaged) for + # diagnosis, then drop only the damaged workspace dirs so Gradle regenerates them (only the + # entries that would fail anyway are removed). See gradle/gradle#28974. + - | + GRADLE_METADATA_EVIDENCE_DIR="gradle-cache-metadata" + mkdir -p "$GRADLE_METADATA_EVIDENCE_DIR" + # Manifest of every metadata.bin with its byte size (damaged ones sort to the top at 0/near-0). + find .gradle/caches -type f -name metadata.bin -printf '%10s %p\n' 2>/dev/null \ + | sort -n > "$GRADLE_METADATA_EVIDENCE_DIR/metadata-manifest.txt" || true + damaged="" + for ws in .gradle/caches/*/dependencies-accessors/*/ \ + .gradle/caches/*/groovy-dsl/*/ \ + .gradle/caches/*/kotlin-dsl/*/ \ + .gradle/caches/*/transforms/*/; do + [ -d "$ws" ] || continue + meta="${ws}metadata.bin" + size=$(stat -c%s "$meta" 2>/dev/null || echo 0) + if [ ! -f "$meta" ] || [ "$size" -lt 32 ]; then + if [ -z "$damaged" ]; then + echo -e "${TEXT_BOLD}${TEXT_YELLOW}WARNING: damaged Gradle immutable-workspace metadata (truncated/missing metadata.bin from a partial cache extraction); preserving evidence and clearing so Gradle regenerates:${TEXT_CLEAR}" + damaged="yes" + fi + dest="$GRADLE_METADATA_EVIDENCE_DIR/corrupt/${ws}" + mkdir -p "$dest" || true + if [ -f "$meta" ]; then cp -p "$meta" "$dest/metadata.bin" 2>/dev/null || true; fi + ls -la "$ws" > "$dest/dir-listing.txt" 2>/dev/null || true + echo " - ${ws} (metadata.bin size=${size})" + rm -rf "$ws" || true + fi + done + if [ -z "$damaged" ]; then echo "No damaged Gradle immutable-workspace metadata detected."; fi + # Keep a few intact metadata.bin files for byte-level comparison with the damaged ones. + find .gradle/caches -type f -name metadata.bin -size +32c 2>/dev/null | head -n 10 | while IFS= read -r f; do + dest="$GRADLE_METADATA_EVIDENCE_DIR/good/$(dirname "$f")" + mkdir -p "$dest" || true + cp -p "$f" "$dest/metadata.bin" 2>/dev/null || true + done after_script: - *cgroup_info - *container_info diff --git a/.gitlab/collect_reports.sh b/.gitlab/collect_reports.sh index 6b16d7da472..024e01fed3f 100755 --- a/.gitlab/collect_reports.sh +++ b/.gitlab/collect_reports.sh @@ -42,6 +42,12 @@ cp /tmp/*.trc $REPORTS_DIR 2>/dev/null || true cp /tmp/*.dmp $REPORTS_DIR 2>/dev/null || true cp /tmp/dd-profiler/*.jfr $REPORTS_DIR 2>/dev/null || true +# Gradle immutable-workspace metadata evidence (good + corrupt metadata.bin) captured in the +# gradle_build before_script before corrupt entries are cleared. See .gitlab-ci.yml. +if [ -d ./gradle-cache-metadata ]; then + cp -r ./gradle-cache-metadata "$REPORTS_DIR/" 2>/dev/null || true +fi + function process_reports () { project_to_save=$1 report_path=$REPORTS_DIR/$project_to_save From 9d4ade192fc0f5bdb79ad97b68fda1f8f1f16ba1 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Sat, 30 May 2026 14:32:01 -0400 Subject: [PATCH 2/6] Fixed script. --- .gitlab-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a0feed61789..b79b86e7af8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -286,11 +286,15 @@ default: done if [ -z "$damaged" ]; then echo "No damaged Gradle immutable-workspace metadata detected."; fi # Keep a few intact metadata.bin files for byte-level comparison with the damaged ones. + # `find | head` makes find die with SIGPIPE once head closes the pipe after 10 lines; under + # `set -o pipefail` (GitLab's bash default) that non-zero would propagate as this block's exit + # code and, being the shared default before_script, fail every build job. This is best-effort + # diagnostics and must never fail the job, so guard the pipeline with `|| true` like the rest. find .gradle/caches -type f -name metadata.bin -size +32c 2>/dev/null | head -n 10 | while IFS= read -r f; do dest="$GRADLE_METADATA_EVIDENCE_DIR/good/$(dirname "$f")" mkdir -p "$dest" || true cp -p "$f" "$dest/metadata.bin" 2>/dev/null || true - done + done || true after_script: - *cgroup_info - *container_info From a6c7a58efc070653f620aefa21c0a9fce3368dc6 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Sat, 30 May 2026 15:44:34 -0400 Subject: [PATCH 3/6] Fixed false-positive. --- .gitlab-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b79b86e7af8..28a6c746b1b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -264,9 +264,13 @@ default: find .gradle/caches -type f -name metadata.bin -printf '%10s %p\n' 2>/dev/null \ | sort -n > "$GRADLE_METADATA_EVIDENCE_DIR/metadata-manifest.txt" || true damaged="" + # kotlin-dsl nests workspaces one level deeper than the others: its metadata.bin lives at + # kotlin-dsl/{accessors,scripts}//metadata.bin, so it needs an extra glob level. The + # category dirs (kotlin-dsl/accessors/, kotlin-dsl/scripts/) never hold a metadata.bin and + # would otherwise be false-flagged as damaged (size=0) on every run. for ws in .gradle/caches/*/dependencies-accessors/*/ \ .gradle/caches/*/groovy-dsl/*/ \ - .gradle/caches/*/kotlin-dsl/*/ \ + .gradle/caches/*/kotlin-dsl/*/*/ \ .gradle/caches/*/transforms/*/; do [ -d "$ws" ] || continue meta="${ws}metadata.bin" From 386c24217a5dc7726af1aaa32ee8f2538fa0f157 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Sat, 30 May 2026 16:01:49 -0400 Subject: [PATCH 4/6] Refactored to Java checker. --- .gitlab-ci.yml | 69 +++++-- .../gradle-cache/ValidateGradleMetadata.java | 181 ++++++++++++++++++ .gitlab/validate_gradle_metadata.sh | 75 ++++++++ 3 files changed, 305 insertions(+), 20 deletions(-) create mode 100644 .gitlab/gradle-cache/ValidateGradleMetadata.java create mode 100755 .gitlab/validate_gradle_metadata.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 28a6c746b1b..414fa9a31b9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -254,9 +254,9 @@ default: # (dependencies-accessors, groovy-dsl, kotlin-dsl, transforms) whose metadata.bin is either # TRUNCATED (EOFException) or entirely MISSING (FileNotFoundException). Either way Gradle # hard-fails during configuration with "Could not read workspace metadata" and does NOT - # self-heal. A valid metadata.bin is ~110 bytes. Capture evidence (good + damaged) for - # diagnosis, then drop only the damaged workspace dirs so Gradle regenerates them (only the - # entries that would fail anyway are removed). See gradle/gradle#28974. + # self-heal. Capture evidence (good + damaged) for diagnosis, then use Gradle's own metadata + # reader to identify and drop only the damaged workspace dirs so Gradle regenerates them (only + # the entries that would fail anyway are removed). See gradle/gradle#28974. - | GRADLE_METADATA_EVIDENCE_DIR="gradle-cache-metadata" mkdir -p "$GRADLE_METADATA_EVIDENCE_DIR" @@ -264,30 +264,59 @@ default: find .gradle/caches -type f -name metadata.bin -printf '%10s %p\n' 2>/dev/null \ | sort -n > "$GRADLE_METADATA_EVIDENCE_DIR/metadata-manifest.txt" || true damaged="" + workspace_list="$GRADLE_METADATA_EVIDENCE_DIR/workspaces.nul" + : > "$workspace_list" + candidate_count=0 # kotlin-dsl nests workspaces one level deeper than the others: its metadata.bin lives at # kotlin-dsl/{accessors,scripts}//metadata.bin, so it needs an extra glob level. The # category dirs (kotlin-dsl/accessors/, kotlin-dsl/scripts/) never hold a metadata.bin and # would otherwise be false-flagged as damaged (size=0) on every run. - for ws in .gradle/caches/*/dependencies-accessors/*/ \ - .gradle/caches/*/groovy-dsl/*/ \ - .gradle/caches/*/kotlin-dsl/*/*/ \ - .gradle/caches/*/transforms/*/; do + metadata_cache_version="${GRADLE_VERSION:-*}" + uuid_suffix='[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}' + uuid_suffix="$uuid_suffix-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" + temporary_workspace_suffix="-$uuid_suffix$" + for ws in .gradle/caches/${metadata_cache_version}/dependencies-accessors/*/ \ + .gradle/caches/${metadata_cache_version}/groovy-dsl/*/ \ + .gradle/caches/${metadata_cache_version}/kotlin-dsl/*/*/ \ + .gradle/caches/${metadata_cache_version}/transforms/*/; do [ -d "$ws" ] || continue - meta="${ws}metadata.bin" - size=$(stat -c%s "$meta" 2>/dev/null || echo 0) - if [ ! -f "$meta" ] || [ "$size" -lt 32 ]; then - if [ -z "$damaged" ]; then - echo -e "${TEXT_BOLD}${TEXT_YELLOW}WARNING: damaged Gradle immutable-workspace metadata (truncated/missing metadata.bin from a partial cache extraction); preserving evidence and clearing so Gradle regenerates:${TEXT_CLEAR}" - damaged="yes" - fi - dest="$GRADLE_METADATA_EVIDENCE_DIR/corrupt/${ws}" - mkdir -p "$dest" || true - if [ -f "$meta" ]; then cp -p "$meta" "$dest/metadata.bin" 2>/dev/null || true; fi - ls -la "$ws" > "$dest/dir-listing.txt" 2>/dev/null || true - echo " - ${ws} (metadata.bin size=${size})" - rm -rf "$ws" || true + # Gradle temporary workspaces are named - and intentionally + # may not have metadata.bin until Gradle moves them into their immutable location. + ws_name="${ws%/}" + ws_name="${ws_name##*/}" + if [[ "$ws_name" =~ $temporary_workspace_suffix ]]; then + continue fi + printf '%s\0' "$ws" >> "$workspace_list" + candidate_count=$((candidate_count + 1)) done + if [ "$candidate_count" -gt 0 ]; then + validator_output="$GRADLE_METADATA_EVIDENCE_DIR/validator-output.txt" + validator_error="$GRADLE_METADATA_EVIDENCE_DIR/validator-error.txt" + validator_status=0 + .gitlab/validate_gradle_metadata.sh --workspace-list "$workspace_list" \ + > "$validator_output" 2> "$validator_error" || validator_status=$? + if [ "$validator_status" -eq 1 ]; then + while IFS=$'\t' read -r size reason ws; do + [ -n "$ws" ] || continue + [ -d "$ws" ] || continue + meta="${ws}metadata.bin" + if [ -z "$damaged" ]; then + echo -e "${TEXT_BOLD}${TEXT_YELLOW}WARNING: damaged Gradle immutable-workspace metadata (truncated/missing metadata.bin from a partial cache extraction); preserving evidence and clearing so Gradle regenerates:${TEXT_CLEAR}" + damaged="yes" + fi + dest="$GRADLE_METADATA_EVIDENCE_DIR/corrupt/${ws}" + mkdir -p "$dest" || true + if [ -f "$meta" ]; then cp -p "$meta" "$dest/metadata.bin" 2>/dev/null || true; fi + ls -la "$ws" > "$dest/dir-listing.txt" 2>/dev/null || true + echo " - ${ws} (metadata.bin size=${size}; ${reason})" + rm -rf "$ws" || true + done < "$validator_output" + elif [ "$validator_status" -ne 0 ]; then + echo -e "${TEXT_BOLD}${TEXT_YELLOW}WARNING: Gradle metadata validator was unavailable; leaving cache unchanged to avoid false positives.${TEXT_CLEAR}" + cat "$validator_error" || true + fi + fi if [ -z "$damaged" ]; then echo "No damaged Gradle immutable-workspace metadata detected."; fi # Keep a few intact metadata.bin files for byte-level comparison with the damaged ones. # `find | head` makes find die with SIGPIPE once head closes the pipe after 10 lines; under diff --git a/.gitlab/gradle-cache/ValidateGradleMetadata.java b/.gitlab/gradle-cache/ValidateGradleMetadata.java new file mode 100644 index 00000000000..b7dc8f6a8a4 --- /dev/null +++ b/.gitlab/gradle-cache/ValidateGradleMetadata.java @@ -0,0 +1,181 @@ +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.regex.Pattern; + +/** + * Validates Gradle immutable-workspace metadata files before CI decides whether to clear them. + */ +class ValidateGradleMetadata { + private static final int EXIT_VALID = 0; + // The shell wrapper maps this to 1 after the Java source launcher succeeds. + // This keeps launcher failures from looking like damaged metadata to the CI cleanup block. + private static final int EXIT_DAMAGED = 42; + private static final int EXIT_VALIDATOR_UNAVAILABLE = 2; + private static final Pattern TEMPORARY_WORKSPACE = + Pattern.compile( + ".*-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + + "[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"); + + public static void main(String[] args) { + try { + var workspaces = workspacesFrom(args); + var reader = new GradleMetadataReader(); + var damaged = false; + + for (var workspace : workspaces) { + var result = reader.validate(workspace); + if (result != null) { + damaged = true; + System.out.printf("%s\t%s\t%s%n", result.size(), result.reason(), result.workspace()); + } + } + + System.exit(damaged ? EXIT_DAMAGED : EXIT_VALID); + } catch (ValidatorUnavailableException e) { + System.err.println(e.getMessage()); + System.exit(EXIT_VALIDATOR_UNAVAILABLE); + } catch (Exception e) { + System.err.println("Gradle metadata validator failed: " + summarize(e)); + System.exit(EXIT_VALIDATOR_UNAVAILABLE); + } + } + + private static List workspacesFrom(String[] args) throws IOException { + if (args.length == 2 && "--workspace-list".equals(args[0])) { + return readNulSeparatedPaths(Path.of(args[1])); + } + if (args.length > 0 && args[0].startsWith("--")) { + throw new IllegalArgumentException( + "usage: ValidateGradleMetadata [--workspace-list file] [workspace...]"); + } + + var workspaces = new ArrayList(); + for (var arg : args) { + workspaces.add(Path.of(arg)); + } + return workspaces; + } + + private static List readNulSeparatedPaths(Path listFile) throws IOException { + var bytes = Files.readAllBytes(listFile); + var workspaces = new ArrayList(); + var start = 0; + for (var i = 0; i < bytes.length; i++) { + if (bytes[i] == 0) { + addPath(bytes, start, i, workspaces); + start = i + 1; + } + } + addPath(bytes, start, bytes.length, workspaces); + return workspaces; + } + + private static void addPath(byte[] bytes, int start, int end, List workspaces) { + if (end <= start) { + return; + } + var path = new String(bytes, start, end - start, StandardCharsets.UTF_8); + if (!path.isBlank()) { + workspaces.add(Path.of(path)); + } + } + + private record DamagedWorkspace(Path workspace, String size, String reason) { + } + + private static final class GradleMetadataReader { + private final Object store; + private final Method loadWorkspaceMetadata; + + private GradleMetadataReader() { + try { + var storeClass = + Class.forName( + "org.gradle.internal.execution.history.impl.DefaultImmutableWorkspaceMetadataStore"); + store = storeClass.getDeclaredConstructor().newInstance(); + loadWorkspaceMetadata = + storeClass.getMethod("loadWorkspaceMetadata", java.io.File.class); + } catch (ReflectiveOperationException | LinkageError e) { + throw new ValidatorUnavailableException( + "Gradle metadata validator could not load Gradle metadata reader: " + summarize(e), e); + } + } + + private DamagedWorkspace validate(Path workspace) { + if (TEMPORARY_WORKSPACE.matcher(workspace.getFileName().toString()).matches()) { + return null; + } + + var metadata = workspace.resolve("metadata.bin"); + if (!Files.isRegularFile(metadata)) { + return new DamagedWorkspace(workspace, "missing", "metadata.bin is missing"); + } + + String size; + try { + size = Long.toString(Files.size(metadata)); + } catch (IOException e) { + return new DamagedWorkspace(workspace, "unknown", "metadata.bin size check failed"); + } + + try { + var result = loadWorkspaceMetadata.invoke(store, workspace.toFile()); + if (result instanceof Optional optional && optional.isEmpty()) { + return new DamagedWorkspace(workspace, size, "Gradle returned empty metadata"); + } + if (result == null) { + return new DamagedWorkspace(workspace, size, "Gradle returned null metadata"); + } + return null; + } catch (InvocationTargetException e) { + var cause = e.getCause(); + if (cause instanceof LinkageError) { + throw new ValidatorUnavailableException( + "Gradle metadata validator could not execute Gradle metadata reader: " + + summarize(cause), + cause); + } + return new DamagedWorkspace(workspace, size, summarize(cause)); + } catch (ReflectiveOperationException | RuntimeException e) { + throw new ValidatorUnavailableException( + "Gradle metadata validator could not execute Gradle metadata reader: " + + summarize(e), + e); + } + } + } + + private static String summarize(Throwable throwable) { + if (throwable == null) { + return "unknown failure"; + } + + var root = throwable; + while ((root instanceof InvocationTargetException + || root instanceof UncheckedIOException + || root.getClass().getName().equals("org.gradle.internal.UncheckedException")) + && root.getCause() != null) { + root = root.getCause(); + } + + var message = root.getMessage(); + if (message == null || message.isBlank()) { + return root.getClass().getSimpleName(); + } + return root.getClass().getSimpleName() + ": " + message.replace('\t', ' '); + } + + private static final class ValidatorUnavailableException extends RuntimeException { + private ValidatorUnavailableException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/.gitlab/validate_gradle_metadata.sh b/.gitlab/validate_gradle_metadata.sh new file mode 100755 index 00000000000..2c19f9b06d6 --- /dev/null +++ b/.gitlab/validate_gradle_metadata.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -euo pipefail + +java_bin="${JAVA_25_HOME:-}" +if [[ -n "$java_bin" ]]; then + java_bin="$java_bin/bin/java" +else + java_bin="java" +fi + +if [[ ! -x "$java_bin" && "$java_bin" != "java" ]]; then + echo "Gradle metadata validator could not find Java executable: $java_bin" >&2 + exit 2 +fi + +script_dir="$(cd "$(dirname "$0")" && pwd)" +gradle_version="${GRADLE_VERSION:-}" +if [[ -z "$gradle_version" && -f gradle/wrapper/gradle-wrapper.properties ]]; then + gradle_version="$( + sed -nE 's#^distributionUrl=.*gradle-([0-9][^-]*)-(bin|all)\.zip$#\1#p' \ + gradle/wrapper/gradle-wrapper.properties \ + | head -n 1 + )" +fi + +if [[ -z "$gradle_version" ]]; then + echo "Gradle metadata validator could not determine Gradle version" >&2 + exit 2 +fi + +gradle_lib="" +for gradle_home in "${GRADLE_USER_HOME:-$PWD/.gradle}" "$HOME/.gradle"; do + [[ -d "$gradle_home/wrapper/dists" ]] || continue + gradle_lib="$( + find "$gradle_home/wrapper/dists" \ + -path "*/gradle-${gradle_version}/lib" \ + -type d \ + -print \ + -quit 2>/dev/null + )" + if [[ -n "$gradle_lib" ]]; then + break + fi +done + +if [[ -z "$gradle_lib" ]]; then + echo \ + "Gradle metadata validator could not find Gradle $gradle_version distribution lib directory" \ + >&2 + exit 2 +fi + +set +e +"$java_bin" --class-path "$gradle_lib/*" \ + "$script_dir/gradle-cache/ValidateGradleMetadata.java" "$@" +status=$? +set -e + +# The Java program uses 42 for damaged metadata so Java source-launcher failures, which commonly +# exit 1 before main() runs, are treated as validator-unavailable instead of cache corruption. +case "$status" in + 0) + exit 0 + ;; + 42) + exit 1 + ;; + 2) + exit 2 + ;; + *) + exit 2 + ;; +esac From 4256a435320ae2aa03f58554db6f12bec80c2878 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Sat, 30 May 2026 17:47:12 -0400 Subject: [PATCH 5/6] Improved Java code. --- .gitlab-ci.yml | 78 +++---- .../gradle-cache/ValidateGradleMetadata.java | 200 ++++++++---------- .gitlab/validate_gradle_metadata.sh | 43 ++-- 3 files changed, 140 insertions(+), 181 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 414fa9a31b9..3d3fe05f542 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -264,58 +264,32 @@ default: find .gradle/caches -type f -name metadata.bin -printf '%10s %p\n' 2>/dev/null \ | sort -n > "$GRADLE_METADATA_EVIDENCE_DIR/metadata-manifest.txt" || true damaged="" - workspace_list="$GRADLE_METADATA_EVIDENCE_DIR/workspaces.nul" - : > "$workspace_list" - candidate_count=0 - # kotlin-dsl nests workspaces one level deeper than the others: its metadata.bin lives at - # kotlin-dsl/{accessors,scripts}//metadata.bin, so it needs an extra glob level. The - # category dirs (kotlin-dsl/accessors/, kotlin-dsl/scripts/) never hold a metadata.bin and - # would otherwise be false-flagged as damaged (size=0) on every run. - metadata_cache_version="${GRADLE_VERSION:-*}" - uuid_suffix='[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}' - uuid_suffix="$uuid_suffix-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" - temporary_workspace_suffix="-$uuid_suffix$" - for ws in .gradle/caches/${metadata_cache_version}/dependencies-accessors/*/ \ - .gradle/caches/${metadata_cache_version}/groovy-dsl/*/ \ - .gradle/caches/${metadata_cache_version}/kotlin-dsl/*/*/ \ - .gradle/caches/${metadata_cache_version}/transforms/*/; do - [ -d "$ws" ] || continue - # Gradle temporary workspaces are named - and intentionally - # may not have metadata.bin until Gradle moves them into their immutable location. - ws_name="${ws%/}" - ws_name="${ws_name##*/}" - if [[ "$ws_name" =~ $temporary_workspace_suffix ]]; then - continue - fi - printf '%s\0' "$ws" >> "$workspace_list" - candidate_count=$((candidate_count + 1)) - done - if [ "$candidate_count" -gt 0 ]; then - validator_output="$GRADLE_METADATA_EVIDENCE_DIR/validator-output.txt" - validator_error="$GRADLE_METADATA_EVIDENCE_DIR/validator-error.txt" - validator_status=0 - .gitlab/validate_gradle_metadata.sh --workspace-list "$workspace_list" \ - > "$validator_output" 2> "$validator_error" || validator_status=$? - if [ "$validator_status" -eq 1 ]; then - while IFS=$'\t' read -r size reason ws; do - [ -n "$ws" ] || continue - [ -d "$ws" ] || continue - meta="${ws}metadata.bin" - if [ -z "$damaged" ]; then - echo -e "${TEXT_BOLD}${TEXT_YELLOW}WARNING: damaged Gradle immutable-workspace metadata (truncated/missing metadata.bin from a partial cache extraction); preserving evidence and clearing so Gradle regenerates:${TEXT_CLEAR}" - damaged="yes" - fi - dest="$GRADLE_METADATA_EVIDENCE_DIR/corrupt/${ws}" - mkdir -p "$dest" || true - if [ -f "$meta" ]; then cp -p "$meta" "$dest/metadata.bin" 2>/dev/null || true; fi - ls -la "$ws" > "$dest/dir-listing.txt" 2>/dev/null || true - echo " - ${ws} (metadata.bin size=${size}; ${reason})" - rm -rf "$ws" || true - done < "$validator_output" - elif [ "$validator_status" -ne 0 ]; then - echo -e "${TEXT_BOLD}${TEXT_YELLOW}WARNING: Gradle metadata validator was unavailable; leaving cache unchanged to avoid false positives.${TEXT_CLEAR}" - cat "$validator_error" || true - fi + # ValidateGradleMetadata enumerates the immutable-workspace dirs, skips Gradle temporary + # workspaces, and prints damaged ones as "\t\t" (exit 1 via wrapper). + validator_output="$GRADLE_METADATA_EVIDENCE_DIR/validator-output.txt" + validator_error="$GRADLE_METADATA_EVIDENCE_DIR/validator-error.txt" + validator_status=0 + .gitlab/validate_gradle_metadata.sh "$GRADLE_VERSION" \ + > "$validator_output" 2> "$validator_error" || validator_status=$? + if [ "$validator_status" -eq 1 ]; then + while IFS=$'\t' read -r size reason ws; do + [ -n "$ws" ] || continue + [ -d "$ws" ] || continue + meta="${ws}/metadata.bin" + if [ -z "$damaged" ]; then + echo -e "${TEXT_BOLD}${TEXT_YELLOW}WARNING: damaged Gradle immutable-workspace metadata (truncated/missing metadata.bin from a partial cache extraction); preserving evidence and clearing so Gradle regenerates:${TEXT_CLEAR}" + damaged="yes" + fi + dest="$GRADLE_METADATA_EVIDENCE_DIR/corrupt/${ws}" + mkdir -p "$dest" || true + if [ -f "$meta" ]; then cp -p "$meta" "$dest/metadata.bin" 2>/dev/null || true; fi + ls -la "$ws" > "$dest/dir-listing.txt" 2>/dev/null || true + echo " - ${ws} (metadata.bin size=${size}; ${reason})" + rm -rf "$ws" || true + done < "$validator_output" + elif [ "$validator_status" -ne 0 ]; then + echo -e "${TEXT_BOLD}${TEXT_YELLOW}WARNING: Gradle metadata validator was unavailable; leaving cache unchanged to avoid false positives.${TEXT_CLEAR}" + cat "$validator_error" || true fi if [ -z "$damaged" ]; then echo "No damaged Gradle immutable-workspace metadata detected."; fi # Keep a few intact metadata.bin files for byte-level comparison with the damaged ones. diff --git a/.gitlab/gradle-cache/ValidateGradleMetadata.java b/.gitlab/gradle-cache/ValidateGradleMetadata.java index b7dc8f6a8a4..0dd747bc1c4 100644 --- a/.gitlab/gradle-cache/ValidateGradleMetadata.java +++ b/.gitlab/gradle-cache/ValidateGradleMetadata.java @@ -2,157 +2,144 @@ import java.io.UncheckedIOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.Optional; +import java.util.Map; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Validates Gradle immutable-workspace metadata files before CI decides whether to clear them. + * + *

Given a Gradle version, enumerates the immutable-workspace dirs under the project-local cache + * and, for each, asks Gradle's own reader to deserialize its {@code metadata.bin}. Damaged + * (truncated or missing) entries are printed to stdout as {@code \t\t} and + * the process exits {@code 65} (EX_DATAERR). Any other failure means the validator could not run, + * so it exits {@code 2} and CI leaves the cache untouched rather than mistaking an unavailable + * validator for cache corruption. */ class ValidateGradleMetadata { private static final int EXIT_VALID = 0; - // The shell wrapper maps this to 1 after the Java source launcher succeeds. - // This keeps launcher failures from looking like damaged metadata to the CI cleanup block. - private static final int EXIT_DAMAGED = 42; - private static final int EXIT_VALIDATOR_UNAVAILABLE = 2; + private static final int EXIT_DAMAGED = 65; + private static final int EXIT_UNAVAILABLE = 2; + + // CI always runs from the repository root, where the project-local Gradle cache lives. + private static final Path CACHES_DIR = Path.of(".gradle/caches"); + + // Folders with the directory depth to check. + private static final Map WORKSPACE_CATEGORIES = + Map.of( + "dependencies-accessors", 1, + "groovy-dsl", 1, + "kotlin-dsl", 2, + "transforms", 1); + private static final Pattern TEMPORARY_WORKSPACE = - Pattern.compile( - ".*-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" - + "[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"); + Pattern.compile(".*-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"); + + // Gradle's own metadata reader, resolved reflectively. + private static Object metadataStore; + private static Method metadataLoadMethod; public static void main(String[] args) { try { - var workspaces = workspacesFrom(args); - var reader = new GradleMetadataReader(); - var damaged = false; + loadMetadataReader(); - for (var workspace : workspaces) { - var result = reader.validate(workspace); + var damaged = false; + for (var workspace : enumerateWorkspaces(args[0])) { + var result = validate(workspace); if (result != null) { damaged = true; System.out.printf("%s\t%s\t%s%n", result.size(), result.reason(), result.workspace()); } } - System.exit(damaged ? EXIT_DAMAGED : EXIT_VALID); - } catch (ValidatorUnavailableException e) { - System.err.println(e.getMessage()); - System.exit(EXIT_VALIDATOR_UNAVAILABLE); } catch (Exception e) { - System.err.println("Gradle metadata validator failed: " + summarize(e)); - System.exit(EXIT_VALIDATOR_UNAVAILABLE); + System.err.println("Gradle metadata validator unavailable: " + summarize(e)); + System.exit(EXIT_UNAVAILABLE); } } - private static List workspacesFrom(String[] args) throws IOException { - if (args.length == 2 && "--workspace-list".equals(args[0])) { - return readNulSeparatedPaths(Path.of(args[1])); - } - if (args.length > 0 && args[0].startsWith("--")) { - throw new IllegalArgumentException( - "usage: ValidateGradleMetadata [--workspace-list file] [workspace...]"); - } - + private static List enumerateWorkspaces(String gradleVersion) throws IOException { + var versionDir = CACHES_DIR.resolve(gradleVersion); var workspaces = new ArrayList(); - for (var arg : args) { - workspaces.add(Path.of(arg)); + for (var category : WORKSPACE_CATEGORIES.entrySet()) { + collectAtDepth(versionDir.resolve(category.getKey()), category.getValue(), workspaces); } return workspaces; } - private static List readNulSeparatedPaths(Path listFile) throws IOException { - var bytes = Files.readAllBytes(listFile); - var workspaces = new ArrayList(); - var start = 0; - for (var i = 0; i < bytes.length; i++) { - if (bytes[i] == 0) { - addPath(bytes, start, i, workspaces); - start = i + 1; - } + private static void collectAtDepth(Path dir, int depth, List out) throws IOException { + if (!Files.isDirectory(dir)) { + return; } - addPath(bytes, start, bytes.length, workspaces); - return workspaces; - } - - private static void addPath(byte[] bytes, int start, int end, List workspaces) { - if (end <= start) { + if (depth == 0) { + out.add(dir); return; } - var path = new String(bytes, start, end - start, StandardCharsets.UTF_8); - if (!path.isBlank()) { - workspaces.add(Path.of(path)); + for (var child : childDirectories(dir)) { + collectAtDepth(child, depth - 1, out); } } - private record DamagedWorkspace(Path workspace, String size, String reason) { + private static List childDirectories(Path dir) throws IOException { + try (var entries = Files.list(dir)) { + return entries.filter(Files::isDirectory).collect(Collectors.toList()); + } } - private static final class GradleMetadataReader { - private final Object store; - private final Method loadWorkspaceMetadata; - - private GradleMetadataReader() { - try { - var storeClass = - Class.forName( - "org.gradle.internal.execution.history.impl.DefaultImmutableWorkspaceMetadataStore"); - store = storeClass.getDeclaredConstructor().newInstance(); - loadWorkspaceMetadata = - storeClass.getMethod("loadWorkspaceMetadata", java.io.File.class); - } catch (ReflectiveOperationException | LinkageError e) { - throw new ValidatorUnavailableException( - "Gradle metadata validator could not load Gradle metadata reader: " + summarize(e), e); - } + private static void loadMetadataReader() { + try { + var storeClass = + Class.forName("org.gradle.internal.execution.history.impl.DefaultImmutableWorkspaceMetadataStore"); + metadataStore = storeClass.getDeclaredConstructor().newInstance(); + metadataLoadMethod = storeClass.getMethod("loadWorkspaceMetadata", java.io.File.class); + } catch (ReflectiveOperationException | LinkageError e) { + throw new IllegalStateException("could not load Gradle metadata reader: " + summarize(e)); } + } - private DamagedWorkspace validate(Path workspace) { - if (TEMPORARY_WORKSPACE.matcher(workspace.getFileName().toString()).matches()) { - return null; - } + private static DamagedWorkspace validate(Path workspace) { + // Gradle temporary workspaces are named - and intentionally may not + // have metadata.bin until Gradle moves them into their immutable location. + if (TEMPORARY_WORKSPACE.matcher(workspace.getFileName().toString()).matches()) { + return null; + } - var metadata = workspace.resolve("metadata.bin"); - if (!Files.isRegularFile(metadata)) { - return new DamagedWorkspace(workspace, "missing", "metadata.bin is missing"); - } + var metadata = workspace.resolve("metadata.bin"); + if (!Files.isRegularFile(metadata)) { + return new DamagedWorkspace(workspace, "missing", "metadata.bin is missing"); + } - String size; - try { - size = Long.toString(Files.size(metadata)); - } catch (IOException e) { - return new DamagedWorkspace(workspace, "unknown", "metadata.bin size check failed"); - } + String size; + try { + size = Long.toString(Files.size(metadata)); + } catch (IOException e) { + return new DamagedWorkspace(workspace, "unknown", "metadata.bin size check failed"); + } - try { - var result = loadWorkspaceMetadata.invoke(store, workspace.toFile()); - if (result instanceof Optional optional && optional.isEmpty()) { - return new DamagedWorkspace(workspace, size, "Gradle returned empty metadata"); - } - if (result == null) { - return new DamagedWorkspace(workspace, size, "Gradle returned null metadata"); - } - return null; - } catch (InvocationTargetException e) { - var cause = e.getCause(); - if (cause instanceof LinkageError) { - throw new ValidatorUnavailableException( - "Gradle metadata validator could not execute Gradle metadata reader: " - + summarize(cause), - cause); - } - return new DamagedWorkspace(workspace, size, summarize(cause)); - } catch (ReflectiveOperationException | RuntimeException e) { - throw new ValidatorUnavailableException( - "Gradle metadata validator could not execute Gradle metadata reader: " - + summarize(e), - e); + // A successful return means Gradle's own reader fully deserialized metadata.bin; + // a truncated file throws mid-read and surfaces here as an InvocationTargetException. + try { + metadataLoadMethod.invoke(metadataStore, workspace.toFile()); + return null; + } catch (InvocationTargetException e) { + var cause = e.getCause(); + if (cause instanceof LinkageError) { + throw new IllegalStateException( + "could not execute Gradle metadata reader: " + summarize(cause)); } + return new DamagedWorkspace(workspace, size, summarize(cause)); + } catch (ReflectiveOperationException | RuntimeException e) { + throw new IllegalStateException("could not execute Gradle metadata reader: " + summarize(e)); } } + private record DamagedWorkspace(Path workspace, String size, String reason) { + } + private static String summarize(Throwable throwable) { if (throwable == null) { return "unknown failure"; @@ -170,12 +157,7 @@ private static String summarize(Throwable throwable) { if (message == null || message.isBlank()) { return root.getClass().getSimpleName(); } - return root.getClass().getSimpleName() + ": " + message.replace('\t', ' '); - } - private static final class ValidatorUnavailableException extends RuntimeException { - private ValidatorUnavailableException(String message, Throwable cause) { - super(message, cause); - } + return root.getClass().getSimpleName() + ": " + message.replace('\t', ' '); } } diff --git a/.gitlab/validate_gradle_metadata.sh b/.gitlab/validate_gradle_metadata.sh index 2c19f9b06d6..91950ab901c 100755 --- a/.gitlab/validate_gradle_metadata.sh +++ b/.gitlab/validate_gradle_metadata.sh @@ -2,14 +2,16 @@ set -euo pipefail -java_bin="${JAVA_25_HOME:-}" -if [[ -n "$java_bin" ]]; then - java_bin="$java_bin/bin/java" +java_home="${JAVA_25_HOME:-}" +if [[ -n "$java_home" ]]; then + java_bin="$java_home/bin/java" + javac_bin="$java_home/bin/javac" else java_bin="java" + javac_bin="javac" fi -if [[ ! -x "$java_bin" && "$java_bin" != "java" ]]; then +if [[ "$java_bin" != "java" && ! -x "$java_bin" ]]; then echo "Gradle metadata validator could not find Java executable: $java_bin" >&2 exit 2 fi @@ -51,25 +53,26 @@ if [[ -z "$gradle_lib" ]]; then exit 2 fi +# Pre-compile with -proc:none rather than using the `java .java` source launcher: the launcher +# would otherwise discover and run annotation processors bundled in the Gradle jars on the classpath. +build_dir="$(mktemp -d)" +trap 'rm -rf "$build_dir"' EXIT + +if ! "$javac_bin" -proc:none -classpath "$gradle_lib/*" -d "$build_dir" \ + "$script_dir/gradle-cache/ValidateGradleMetadata.java"; then + echo "Gradle metadata validator could not compile ValidateGradleMetadata" >&2 + exit 2 +fi + set +e -"$java_bin" --class-path "$gradle_lib/*" \ - "$script_dir/gradle-cache/ValidateGradleMetadata.java" "$@" +"$java_bin" -classpath "$build_dir:$gradle_lib/*" ValidateGradleMetadata "$@" status=$? set -e -# The Java program uses 42 for damaged metadata so Java source-launcher failures, which commonly -# exit 1 before main() runs, are treated as validator-unavailable instead of cache corruption. +# ValidateGradleMetadata exits 65 (EX_DATAERR) for damaged metadata so that a JVM that exits 1 +# before main() runs is treated as validator-unavailable instead of as cache corruption. case "$status" in - 0) - exit 0 - ;; - 42) - exit 1 - ;; - 2) - exit 2 - ;; - *) - exit 2 - ;; + 0) exit 0 ;; + 65) exit 1 ;; + *) exit 2 ;; esac From b95da02f4ef8bf75ad33a3c842788c311816a286 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Sat, 30 May 2026 19:12:15 -0400 Subject: [PATCH 6/6] Polished log messages to simplify log search. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3d3fe05f542..7494a628a87 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -277,7 +277,7 @@ default: [ -d "$ws" ] || continue meta="${ws}/metadata.bin" if [ -z "$damaged" ]; then - echo -e "${TEXT_BOLD}${TEXT_YELLOW}WARNING: damaged Gradle immutable-workspace metadata (truncated/missing metadata.bin from a partial cache extraction); preserving evidence and clearing so Gradle regenerates:${TEXT_CLEAR}" + echo -e "${TEXT_BOLD}${TEXT_YELLOW}[WARNING] Damaged Gradle metadata found, fixed:${TEXT_CLEAR}" damaged="yes" fi dest="$GRADLE_METADATA_EVIDENCE_DIR/corrupt/${ws}" @@ -288,7 +288,7 @@ default: rm -rf "$ws" || true done < "$validator_output" elif [ "$validator_status" -ne 0 ]; then - echo -e "${TEXT_BOLD}${TEXT_YELLOW}WARNING: Gradle metadata validator was unavailable; leaving cache unchanged to avoid false positives.${TEXT_CLEAR}" + echo -e "${TEXT_BOLD}${TEXT_YELLOW}[WARNING] Gradle metadata validator unavailable; leaving cache unchanged.${TEXT_CLEAR}" cat "$validator_error" || true fi if [ -z "$damaged" ]; then echo "No damaged Gradle immutable-workspace metadata detected."; fi