diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb50fc15a..7bb07c4b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -662,6 +662,11 @@ jobs: [ -n "$path" ] && saw_invalid_path=true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*Failed to convert config\.cachix to JSON' && saw_cachix_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*while evaluating the option.*cachix\.package' && saw_cachix_signature=true || true + # Nix can surface fetched-source corruption as missing subpaths under + # \u00ABgithub:owner/repo/rev\u00BB/... during arbitrary flake evaluation, not only + # while resolving the devenv CLI. Retry after clearing fetch/eval caches: + # this indicates an incomplete or stale fetched-source view, not project source. + printf '%s' "$flattened" | grep -Eq "error:[[:space:]]*path '\u00AB(github|https?)[:/][^\u00BB]+\u00BB/[^']+' does not exist" && saw_fetch_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*cannot read file from tarball:[[:space:]]*Truncated tar archive detected while reading data' && saw_fetch_signature=true || true rm -f "$log" @@ -682,7 +687,7 @@ jobs: fi [ -z "$path" ] || nix-store --realise "$path" 2>/dev/null || true - rm -rf ~/.cache/nix/eval-cache-* + rm -rf ~/.cache/nix/eval-cache-* ~/.cache/nix/gitv3 ~/.cache/nix/tarball-cache ~/.cache/nix/tarball-cache-v2 ~/.cache/nix/fetcher-cache*.sqlite* attempt=$((attempt + 1)) done @@ -787,6 +792,11 @@ jobs: [ -n "$path" ] && saw_invalid_path=true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*Failed to convert config\.cachix to JSON' && saw_cachix_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*while evaluating the option.*cachix\.package' && saw_cachix_signature=true || true + # Nix can surface fetched-source corruption as missing subpaths under + # \u00ABgithub:owner/repo/rev\u00BB/... during arbitrary flake evaluation, not only + # while resolving the devenv CLI. Retry after clearing fetch/eval caches: + # this indicates an incomplete or stale fetched-source view, not project source. + printf '%s' "$flattened" | grep -Eq "error:[[:space:]]*path '\u00AB(github|https?)[:/][^\u00BB]+\u00BB/[^']+' does not exist" && saw_fetch_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*cannot read file from tarball:[[:space:]]*Truncated tar archive detected while reading data' && saw_fetch_signature=true || true rm -f "$log" @@ -807,7 +817,7 @@ jobs: fi [ -z "$path" ] || nix-store --realise "$path" 2>/dev/null || true - rm -rf ~/.cache/nix/eval-cache-* + rm -rf ~/.cache/nix/eval-cache-* ~/.cache/nix/gitv3 ~/.cache/nix/tarball-cache ~/.cache/nix/tarball-cache-v2 ~/.cache/nix/fetcher-cache*.sqlite* attempt=$((attempt + 1)) done @@ -1143,6 +1153,11 @@ jobs: [ -n "$path" ] && saw_invalid_path=true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*Failed to convert config\.cachix to JSON' && saw_cachix_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*while evaluating the option.*cachix\.package' && saw_cachix_signature=true || true + # Nix can surface fetched-source corruption as missing subpaths under + # \u00ABgithub:owner/repo/rev\u00BB/... during arbitrary flake evaluation, not only + # while resolving the devenv CLI. Retry after clearing fetch/eval caches: + # this indicates an incomplete or stale fetched-source view, not project source. + printf '%s' "$flattened" | grep -Eq "error:[[:space:]]*path '\u00AB(github|https?)[:/][^\u00BB]+\u00BB/[^']+' does not exist" && saw_fetch_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*cannot read file from tarball:[[:space:]]*Truncated tar archive detected while reading data' && saw_fetch_signature=true || true rm -f "$log" @@ -1163,7 +1178,7 @@ jobs: fi [ -z "$path" ] || nix-store --realise "$path" 2>/dev/null || true - rm -rf ~/.cache/nix/eval-cache-* + rm -rf ~/.cache/nix/eval-cache-* ~/.cache/nix/gitv3 ~/.cache/nix/tarball-cache ~/.cache/nix/tarball-cache-v2 ~/.cache/nix/fetcher-cache*.sqlite* attempt=$((attempt + 1)) done @@ -1502,6 +1517,11 @@ jobs: [ -n "$path" ] && saw_invalid_path=true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*Failed to convert config\.cachix to JSON' && saw_cachix_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*while evaluating the option.*cachix\.package' && saw_cachix_signature=true || true + # Nix can surface fetched-source corruption as missing subpaths under + # \u00ABgithub:owner/repo/rev\u00BB/... during arbitrary flake evaluation, not only + # while resolving the devenv CLI. Retry after clearing fetch/eval caches: + # this indicates an incomplete or stale fetched-source view, not project source. + printf '%s' "$flattened" | grep -Eq "error:[[:space:]]*path '\u00AB(github|https?)[:/][^\u00BB]+\u00BB/[^']+' does not exist" && saw_fetch_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*cannot read file from tarball:[[:space:]]*Truncated tar archive detected while reading data' && saw_fetch_signature=true || true rm -f "$log" @@ -1522,7 +1542,7 @@ jobs: fi [ -z "$path" ] || nix-store --realise "$path" 2>/dev/null || true - rm -rf ~/.cache/nix/eval-cache-* + rm -rf ~/.cache/nix/eval-cache-* ~/.cache/nix/gitv3 ~/.cache/nix/tarball-cache ~/.cache/nix/tarball-cache-v2 ~/.cache/nix/fetcher-cache*.sqlite* attempt=$((attempt + 1)) done @@ -1861,6 +1881,11 @@ jobs: [ -n "$path" ] && saw_invalid_path=true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*Failed to convert config\.cachix to JSON' && saw_cachix_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*while evaluating the option.*cachix\.package' && saw_cachix_signature=true || true + # Nix can surface fetched-source corruption as missing subpaths under + # \u00ABgithub:owner/repo/rev\u00BB/... during arbitrary flake evaluation, not only + # while resolving the devenv CLI. Retry after clearing fetch/eval caches: + # this indicates an incomplete or stale fetched-source view, not project source. + printf '%s' "$flattened" | grep -Eq "error:[[:space:]]*path '\u00AB(github|https?)[:/][^\u00BB]+\u00BB/[^']+' does not exist" && saw_fetch_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*cannot read file from tarball:[[:space:]]*Truncated tar archive detected while reading data' && saw_fetch_signature=true || true rm -f "$log" @@ -1881,7 +1906,7 @@ jobs: fi [ -z "$path" ] || nix-store --realise "$path" 2>/dev/null || true - rm -rf ~/.cache/nix/eval-cache-* + rm -rf ~/.cache/nix/eval-cache-* ~/.cache/nix/gitv3 ~/.cache/nix/tarball-cache ~/.cache/nix/tarball-cache-v2 ~/.cache/nix/fetcher-cache*.sqlite* attempt=$((attempt + 1)) done @@ -2137,6 +2162,11 @@ jobs: [ -n "$path" ] && saw_invalid_path=true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*Failed to convert config\.cachix to JSON' && saw_cachix_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*while evaluating the option.*cachix\.package' && saw_cachix_signature=true || true + # Nix can surface fetched-source corruption as missing subpaths under + # \u00ABgithub:owner/repo/rev\u00BB/... during arbitrary flake evaluation, not only + # while resolving the devenv CLI. Retry after clearing fetch/eval caches: + # this indicates an incomplete or stale fetched-source view, not project source. + printf '%s' "$flattened" | grep -Eq "error:[[:space:]]*path '\u00AB(github|https?)[:/][^\u00BB]+\u00BB/[^']+' does not exist" && saw_fetch_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*cannot read file from tarball:[[:space:]]*Truncated tar archive detected while reading data' && saw_fetch_signature=true || true rm -f "$log" @@ -2157,7 +2187,7 @@ jobs: fi [ -z "$path" ] || nix-store --realise "$path" 2>/dev/null || true - rm -rf ~/.cache/nix/eval-cache-* + rm -rf ~/.cache/nix/eval-cache-* ~/.cache/nix/gitv3 ~/.cache/nix/tarball-cache ~/.cache/nix/tarball-cache-v2 ~/.cache/nix/fetcher-cache*.sqlite* attempt=$((attempt + 1)) done @@ -7330,6 +7360,11 @@ jobs: [ -n "$path" ] && saw_invalid_path=true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*Failed to convert config\.cachix to JSON' && saw_cachix_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*while evaluating the option.*cachix\.package' && saw_cachix_signature=true || true + # Nix can surface fetched-source corruption as missing subpaths under + # \u00ABgithub:owner/repo/rev\u00BB/... during arbitrary flake evaluation, not only + # while resolving the devenv CLI. Retry after clearing fetch/eval caches: + # this indicates an incomplete or stale fetched-source view, not project source. + printf '%s' "$flattened" | grep -Eq "error:[[:space:]]*path '\u00AB(github|https?)[:/][^\u00BB]+\u00BB/[^']+' does not exist" && saw_fetch_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*cannot read file from tarball:[[:space:]]*Truncated tar archive detected while reading data' && saw_fetch_signature=true || true rm -f "$log" @@ -7350,7 +7385,7 @@ jobs: fi [ -z "$path" ] || nix-store --realise "$path" 2>/dev/null || true - rm -rf ~/.cache/nix/eval-cache-* + rm -rf ~/.cache/nix/eval-cache-* ~/.cache/nix/gitv3 ~/.cache/nix/tarball-cache ~/.cache/nix/tarball-cache-v2 ~/.cache/nix/fetcher-cache*.sqlite* attempt=$((attempt + 1)) done @@ -7699,6 +7734,11 @@ jobs: [ -n "$path" ] && saw_invalid_path=true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*Failed to convert config\.cachix to JSON' && saw_cachix_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*while evaluating the option.*cachix\.package' && saw_cachix_signature=true || true + # Nix can surface fetched-source corruption as missing subpaths under + # \u00ABgithub:owner/repo/rev\u00BB/... during arbitrary flake evaluation, not only + # while resolving the devenv CLI. Retry after clearing fetch/eval caches: + # this indicates an incomplete or stale fetched-source view, not project source. + printf '%s' "$flattened" | grep -Eq "error:[[:space:]]*path '\u00AB(github|https?)[:/][^\u00BB]+\u00BB/[^']+' does not exist" && saw_fetch_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*cannot read file from tarball:[[:space:]]*Truncated tar archive detected while reading data' && saw_fetch_signature=true || true rm -f "$log" @@ -7719,7 +7759,7 @@ jobs: fi [ -z "$path" ] || nix-store --realise "$path" 2>/dev/null || true - rm -rf ~/.cache/nix/eval-cache-* + rm -rf ~/.cache/nix/eval-cache-* ~/.cache/nix/gitv3 ~/.cache/nix/tarball-cache ~/.cache/nix/tarball-cache-v2 ~/.cache/nix/fetcher-cache*.sqlite* attempt=$((attempt + 1)) done @@ -7812,6 +7852,11 @@ jobs: [ -n "$path" ] && saw_invalid_path=true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*Failed to convert config\.cachix to JSON' && saw_cachix_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*while evaluating the option.*cachix\.package' && saw_cachix_signature=true || true + # Nix can surface fetched-source corruption as missing subpaths under + # \u00ABgithub:owner/repo/rev\u00BB/... during arbitrary flake evaluation, not only + # while resolving the devenv CLI. Retry after clearing fetch/eval caches: + # this indicates an incomplete or stale fetched-source view, not project source. + printf '%s' "$flattened" | grep -Eq "error:[[:space:]]*path '\u00AB(github|https?)[:/][^\u00BB]+\u00BB/[^']+' does not exist" && saw_fetch_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*cannot read file from tarball:[[:space:]]*Truncated tar archive detected while reading data' && saw_fetch_signature=true || true rm -f "$log" @@ -7832,7 +7877,7 @@ jobs: fi [ -z "$path" ] || nix-store --realise "$path" 2>/dev/null || true - rm -rf ~/.cache/nix/eval-cache-* + rm -rf ~/.cache/nix/eval-cache-* ~/.cache/nix/gitv3 ~/.cache/nix/tarball-cache ~/.cache/nix/tarball-cache-v2 ~/.cache/nix/fetcher-cache*.sqlite* attempt=$((attempt + 1)) done diff --git a/genie/ci-scripts/nix-gc-race-retry.sh b/genie/ci-scripts/nix-gc-race-retry.sh index e7d3d056f..663d9b93a 100644 --- a/genie/ci-scripts/nix-gc-race-retry.sh +++ b/genie/ci-scripts/nix-gc-race-retry.sh @@ -74,6 +74,11 @@ run_nix_gc_race_retry() { [ -n "$path" ] && saw_invalid_path=true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*Failed to convert config\.cachix to JSON' && saw_cachix_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*while evaluating the option.*cachix\.package' && saw_cachix_signature=true || true + # Nix can surface fetched-source corruption as missing subpaths under + # «github:owner/repo/rev»/... during arbitrary flake evaluation, not only + # while resolving the devenv CLI. Retry after clearing fetch/eval caches: + # this indicates an incomplete or stale fetched-source view, not project source. + printf '%s' "$flattened" | grep -Eq "error:[[:space:]]*path '«(github|https?)[:/][^»]+»/[^']+' does not exist" && saw_fetch_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*cannot read file from tarball:[[:space:]]*Truncated tar archive detected while reading data' && saw_fetch_signature=true || true rm -f "$log" @@ -94,7 +99,7 @@ run_nix_gc_race_retry() { fi [ -z "$path" ] || nix-store --realise "$path" 2>/dev/null || true - rm -rf ~/.cache/nix/eval-cache-* + rm -rf ~/.cache/nix/eval-cache-* ~/.cache/nix/gitv3 ~/.cache/nix/tarball-cache ~/.cache/nix/tarball-cache-v2 ~/.cache/nix/fetcher-cache*.sqlite* attempt=$((attempt + 1)) done diff --git a/genie/ci-scripts/nix-gc-race-retry.test.sh b/genie/ci-scripts/nix-gc-race-retry.test.sh index 300589dc0..1ec5f2820 100644 --- a/genie/ci-scripts/nix-gc-race-retry.test.sh +++ b/genie/ci-scripts/nix-gc-race-retry.test.sh @@ -101,7 +101,28 @@ chmod +x "$fetch_fixture" CI_PROGRESS_HEARTBEAT_SECONDS=1 NIX_GC_RACE_MAX_RETRIES=2 run_nix_gc_race_retry "fetch-fixture" "$fetch_fixture" >/dev/null assert_eq "2" "$(cat "$test_dir/fetch-attempt")" "truncated tarball retry count" -echo "Test 4: does not retry when literal signature strings appear outside Nix error context" +echo "Test 4: retries missing pretty flake source subpath failures" +pretty_source_fixture="$test_dir/pretty-source-fixture.sh" +cat > "$pretty_source_fixture" < "\$attempt_file" + echo "error: path '«github:cachix/devenv/ea3d94a»/xtask' does not exist" >&2 + exit 1 +fi +echo "pretty source recovered" +EOF +chmod +x "$pretty_source_fixture" +CI_PROGRESS_HEARTBEAT_SECONDS=1 NIX_GC_RACE_MAX_RETRIES=2 run_nix_gc_race_retry "pretty-source-fixture" "$pretty_source_fixture" >/dev/null +assert_eq "2" "$(cat "$test_dir/pretty-source-attempt")" "pretty flake source retry count" + +echo "Test 5: does not retry when literal signature strings appear outside Nix error context" false_positive_fixture="$test_dir/false-positive-fixture.sh" cat > "$false_positive_fixture" <<'EOF' #!/usr/bin/env bash @@ -117,7 +138,7 @@ exit_code=$? set -e assert_exit_code 9 "$exit_code" "non-error-context strings do not trigger retries" -echo "Test 5: preserves the original exit code when no retry signature is present" +echo "Test 6: preserves the original exit code when no retry signature is present" non_retry_fixture="$test_dir/non-retry-fixture.sh" cat > "$non_retry_fixture" <<'EOF' #!/usr/bin/env bash diff --git a/genie/ci-workflow.ts b/genie/ci-workflow.ts index d99db8a92..3757bbc2f 100644 --- a/genie/ci-workflow.ts +++ b/genie/ci-workflow.ts @@ -88,6 +88,7 @@ export { nixExtraConf, runDevenvTasksBefore, standardCIEnv, + withGcRaceRetry, workspaceLocalNixCachePath, workspaceLocalNixCacheRoot, type NixBinaryCache, diff --git a/genie/ci-workflow/shared.ts b/genie/ci-workflow/shared.ts index 79a4f7ae0..33f9e2b3d 100644 --- a/genie/ci-workflow/shared.ts +++ b/genie/ci-workflow/shared.ts @@ -402,6 +402,11 @@ run_nix_gc_race_retry() { [ -n "$path" ] && saw_invalid_path=true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*Failed to convert config\.cachix to JSON' && saw_cachix_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*while evaluating the option.*cachix\.package' && saw_cachix_signature=true || true + # Nix can surface fetched-source corruption as missing subpaths under + # «github:owner/repo/rev»/... during arbitrary flake evaluation, not only + # while resolving the devenv CLI. Retry after clearing fetch/eval caches: + # this indicates an incomplete or stale fetched-source view, not project source. + printf '%s' "$flattened" | grep -Eq "error:[[:space:]]*path '«(github|https?)[:/][^»]+»/[^']+' does not exist" && saw_fetch_signature=true || true printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*cannot read file from tarball:[[:space:]]*Truncated tar archive detected while reading data' && saw_fetch_signature=true || true rm -f "$log" @@ -422,7 +427,7 @@ run_nix_gc_race_retry() { fi [ -z "$path" ] || nix-store --realise "$path" 2>/dev/null || true - rm -rf ~/.cache/nix/eval-cache-* + rm -rf ~/.cache/nix/eval-cache-* ~/.cache/nix/gitv3 ~/.cache/nix/tarball-cache ~/.cache/nix/tarball-cache-v2 ~/.cache/nix/fetcher-cache*.sqlite* attempt=$((attempt + 1)) done