From 4fcaaac62b4bf51e0acf2fa047e6427b02bb07b8 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sun, 12 Jul 2026 15:52:32 -0400 Subject: [PATCH 1/5] ci: shard frontier_amd case-opt 3-way and trim to run-tested cases The frontier_amd case-optimization pre-build timed out at the ~2h batch walltime: AMD flang case-opt sim builds run ~30 min/case, and the 2-way shard put 4 cases (~120 min) on the heavy shard. The pre-build also globbed all 7 benchmarks/*/case.py while the run only tests 5, wasting two builds. Make the 5 run-tested cases a single source of truth (new case-optimization-benchmarks.sh, sourced by both scripts) so pre-built binaries and run cases never drift, and shard both the pre-build and the run 3-way (heavy shard ~2 cases, ~60 min). Unsharded clusters are unaffected (shard helper is inert when job_shard is empty). --- .../scripts/case-optimization-benchmarks.sh | 38 +++++++++++++++ .github/scripts/prebuild-case-optimization.sh | 45 +++++------------- .github/scripts/run_case_optimization.sh | 18 ++++--- .github/workflows/test.yml | 47 ++++++++++++------- 4 files changed, 90 insertions(+), 58 deletions(-) create mode 100644 .github/scripts/case-optimization-benchmarks.sh diff --git a/.github/scripts/case-optimization-benchmarks.sh b/.github/scripts/case-optimization-benchmarks.sh new file mode 100644 index 0000000000..fb8f019c8a --- /dev/null +++ b/.github/scripts/case-optimization-benchmarks.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Single source of truth for the benchmark cases exercised by the +# case-optimization CI. The pre-build (compiles the case-optimized binaries) +# and the run (executes them) MUST iterate the same list in the same order: +# sharding partitions this list by index, so any drift silently pre-builds or +# runs the wrong cases. Sourced by prebuild-case-optimization.sh and +# run_case_optimization.sh. +benchmarks=( + benchmarks/5eq_rk3_weno3_hllc/case.py + benchmarks/viscous_weno5_sgb_acoustic/case.py + benchmarks/hypo_hll/case.py + benchmarks/ibm/case.py + benchmarks/igr/case.py +) + +# Parse an optional "$job_shard" of the form "i/N" (e.g. "2/3"). On success +# sets caseopt_shard_idx / caseopt_shard_count (both 1 when unset — a single +# shard covering every case). Aborts on a malformed value. +caseopt_parse_shard() { + caseopt_shard_idx=1 + caseopt_shard_count=1 + [ -z "${job_shard:-}" ] && return 0 + caseopt_shard_idx="${job_shard%%/*}" + caseopt_shard_count="${job_shard##*/}" + case "$caseopt_shard_idx" in ''|*[!0-9]*|0*) echo "ERROR: bad shard '$job_shard' (expected i/N)"; exit 1 ;; esac + case "$caseopt_shard_count" in ''|*[!0-9]*|0*) echo "ERROR: bad shard '$job_shard' (expected i/N)"; exit 1 ;; esac + if [ "$job_shard" != "$caseopt_shard_idx/$caseopt_shard_count" ] \ + || [ "$caseopt_shard_idx" -lt 1 ] || [ "$caseopt_shard_idx" -gt "$caseopt_shard_count" ]; then + echo "ERROR: bad shard '$job_shard' (expected i/N with 1 <= i <= N)"; exit 1 + fi +} + +# 0 (true) if the 1-based case index $1 belongs to this shard. Shard i owns +# every Nth case: indices i, i+N, i+2N, ... +caseopt_case_in_shard() { + [ $((($1 - 1) % caseopt_shard_count)) -eq $((caseopt_shard_idx - 1)) ] +} diff --git a/.github/scripts/prebuild-case-optimization.sh b/.github/scripts/prebuild-case-optimization.sh index 7fb9af0041..b0053af28f 100755 --- a/.github/scripts/prebuild-case-optimization.sh +++ b/.github/scripts/prebuild-case-optimization.sh @@ -22,34 +22,16 @@ case "$cluster" in *) echo "ERROR: Unknown cluster '$cluster'"; exit 1 ;; esac -# Optional sharding (format "i/N", e.g. "1/2"), set by submit-slurm-job.sh's -# [shard] argument via $job_shard: shard i builds every Nth case of the sorted -# case list. Unset = build all cases in one job (default; other clusters). +# Benchmark list + optional sharding ("i/N", e.g. "1/3", set by +# submit-slurm-job.sh's [shard] argument via $job_shard): shard i builds every +# Nth case of the shared list. Unset = build all cases in one job (default; +# other clusters). The list is the single source of truth shared with +# run_case_optimization.sh so pre-built binaries and run cases never drift. +source .github/scripts/case-optimization-benchmarks.sh +caseopt_parse_shard shard="${job_shard:-}" -if [ -n "$shard" ]; then - # Validate full shape: must be exactly "digits/digits" — one slash with - # non-empty, purely numeric, non-leading-zero parts on both sides. - # Split first, then validate each part independently so that inputs like - # "1/" "/2" "//" "1/2/3" "a/b" "12" are all caught before any arithmetic. - shard_idx="${shard%%/*}" - shard_count="${shard##*/}" - # Reject if no slash (idx and count are equal and equal to the whole string) - case "$shard_idx" in - ''|*[!0-9]*|0*) echo "ERROR: bad shard '$shard' (expected i/N)"; exit 1 ;; - esac - case "$shard_count" in - ''|*[!0-9]*|0*) echo "ERROR: bad shard '$shard' (expected i/N)"; exit 1 ;; - esac - # Confirm the string is exactly "idx/count" — catches "12" (no slash) and - # "1/2/3" (extra slash, where idx=1 and count=2/3 would have failed above, - # but this is an extra safety net). - if [ "$shard" != "$shard_idx/$shard_count" ]; then - echo "ERROR: bad shard '$shard' (expected i/N)"; exit 1 - fi - if [ "$shard_idx" -lt 1 ] || [ "$shard_idx" -gt "$shard_count" ]; then - echo "ERROR: bad shard '$shard' (expected i/N with 1 <= i <= N)"; exit 1 - fi -fi +shard_idx="$caseopt_shard_idx" +shard_count="$caseopt_shard_count" # Phoenix starts fresh (no prior dep build); other clusters pre-build deps via # build.sh first, so we must preserve them and only clean MFC target staging. @@ -80,8 +62,7 @@ esac if [ -n "$shard" ] && [ "$shard_count" -gt 1 ]; then shared_marker_done="build/.prebuild-shared-targets-done" shared_marker_failed="build/.prebuild-shared-targets-failed" - set -- benchmarks/*/case.py - first_case="$1" + first_case="${benchmarks[0]}" if [ "$shard_idx" -eq 1 ]; then # Remove both markers at the start so reruns and manual invocations # never observe stale state from a prior run. @@ -110,11 +91,9 @@ if [ -n "$shard" ] && [ "$shard_count" -gt 1 ]; then fi idx=0 -for case in benchmarks/*/case.py; do +for case in "${benchmarks[@]}"; do idx=$((idx + 1)) - if [ -n "$shard" ] && [ $(((idx - 1) % shard_count)) -ne $((shard_idx - 1)) ]; then - continue - fi + caseopt_case_in_shard "$idx" || continue echo "=== Pre-building: $case ===" ./mfc.sh run "$case" --case-optimization $gpu_opts -j 8 --dry-run done diff --git a/.github/scripts/run_case_optimization.sh b/.github/scripts/run_case_optimization.sh index e0d2f797b6..57bf42b4cf 100755 --- a/.github/scripts/run_case_optimization.sh +++ b/.github/scripts/run_case_optimization.sh @@ -13,13 +13,11 @@ if [ "$job_device" = "gpu" ] && [ "$ngpus" -eq 0 ]; then ngpus=1 fi -benchmarks=( - benchmarks/5eq_rk3_weno3_hllc/case.py - benchmarks/viscous_weno5_sgb_acoustic/case.py - benchmarks/hypo_hll/case.py - benchmarks/ibm/case.py - benchmarks/igr/case.py -) +# Benchmark list (single source of truth shared with the pre-build) + optional +# sharding: a sharded run executes only the subset it has pre-built binaries +# for. $job_shard mirrors the pre-build shard so the two stay aligned. +source .github/scripts/case-optimization-benchmarks.sh +caseopt_parse_shard # For Frontier/Frontier AMD: deps were fetched on the login node via --deps-only; # build case-optimized binaries here on the compute node before running. @@ -39,7 +37,10 @@ if [ "$job_cluster" != "phoenix" ] && [ "$job_cluster" != "frontier_amd" ]; then find build/install -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true echo "=== Building case-optimized binaries on compute node ===" + idx=0 for case in "${benchmarks[@]}"; do + idx=$((idx + 1)) + caseopt_case_in_shard "$idx" || continue echo "--- Building: $case ---" ./mfc.sh build -i "$case" --case-optimization $gpu_opts -j 8 done @@ -50,7 +51,10 @@ passed=0 failed=0 failed_cases="" +idx=0 for case in "${benchmarks[@]}"; do + idx=$((idx + 1)) + caseopt_case_in_shard "$idx" || continue case_dir="$(dirname "$case")" case_name="$(basename "$case_dir")" echo "" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cb97242f4f..c5b7737808 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -530,21 +530,21 @@ jobs: - name: Pre-Build (SLURM) if: matrix.cluster == 'frontier_amd' - # AMD flang is slow enough that one serial pre-build job exceeds its - # walltime, so split the case list across two concurrent SLURM jobs. - # The shards share this workspace and skip their in-job staging clean, - # so clean once here on the login node before submitting. + # AMD flang case-opt builds are ~30 min/case, so one serial pre-build + # job exceeds its walltime. Split the case list across three concurrent + # SLURM jobs (heavy shard ~2 cases). The shards share this workspace and + # skip their in-job staging clean, so clean once here first. run: | find build/staging -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true find build/install -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true - rm -f build/.prebuild-shared-targets-done - bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh gpu ${{ matrix.interface }} ${{ matrix.cluster }} 1/2 & - pid1=$! - bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh gpu ${{ matrix.interface }} ${{ matrix.cluster }} 2/2 & - pid2=$! + rm -f build/.prebuild-shared-targets-done build/.prebuild-shared-targets-failed + pids="" + for s in 1/3 2/3 3/3; do + bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh gpu ${{ matrix.interface }} ${{ matrix.cluster }} "$s" & + pids="$pids $!" + done rc=0 - wait "$pid1" || rc=1 - wait "$pid2" || rc=1 + for p in $pids; do wait "$p" || rc=1; done exit $rc - name: Build & Run Case-Optimization Tests @@ -552,9 +552,23 @@ jobs: run: bash .github/scripts/submit-slurm-job.sh .github/scripts/run_case_optimization.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} - name: Run Case-Optimization Tests - if: matrix.cluster == 'phoenix' || matrix.cluster == 'frontier_amd' + if: matrix.cluster == 'phoenix' run: bash .github/scripts/submit-slurm-job.sh .github/scripts/run_case_optimization.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} + - name: Run Case-Optimization Tests + if: matrix.cluster == 'frontier_amd' + # Mirror the pre-build sharding so each run job executes only the cases + # it has binaries for, concurrently within the batch walltime. + run: | + pids="" + for s in 1/3 2/3 3/3; do + bash .github/scripts/submit-slurm-job.sh .github/scripts/run_case_optimization.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} "$s" & + pids="$pids $!" + done + rc=0 + for p in $pids; do wait "$p" || rc=1; done + exit $rc + - name: Cancel SLURM Jobs if: cancelled() run: | @@ -567,10 +581,7 @@ jobs: - name: Print Logs if: always() run: | - for f in prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out \ - prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}-1-of-2.out \ - prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}-2-of-2.out \ - run-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out; do + for f in prebuild-case-optimization-*.out run-case-optimization-*.out; do [ -f "$f" ] && echo "=== $f ===" && cat "$f" done @@ -580,5 +591,5 @@ jobs: with: name: case-opt-${{ strategy.job-index }}-${{ matrix.cluster }}-${{ matrix.interface }} path: | - prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}*.out - run-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out + prebuild-case-optimization-*.out + run-case-optimization-*.out From d36d8add1bf78d887c9ece2385bc3fdb30ad02e5 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sun, 12 Jul 2026 16:04:22 -0400 Subject: [PATCH 2/5] ci: label the Frontier CCE test lanes '(CCE)' to match what they run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The non-AMD Frontier lanes in test.yml run the CCE (Cray) compiler (cpe/25.03, default PrgEnv-cray) — the matrix comment says so — but displayed as bare 'Oak Ridge | Frontier' while their AMD sibling is tagged '(AMD)' and bench.yml already names the same lane 'Oak Ridge | Frontier (CCE)'. Tag them '(CCE)' so the check name matches the compiler and is consistent across workflows. Display-only; the CPU lane genuinely uses CCE too. --- .github/workflows/test.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c5b7737808..507d8eaf65 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -357,31 +357,31 @@ jobs: # Frontier (ORNL) — CCE - runner: 'frontier' cluster: 'frontier' - cluster_name: 'Oak Ridge | Frontier' + cluster_name: 'Oak Ridge | Frontier (CCE)' device: 'gpu' interface: 'acc' shard: '1/2' - runner: 'frontier' cluster: 'frontier' - cluster_name: 'Oak Ridge | Frontier' + cluster_name: 'Oak Ridge | Frontier (CCE)' device: 'gpu' interface: 'acc' shard: '2/2' - runner: 'frontier' cluster: 'frontier' - cluster_name: 'Oak Ridge | Frontier' + cluster_name: 'Oak Ridge | Frontier (CCE)' device: 'gpu' interface: 'omp' shard: '1/2' - runner: 'frontier' cluster: 'frontier' - cluster_name: 'Oak Ridge | Frontier' + cluster_name: 'Oak Ridge | Frontier (CCE)' device: 'gpu' interface: 'omp' shard: '2/2' - runner: 'frontier' cluster: 'frontier' - cluster_name: 'Oak Ridge | Frontier' + cluster_name: 'Oak Ridge | Frontier (CCE)' device: 'cpu' interface: 'none' # Frontier AMD — build on login node, GPU tests sharded for batch partition @@ -495,12 +495,12 @@ jobs: interface: 'omp' - runner: 'frontier' cluster: 'frontier' - cluster_name: 'Oak Ridge | Frontier' + cluster_name: 'Oak Ridge | Frontier (CCE)' device: 'gpu' interface: 'acc' - runner: 'frontier' cluster: 'frontier' - cluster_name: 'Oak Ridge | Frontier' + cluster_name: 'Oak Ridge | Frontier (CCE)' device: 'gpu' interface: 'omp' - runner: 'frontier' From 81fcad999c891b8e2df6397f10082c07733d80b6 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sun, 12 Jul 2026 16:11:53 -0400 Subject: [PATCH 3/5] ci: tag Phoenix lanes by actual compiler (GPU NVHPC, CPU GNU) Phoenix is split by device: p-gpu loads nvhpc/24.5 (nvfortran) while p-cpu loads gcc/12.3.0 (toolchain/modules). The lanes were displayed as bare 'Georgia Tech | Phoenix', and bench.yml's Phoenix CPU benchmark was tagged '(NVHPC)' though it actually builds with GNU. Tag each lane by the compiler it runs -- GPU '(NVHPC)', CPU '(GNU)' -- matching the Frontier (CCE)/(AMD) convention and fixing the bench.yml CPU mislabel. Display-only. --- .github/workflows/bench.yml | 2 +- .github/workflows/test.yml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 652ab6bef3..e8dd047f53 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -38,7 +38,7 @@ jobs: matrix: include: - cluster: phoenix - name: Georgia Tech | Phoenix (NVHPC) + name: Georgia Tech | Phoenix (GNU) group: phoenix labels: gt flag: p diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 507d8eaf65..8ec2ef040f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -341,17 +341,17 @@ jobs: # Phoenix (GT) — build+test combined in SLURM job - runner: 'gt' cluster: 'phoenix' - cluster_name: 'Georgia Tech | Phoenix' + cluster_name: 'Georgia Tech | Phoenix (NVHPC)' device: 'gpu' interface: 'acc' - runner: 'gt' cluster: 'phoenix' - cluster_name: 'Georgia Tech | Phoenix' + cluster_name: 'Georgia Tech | Phoenix (NVHPC)' device: 'gpu' interface: 'omp' - runner: 'gt' cluster: 'phoenix' - cluster_name: 'Georgia Tech | Phoenix' + cluster_name: 'Georgia Tech | Phoenix (GNU)' device: 'cpu' interface: 'none' # Frontier (ORNL) — CCE @@ -485,12 +485,12 @@ jobs: include: - runner: 'gt' cluster: 'phoenix' - cluster_name: 'Georgia Tech | Phoenix' + cluster_name: 'Georgia Tech | Phoenix (NVHPC)' device: 'gpu' interface: 'acc' - runner: 'gt' cluster: 'phoenix' - cluster_name: 'Georgia Tech | Phoenix' + cluster_name: 'Georgia Tech | Phoenix (NVHPC)' device: 'gpu' interface: 'omp' - runner: 'frontier' From 04b06882f916eef1db356bc4f69baf7d7b3b711b Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sun, 12 Jul 2026 16:48:39 -0400 Subject: [PATCH 4/5] ci: fix dead case-opt staging clean (regex never matched slug names) The stale-staging cleanup used -regex '.*/[0-9a-f]+', which matches only pure-hex names, but MFC staging slugs are '-' with prefix in {gpu-acc, gpu-mp, cpu} (+ optional -debug/-chem/... ), e.g. 'gpu-mp-b86d2efce8' (build.py get_slug). So the clean matched nothing and removed no staging at all. Consequence on self-hosted runners: a prior case-opt build that was killed mid-configure (e.g. the frontier_amd pre-build hitting its walltime) leaves a corrupt staging dir that the clean fails to remove; the next run reuses it and CMake configure fails deterministically ('CMAKE_Fortran_COMPILER not set' in FindOpenMP try_compile, + 'Directory not empty' on the leftover CMakeFiles). Observed across all 3 sbatch retries of run 29198726737. Match '-*' instead so target staging/install dirs are removed while dependency dirs (fftw, hipfort, hdf5, silo, lapack) are preserved. Verified the new regex removes every prefix variant and keeps the deps. All 6 sites (prebuild, run, workflow x staging+install). --- .github/scripts/prebuild-case-optimization.sh | 4 ++-- .github/scripts/run_case_optimization.sh | 4 ++-- .github/workflows/test.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/scripts/prebuild-case-optimization.sh b/.github/scripts/prebuild-case-optimization.sh index b0053af28f..c31058a08a 100755 --- a/.github/scripts/prebuild-case-optimization.sh +++ b/.github/scripts/prebuild-case-optimization.sh @@ -42,8 +42,8 @@ if [ "$cluster" = "phoenix" ]; then source .github/scripts/clean-build.sh clean_build elif [ -z "$shard" ]; then - find build/staging -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true - find build/install -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true + find build/staging -maxdepth 1 -regex '.*/\(gpu-acc\|gpu-mp\|cpu\)-.*' -type d -exec rm -rf {} + 2>/dev/null || true + find build/install -maxdepth 1 -regex '.*/\(gpu-acc\|gpu-mp\|cpu\)-.*' -type d -exec rm -rf {} + 2>/dev/null || true fi . ./mfc.sh load -c "$flag" -m g diff --git a/.github/scripts/run_case_optimization.sh b/.github/scripts/run_case_optimization.sh index 57bf42b4cf..2391f82339 100755 --- a/.github/scripts/run_case_optimization.sh +++ b/.github/scripts/run_case_optimization.sh @@ -33,8 +33,8 @@ if [ "$job_cluster" != "phoenix" ] && [ "$job_cluster" != "frontier_amd" ]; then # preserve dependency dirs (hipfort, fftw, etc.) since the compute # node has no internet to re-fetch them. echo "=== Cleaning stale MFC target staging/install ===" - find build/staging -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true - find build/install -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true + find build/staging -maxdepth 1 -regex '.*/\(gpu-acc\|gpu-mp\|cpu\)-.*' -type d -exec rm -rf {} + 2>/dev/null || true + find build/install -maxdepth 1 -regex '.*/\(gpu-acc\|gpu-mp\|cpu\)-.*' -type d -exec rm -rf {} + 2>/dev/null || true echo "=== Building case-optimized binaries on compute node ===" idx=0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8ec2ef040f..9536645cf2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -535,8 +535,8 @@ jobs: # SLURM jobs (heavy shard ~2 cases). The shards share this workspace and # skip their in-job staging clean, so clean once here first. run: | - find build/staging -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true - find build/install -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true + find build/staging -maxdepth 1 -regex '.*/\(gpu-acc\|gpu-mp\|cpu\)-.*' -type d -exec rm -rf {} + 2>/dev/null || true + find build/install -maxdepth 1 -regex '.*/\(gpu-acc\|gpu-mp\|cpu\)-.*' -type d -exec rm -rf {} + 2>/dev/null || true rm -f build/.prebuild-shared-targets-done build/.prebuild-shared-targets-failed pids="" for s in 1/3 2/3 3/3; do From b958ca4b6e0f4fecfe904d5934cef282dd29ea04 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sun, 12 Jul 2026 18:23:25 -0400 Subject: [PATCH 5/5] ci: stream SLURM output via backgrounded tail, not timed read over procsub The monitor streamed job output with 'read -t 1' over a 'tail -f' process substitution (fd 3). When that pipe broke, bash could crash in the read builtin's alarm/longjmp unwind path (SIGSEGV 139, 'unwind_frame_run: read_builtin: frame not found'); the EXIT trap then scancel'd a still-RUNNING, healthy job, producing a red X that mimics a real test failure (observed on a Phoenix lane at test 341/596, all golden-matching). Replace it with a plain backgrounded 'tail -f' streaming straight to the step log, plus the existing status poll loop (state was already polled separately, so nothing is lost). The final 'cat "$output_file"' still reprints the whole file, so output is complete even if tail is killed mid-flush. Removes the crash-prone construct entirely; behavior-preserving otherwise. --- .github/scripts/monitor_slurm_job.sh | 61 +++++++++------------------- 1 file changed, 19 insertions(+), 42 deletions(-) diff --git a/.github/scripts/monitor_slurm_job.sh b/.github/scripts/monitor_slurm_job.sh index 1f70a18d35..5ddc27d8d7 100755 --- a/.github/scripts/monitor_slurm_job.sh +++ b/.github/scripts/monitor_slurm_job.sh @@ -122,62 +122,39 @@ done echo "=== Streaming output for job $job_id ===" -# Start tail and redirect its output to file descriptor 3 for multiplexing -# This allows us to stream tail output while also printing heartbeat messages -exec 3< <(stdbuf -oL -eL tail -f "$output_file" 2>&1) +# Stream the job's output to the step log with a plain backgrounded `tail`. +# This previously used a timed read (`read -t 1`) over a `tail -f` process +# substitution; when that pipe broke, bash could crash in the read-builtin's +# alarm/longjmp unwind path (SIGSEGV), and the EXIT trap would then cancel a +# still-healthy job. A backgrounded tail avoids that construct entirely, and +# the final `cat` below reprints the whole file so nothing is lost if tail is +# killed mid-flush. +stdbuf -oL -eL tail -f "$output_file" 2>&1 & tail_pid=$! -# Monitor job status and stream output simultaneously +# Poll job status until it reaches a terminal state; streaming happens +# independently in the background tail above. last_heartbeat=$(date +%s) - while true; do - # Try to read from tail output (non-blocking via timeout) - # Read multiple lines if available to avoid falling behind - lines_read=0 - while IFS= read -r -t 1 line <&3 2>/dev/null; do - echo "$line" - lines_read=$((lines_read + 1)) - last_heartbeat=$(date +%s) - # Limit burst reads to avoid starving the status check - if [ $lines_read -ge 100 ]; then - break - fi - done - - # Check job status - current_time=$(date +%s) state=$(get_job_state "$job_id") if is_terminal_state "$state"; then echo "[$(date +%H:%M:%S)] Job $job_id reached terminal state: $state" break - else - # Print heartbeat if no output for 60 seconds - if [ $((current_time - last_heartbeat)) -ge 60 ]; then - echo "[$(date +%H:%M:%S)] Job $job_id state=$state (no new output for 60s)..." - last_heartbeat=$current_time - fi fi - # Sleep briefly between status checks - sleep 1 -done - -# Drain any remaining output from tail after job completes -echo "Draining remaining output..." -drain_count=0 -while IFS= read -r -t 1 line <&3 2>/dev/null; do - echo "$line" - drain_count=$((drain_count + 1)) - # Safety limit to avoid infinite loop - if [ $drain_count -ge 10000 ]; then - echo "Warning: Truncating remaining output after 10000 lines" - break + # Periodic heartbeat so the CI log never looks stalled during quiet phases. + current_time=$(date +%s) + if [ $((current_time - last_heartbeat)) -ge 60 ]; then + echo "[$(date +%H:%M:%S)] Job $job_id state=$state..." + last_heartbeat=$current_time fi + + sleep 5 done -# Close the file descriptor and kill tail -exec 3<&- +# Give tail a moment to flush the final lines, then stop streaming. +sleep 2 kill "${tail_pid}" 2>/dev/null || true tail_pid=""