diff --git a/.github/workflows/daily-regen.yml b/.github/workflows/daily-regen.yml index 4c628a73af..d8673e48fa 100644 --- a/.github/workflows/daily-regen.yml +++ b/.github/workflows/daily-regen.yml @@ -4,10 +4,19 @@ run-name: "Scheduled regen (${{ github.event.inputs.specification_id || github.e # Picks the N oldest specs (by most-recent implementation `updated` timestamp) # and re-dispatches `bulk-generate.yml` for each. Default N=1 per cron tick. # -# Schedule: hourly, skipping the 20:00–23:59 Berlin (CEST) evening window. -# Berlin CEST evening hours 20, 21, 22, 23 map to UTC 18, 19, 20, 21 — these -# are intentionally skipped so runs never start during the user's evening. -# All other UTC hours run, giving 20 ticks per day. +# Schedule: every 2 hours, skipping the fixed 18:00–21:59 UTC window so runs +# never start during the user's evening. The window was sized for Berlin +# summer time (CEST, UTC+2 → 20:00–23:59 local); Actions cron is always UTC, +# so under winter time (CET, UTC+1) it covers 19:00–22:59 local instead — +# accepted drift, not a bug. That gives 10 ticks per day. +# +# The tick fires at minute 17 (not :00) on purpose: GitHub's scheduler is +# most overloaded at the top of the hour and delays/drops `schedule` events +# then (github.com docs on `on.schedule`). This repo has seen multi-day +# streaks of silently dropped ticks for exactly this workflow while +# less-frequent crons kept firing. As a second line of defence, +# watchdog-stuck-jobs.yml re-dispatches this workflow when main has seen +# no new run for >10 h outside the quiet window. # # bulk-generate is serialised via its own concurrency group. Default model is # Sonnet for higher implementation quality; can be overridden per manual run. @@ -15,10 +24,11 @@ run-name: "Scheduled regen (${{ github.event.inputs.specification_id || github.e # Triggers: # - schedule: 10× daily (UTC, every 2 hours, skipping the 18–21 UTC window) → Sonnet # - workflow_dispatch: manual, with inputs for spec id, model, count, dry-run +# (also used by the watchdog's cron-liveness rescue, with all defaults) on: schedule: - - cron: '0 0,2,4,6,8,10,12,14,16,22 * * *' + - cron: '17 0,2,4,6,8,10,12,14,16,22 * * *' workflow_dispatch: inputs: specification_id: diff --git a/.github/workflows/impl-generate.yml b/.github/workflows/impl-generate.yml index 40d4598da3..2a4a493173 100644 --- a/.github/workflows/impl-generate.yml +++ b/.github/workflows/impl-generate.yml @@ -422,7 +422,7 @@ jobs: id: claude continue-on-error: true timeout-minutes: 60 - uses: anthropics/claude-code-action@1298632ce7736903d02a1435002705aa2a594a6c # v1 + uses: anthropics/claude-code-action@11ba60486e4aec9ddfeafcf4bb3f00b028ac2c16 # v1 (VALIDATION: July-1 pin, revert after) with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} claude_args: "--model ${{ steps.inputs.outputs.model }}" @@ -442,7 +442,7 @@ jobs: if: steps.claude.outcome == 'failure' id: claude_retry timeout-minutes: 60 - uses: anthropics/claude-code-action@1298632ce7736903d02a1435002705aa2a594a6c # v1 + uses: anthropics/claude-code-action@11ba60486e4aec9ddfeafcf4bb3f00b028ac2c16 # v1 (VALIDATION: July-1 pin, revert after) with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} claude_args: "--model ${{ steps.inputs.outputs.model }}" @@ -458,6 +458,43 @@ jobs: - SPEC_ID: ${{ steps.inputs.outputs.specification_id }} - IS_REGENERATION: ${{ steps.existing.outputs.is_regeneration }} + # ======================================================================== + # VALIDATION TEST — inspect what Claude produced, then STOP before any + # side-effecting steps (PR / GCS / issue comment). Remove after test. + # ======================================================================== + - name: TEST — inspect Claude output and stop (validation run) + if: always() + env: + SPEC_ID: ${{ steps.inputs.outputs.specification_id }} + LANGUAGE: ${{ steps.inputs.outputs.language }} + LIBRARY: ${{ steps.inputs.outputs.library }} + EXT: ${{ steps.inputs.outputs.ext }} + run: | + echo "===== VALIDATION TEST (claude-code-action July-1 pin) =====" + IMPL_DIR="plots/${SPEC_ID}/implementations/${LANGUAGE}" + echo "--- git log (last 3) ---"; git log --oneline -3 || true + echo "--- git status --porcelain ---"; git status --porcelain || true + echo "--- ls IMPL_DIR ---"; ls -la "$IMPL_DIR" || true + for f in "${LIBRARY}${EXT}" plot-light.png plot-dark.png; do + if [ -f "$IMPL_DIR/$f" ]; then echo "PRESENT: $f"; else echo "MISSING: $f"; fi + done + echo "--- claude-execution-output.json summary ---" + python3 - <<'PY' || true + import json, glob + for p in glob.glob('/home/runner/work/_temp/claude-execution-output.json'): + try: + d = json.load(open(p)) + if isinstance(d, list): + d = d[-1] + print('is_error', d.get('is_error'), + 'num_turns', d.get('num_turns'), + 'permission_denials_count', d.get('permission_denials_count')) + except Exception as e: + print('parse err', e) + PY + echo "===== END VALIDATION TEST — stopping before side effects =====" + exit 1 + # ======================================================================== # Create metadata file (before PR) # ======================================================================== @@ -1006,7 +1043,7 @@ jobs: # Failure handling: Track failures via comments and auto-retry # ======================================================================== - name: Handle generation failure - if: failure() && steps.issue.outputs.number != '' + if: false # VALIDATION: disabled during test so it can't comment/retry on #1010 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SPEC_ID: ${{ steps.inputs.outputs.specification_id }} diff --git a/.github/workflows/watchdog-stuck-jobs.yml b/.github/workflows/watchdog-stuck-jobs.yml index 7d4b622b1a..70549b68c3 100644 --- a/.github/workflows/watchdog-stuck-jobs.yml +++ b/.github/workflows/watchdog-stuck-jobs.yml @@ -1,7 +1,7 @@ name: "Watchdog: Stuck Jobs" run-name: "Watchdog: scan ${{ github.event_name == 'workflow_dispatch' && '(manual)' || '(cron)' }}" -# Periodic safety net for the impl pipeline. Catches three failure modes +# Periodic safety net for the impl pipeline. Catches four failure modes # the regular workflows miss: # 1. PRs labeled `ai-review-failed` without `ai-review-rescued` # (impl-review-retry.yml normally handles these; watchdog covers @@ -10,6 +10,8 @@ run-name: "Watchdog: scan ${{ github.event_name == 'workflow_dispatch' && '(manu # (review handed off to repair, repair crashed, nothing else fired). # 3. spec-ready issues with `generate:` or `impl::failed` and # no open PR for that (spec, lib) pair after the staleness window. +# 4. daily-regen's cron silently starved by GitHub's scheduler (no run +# for >10 h outside the quiet window) — re-dispatched manually. # # Retries are bounded per-cause via marker labels: # - `ai-review-rescued` (review failures) @@ -295,4 +297,46 @@ jobs: done done < <(echo "$ISS_JSON" | jq -c '.[]') + ##### C) Cron-liveness: rescue a starved daily-regen schedule ###### + # GitHub silently drops `schedule` ticks for daily-regen (the + # repo's most frequent cron) during scheduler overload — observed + # as multi-day gaps while this watchdog and bot-serving-check kept + # firing. daily-regen ticks are ≤6 h apart (2 h cadence plus the + # 18–21 UTC quiet window), so a >10 h silence means the schedule + # is starved; re-dispatch it manually with default inputs. + # Hours 17–21 UTC are skipped so the rescue never launches a + # pipeline into the fixed 18–21 UTC quiet window the cron itself + # deliberately avoids (17 included because bulk-generate starts + # immediately after dispatch and would spill into it). Like the + # cron, the window is UTC-fixed: it matches Berlin evening under + # CEST and sits an hour earlier in local terms under CET. + LIVENESS_HOURS=10 + HOUR=$(date -u +%-H) + if (( HOUR >= 17 && HOUR <= 21 )); then + echo "::notice::daily-regen liveness: inside/adjacent to quiet window (UTC hour $HOUR) — check skipped" + else + # --branch main: schedule runs (and rescue dispatches) live on the + # default branch; a manual run on a feature branch must not mask a + # starved main schedule. createdAt (not startedAt): always set, + # even while the newest run is still queued. + LAST=$(gh run list --workflow daily-regen.yml --branch main --limit 1 \ + --json createdAt --jq '.[0].createdAt // empty') + if [[ -z "$LAST" ]]; then + echo "::warning::daily-regen liveness: no runs found on main — skipping rescue" + else + last_sec=$(date -u -d "$LAST" +%s) + # Fresh timestamp: NOW from the top of the script is stale by + # however long the A/B scans took. + now_c=$(date -u +%s) + gap=$(( now_c - last_sec )) + gap_hm=$(printf '%dh%02dm' $(( gap / 3600 )) $(( (gap % 3600) / 60 ))) + if (( gap > LIVENESS_HOURS * 3600 )); then + echo "::warning::daily-regen: no new run on main for ${gap_hm} (> ${LIVENESS_HOURS}h) — schedule starved, re-dispatching" + dispatch "daily-regen: cron starved (${gap_hm} silent)" daily-regen.yml + else + echo "daily-regen liveness: newest run on main created ${gap_hm} ago — healthy" + fi + fi + fi + echo "::notice::Watchdog scan complete" diff --git a/CHANGELOG.md b/CHANGELOG.md index 3491012050..e7f6f249a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -129,6 +129,15 @@ aggregate instead: an italic *Catalog* line at the end of the version section an ### Fixed +- **Scheduled implementation regeneration no longer silently starves for days** — GitHub's scheduler + had been dropping `daily-regen.yml`'s cron ticks in multi-day streaks since early June + (last 13 days: only 5 scheduled runs instead of ~130) while less-frequent crons in the same + repo kept firing; every run that did start was green, so nothing alarmed. The cron now fires + at :17 instead of :00 (GitHub documents top-of-hour scheduler overload as a delay/drop + cause), and `watchdog-stuck-jobs.yml` gained a cron-liveness rescue that re-dispatches + daily-regen whenever main has seen no new run created for >10 h outside the Berlin-evening + quiet window, capping any future starvation at typically ~half a day (worst case just under + a day, when the >10 h mark lands in the skipped 17–21 UTC checks) instead of weeks (#9649). - **Tag search uses the GIN index and stops treating `%`/`_` as wildcards** — `SpecRepository.search_by_tags` cast the JSONB `tags` column to text and ran LIKE, which the `ix_specs_tags` GIN index can never serve (sequential scan on every MCP tag search) and which diff --git a/docs/workflows/overview.md b/docs/workflows/overview.md index d42f3ba920..3e9414b551 100644 --- a/docs/workflows/overview.md +++ b/docs/workflows/overview.md @@ -167,7 +167,8 @@ Located in `.github/workflows/`: | `impl-repair.yml` | Fixes rejected implementations | | `impl-merge.yml` | Merges approved PRs | | `bulk-generate.yml` | Batch implementation generation | -| `daily-regen.yml` | Cron-driven regeneration of the oldest implementations | +| `daily-regen.yml` | Cron-driven regeneration of the oldest implementations (10×/day at :17 past UTC hours 0–16 even + 22, dodging GitHub's top-of-hour scheduler overload and the 18–21 UTC quiet window) | +| `watchdog-stuck-jobs.yml` | 6-hourly safety net: re-dispatches stuck reviews/repairs/merges/generations and rescues daily-regen when its cron is silently starved by GitHub (>10 h without a run) | | `report-validate.yml` | Validates user-submitted issue reports | | `sync-postgres.yml` | Syncs `plots/` filesystem state to PostgreSQL on push to main | | `sync-labels.yml` | Auto-syncs spec/impl labels after manual PR merges |