diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 749ac42dd..6c045c2fc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,6 +27,17 @@ jobs: persist-credentials: false fetch-depth: 0 + - name: ⚙️ Setup Go + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version-file: go.mod + cache: false + + - name: 🩹 Validate merge-group heal contract + run: | + go test ./scripts/validate-merge-group-heal + go run ./scripts/validate-merge-group-heal .github/workflows/ci.yaml + - name: 🔍 Filter paths uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 id: filter @@ -334,24 +345,28 @@ jobs: wg-server-private-key: ${{ secrets.WG_SERVER_PRIVATE_KEY }} heal-prod-on-failure: - name: 🩹 Heal Prod (restore main on merge-group failure) + name: 🩹 Heal Prod (restore main after unsuccessful merge-group deploy) needs: [changes, deploy-prod] # The deploy above runs on the SPECULATIVE merge ref, before the PR lands on # `main`. When it fails the merge group is ejected — but the OCI `latest` tag # (and the prod cluster reconciling it) is left pointing at that ejected # artifact, which never lands on `main`, so prod silently diverges from Git # until some future merge overwrites it (devantler-tech/platform#2026). This - # failure-path-only job re-deploys the current tip of `main` via the SAME - # composite, so the ejected artifact is immediately replaced by main's signed - # artifact and prod reconverges with Git. It has ZERO effect on a green merge - # (skipped) and does NOT relax the gate: deploy-prod's failure still fails - # `CI - Required Checks` below, so the PR is still ejected. It is intentionally - # NOT part of that aggregation — it is best-effort cleanup, not a merge gate. + # unsuccessful-path-only job re-deploys the current tip of `main` via the + # SAME composite, so the speculative artifact is immediately replaced by + # main's signed artifact and prod reconverges with Git. It covers explicit + # cancellation too: GitHub re-evaluates job conditions during normal + # cancellation, and `always()` keeps this cleanup eligible after the deploy + # job becomes cancelled. It has ZERO effect on a green merge (skipped) and + # does NOT relax the gate: deploy-prod's failure still fails `CI - Required + # Checks` below, so the PR is still ejected. It is intentionally NOT part of + # that aggregation — it is best-effort cleanup, not a merge gate. if: >- always() && github.event_name == 'merge_group' && needs.changes.outputs.k8s == 'true' && - needs.deploy-prod.result == 'failure' + (needs.deploy-prod.result == 'failure' || + needs.deploy-prod.result == 'cancelled') runs-on: ubuntu-latest environment: prod # Same shared lock as deploy-prod / cd.yaml so the heal can never run against diff --git a/AGENTS.md b/AGENTS.md index 62cf1ee5c..12730519f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -345,6 +345,14 @@ These conventions guide the autonomous **Daily AI Assistant** — and any agenti **Merge queue — `main` IS gated by a GitHub merge queue** (`Require merge queue` ruleset). Merge mechanics differ from non-queue repos: `gh pr merge --auto` *enqueues* (don't pass `--squash` — the queue sets the strategy), and `autoMergeRequest` stays `null` even while a PR is queued, so a queued PR can look un-queued in JSON. A queued PR runs the **`merge_group`** event of `ci.yaml`, whose `deploy-prod` job **deploys to the real prod cluster** — so a `merge_group` failure **evicts the PR from the queue**. **Root-cause a stall/kick-out before re-queuing** (per the monorepo contract *Merge policy → Merge-queue repos*): a PR that "was queued" but didn't merge has usually failed its `merge_group` run — pull it (`gh run list --event merge_group --json headBranch,conclusion` → `pr-` → `gh run view --log-failed`) and diagnose. The `deploy-prod` step's **inline umami/coroot tenant provisioning** intermittently fails the gating verify on the Cilium mutual-auth first-packet drop (tracked in `#2337`); when that is the cause, re-queuing just re-hits it — advance the root-cause fix (e.g. `#2330` heal-on-failure) rather than looping the PR. Only a genuine one-off transient (runner OOM, network) warrants a clean re-queue. +**Safe cancellation:** once a merge-group `deploy-prod` job enters the shared deploy composite, it +may already have pushed the speculative ref to the mutable `latest` tag. Use only a normal workflow +cancellation; the `always()` heal job treats the cancelled deploy as unsuccessful and restores the +current tip of `main` after the production lock is released. Never force-cancel this workflow: +GitHub's force-cancel endpoint bypasses conditions such as `always()` and can strand the speculative +artifact. If a legacy/cancelled run did not execute `🩹 Heal Prod`, dispatch `CD` on `main` and +verify that deployment before treating the production lane as clean. + **Feature flags — four independent layers (feature-flag-first, monorepo#2059).** Land new behaviour **off**, validate it, then flip it on — using the right layer, coarsest first: 1. **Runtime per-request flags → flagd + OpenFeature Operator** (`k8s/bases/infrastructure/controllers/openfeature-operator/`, `#2510`). Flag definitions live in Git as **`FeatureFlag` CRs** (`core.openfeature.dev/v1beta1`) reconciled by Flux; workloads opt in with the `openfeature.dev/enabled` + `openfeature.dev/featureflagsource` pod annotations. Prefer **flagd-proxy** sync (`provider: flagd-proxy` on the `FeatureFlagSource`) so pods need no cluster-wide API RBAC — and so Flux never fights the operator over the `flagd-kubernetes-sync` ClusterRoleBinding (that drift only happens under `provider: kubernetes`). A `FeatureFlag` CR belongs in the **`infrastructure` layer**, never the controllers layer (a CR can't share a Flux Kustomization with the controller that installs its CRD). 2. **Version rollout / traffic shifting → Flagger** (already deployed): the release/canary toggle — "is this build safe to shift traffic to?", metric-analysed auto-rollback. Distinct from per-user flags; not a runtime flag. diff --git a/go.mod b/go.mod new file mode 100644 index 000000000..a6083fe70 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/devantler-tech/platform + +go 1.25.12 diff --git a/scripts/validate-merge-group-heal/main.go b/scripts/validate-merge-group-heal/main.go new file mode 100644 index 000000000..f63434aa3 --- /dev/null +++ b/scripts/validate-merge-group-heal/main.go @@ -0,0 +1,136 @@ +package main + +import ( + "errors" + "fmt" + "io" + "os" + "strings" +) + +const expectedHealCondition = "always() && " + + "github.event_name == 'merge_group' && " + + "needs.changes.outputs.k8s == 'true' && " + + "(needs.deploy-prod.result == 'failure' || " + + "needs.deploy-prod.result == 'cancelled')" + +func validateWorkflowContract(workflow string) error { + healJob, ok := extractHealJob(workflow) + if !ok { + return errors.New("missing heal-prod-on-failure job") + } + + requirements := []struct { + line string + description string + }{ + {line: " needs: [changes, deploy-prod]", description: "deploy dependencies"}, + {line: " group: prod-deploy", description: "shared production lock"}, + {line: " cancel-in-progress: false", description: "non-preempting production lock"}, + {line: " ref: main", description: "current-main checkout"}, + } + for _, requirement := range requirements { + if !containsExactLine(healJob, requirement.line) { + return fmt.Errorf("heal job is missing %s", requirement.description) + } + } + + condition, ok := extractMultilineCondition(healJob) + if !ok { + return errors.New("heal job must use an explicit multiline condition") + } + if strings.Join(strings.Fields(condition), " ") != expectedHealCondition { + return errors.New( + "heal condition must cover exactly failed and cancelled deploys while excluding success", + ) + } + + return nil +} + +func extractHealJob(workflow string) (string, bool) { + lines := strings.Split(workflow, "\n") + start := -1 + for i, line := range lines { + if line == " heal-prod-on-failure:" { + start = i + 1 + break + } + } + if start < 0 { + return "", false + } + + end := len(lines) + for i := start; i < len(lines); i++ { + line := lines[i] + if strings.HasPrefix(line, " ") && + !strings.HasPrefix(line, " ") && + strings.HasSuffix(line, ":") { + end = i + break + } + } + + return strings.Join(lines[start:end], "\n"), true +} + +func containsExactLine(block string, want string) bool { + for _, line := range strings.Split(block, "\n") { + if line == want { + return true + } + } + return false +} + +func extractMultilineCondition(job string) (string, bool) { + lines := strings.Split(job, "\n") + for i, line := range lines { + if line != " if: >-" { + continue + } + + conditionLines := make([]string, 0, 5) + for _, conditionLine := range lines[i+1:] { + if !strings.HasPrefix(conditionLine, " ") { + break + } + conditionLines = append(conditionLines, conditionLine) + } + if len(conditionLines) == 0 { + return "", false + } + return strings.Join(conditionLines, "\n"), true + } + + return "", false +} + +func run(workflowPath string, stdout io.Writer, stderr io.Writer) int { + workflow, err := os.ReadFile(workflowPath) //nolint:gosec // The explicit CLI path is the validator input. + if err != nil { + _, _ = fmt.Fprintf(stderr, "merge-group heal contract: read workflow: %v\n", err) + return 1 + } + + if err := validateWorkflowContract(string(workflow)); err != nil { + _, _ = fmt.Fprintf(stderr, "merge-group heal contract: %v\n", err) + return 1 + } + + _, _ = fmt.Fprintln(stdout, "Merge-group heal workflow contract passed.") + return 0 +} + +func runCLI(args []string, stdout io.Writer, stderr io.Writer) int { + if len(args) != 1 { + _, _ = fmt.Fprintln(stderr, "usage: validate-merge-group-heal ") + return 2 + } + return run(args[0], stdout, stderr) +} + +func main() { + os.Exit(runCLI(os.Args[1:], os.Stdout, os.Stderr)) +} diff --git a/scripts/validate-merge-group-heal/main_test.go b/scripts/validate-merge-group-heal/main_test.go new file mode 100644 index 000000000..fa8ec2d53 --- /dev/null +++ b/scripts/validate-merge-group-heal/main_test.go @@ -0,0 +1,236 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +const validWorkflow = `name: CI + +jobs: + changes: + runs-on: ubuntu-latest + + deploy-prod: + runs-on: ubuntu-latest + + heal-prod-on-failure: + needs: [changes, deploy-prod] + concurrency: + group: prod-deploy + cancel-in-progress: false + if: >- + always() && + github.event_name == 'merge_group' && + needs.changes.outputs.k8s == 'true' && + (needs.deploy-prod.result == 'failure' || + needs.deploy-prod.result == 'cancelled') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@example + with: + ref: main + + required-checks: + runs-on: ubuntu-latest +` + +func TestValidateWorkflowContractAcceptsFailClosedHealJob(t *testing.T) { + t.Parallel() + + if err := validateWorkflowContract(validWorkflow); err != nil { + t.Fatalf("validateWorkflowContract() error = %v", err) + } +} + +func TestValidateWorkflowContractRejectsBrokenHealContracts(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + old string + replacement string + wantError string + }{ + { + name: "missing heal job", + old: " heal-prod-on-failure:", + replacement: " heal-prod-disabled:", + wantError: "missing heal-prod-on-failure job", + }, + { + name: "missing deploy dependencies", + old: " needs: [changes, deploy-prod]", + replacement: " needs: [changes]", + wantError: "missing deploy dependencies", + }, + { + name: "missing production lock", + old: " group: prod-deploy", + replacement: " group: other-deploy", + wantError: "missing shared production lock", + }, + { + name: "preempting production lock", + old: " cancel-in-progress: false", + replacement: " cancel-in-progress: true", + wantError: "missing non-preempting production lock", + }, + { + name: "missing main checkout", + old: " ref: main", + replacement: " ref: merge-group", + wantError: "missing current-main checkout", + }, + { + name: "implicit condition", + old: " if: >-", + replacement: " if: |", + wantError: "must use an explicit multiline condition", + }, + { + name: "condition includes success", + old: "needs.deploy-prod.result == 'failure'", + replacement: "needs.deploy-prod.result == 'success'", + wantError: "must cover exactly failed and cancelled deploys while excluding success", + }, + { + name: "condition drops cancellation", + old: "needs.deploy-prod.result == 'cancelled'", + replacement: "needs.deploy-prod.result == 'failure'", + wantError: "must cover exactly failed and cancelled deploys while excluding success", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + workflow := strings.Replace(validWorkflow, tt.old, tt.replacement, 1) + if workflow == validWorkflow { + t.Fatalf("test replacement %q did not change workflow", tt.old) + } + + err := validateWorkflowContract(workflow) + if err == nil || !strings.Contains(err.Error(), tt.wantError) { + t.Fatalf("validateWorkflowContract() error = %v, want containing %q", err, tt.wantError) + } + }) + } +} + +func TestRunReportsValidationResult(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + workflow string + wantCode int + wantStdout string + wantStderr string + }{ + { + name: "valid contract", + workflow: validWorkflow, + wantCode: 0, + wantStdout: "Merge-group heal workflow contract passed.\n", + }, + { + name: "invalid contract", + workflow: strings.Replace(validWorkflow, " ref: main", " ref: merge-group", 1), + wantCode: 1, + wantStderr: "merge-group heal contract: heal job is missing current-main checkout\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + workflowPath := filepath.Join(t.TempDir(), "ci.yaml") + if err := os.WriteFile(workflowPath, []byte(tt.workflow), 0o600); err != nil { + t.Fatalf("write workflow: %v", err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + if got := run(workflowPath, &stdout, &stderr); got != tt.wantCode { + t.Fatalf("run() code = %d, want %d", got, tt.wantCode) + } + if got := stdout.String(); got != tt.wantStdout { + t.Fatalf("run() stdout = %q, want %q", got, tt.wantStdout) + } + if got := stderr.String(); got != tt.wantStderr { + t.Fatalf("run() stderr = %q, want %q", got, tt.wantStderr) + } + }) + } +} + +func TestRunReportsWorkflowReadFailure(t *testing.T) { + t.Parallel() + + var stdout bytes.Buffer + var stderr bytes.Buffer + workflowPath := filepath.Join(t.TempDir(), "missing.yaml") + + if got := run(workflowPath, &stdout, &stderr); got != 1 { + t.Fatalf("run() code = %d, want 1", got) + } + if stdout.Len() != 0 { + t.Fatalf("run() stdout = %q, want empty", stdout.String()) + } + if got := stderr.String(); !strings.Contains(got, "merge-group heal contract: read workflow:") { + t.Fatalf("run() stderr = %q, want read failure", got) + } +} + +func TestRunCLIUsesExplicitWorkflowPathOutsideRepository(t *testing.T) { + workflowPath := filepath.Join(t.TempDir(), "ci.yaml") + if err := os.WriteFile(workflowPath, []byte(validWorkflow), 0o600); err != nil { + t.Fatalf("write workflow: %v", err) + } + t.Chdir(t.TempDir()) + + var stdout bytes.Buffer + var stderr bytes.Buffer + if got := runCLI([]string{workflowPath}, &stdout, &stderr); got != 0 { + t.Fatalf("runCLI() code = %d, want 0; stderr = %q", got, stderr.String()) + } + if got := stdout.String(); got != "Merge-group heal workflow contract passed.\n" { + t.Fatalf("runCLI() stdout = %q", got) + } +} + +func TestRunCLIRequiresOneWorkflowPath(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + args []string + }{ + {name: "missing path"}, + {name: "extra path", args: []string{"ci.yaml", "other.yaml"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var stdout bytes.Buffer + var stderr bytes.Buffer + if got := runCLI(tt.args, &stdout, &stderr); got != 2 { + t.Fatalf("runCLI() code = %d, want 2", got) + } + if stdout.Len() != 0 { + t.Fatalf("runCLI() stdout = %q, want empty", stdout.String()) + } + if got := stderr.String(); got != "usage: validate-merge-group-heal \n" { + t.Fatalf("runCLI() stderr = %q", got) + } + }) + } +}