fix(ci): heal cancelled merge-group deploys

.github/workflows/ci.yaml

@@ -27,6 +27,17 @@ jobs:
           persist-credentials: false
           fetch-depth: 0
 
+      - name: ⚙️ Setup Go
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: 🩹 Validate merge-group heal contract
+        run: |
+          go test ./scripts/validate-merge-group-heal
+          go run ./scripts/validate-merge-group-heal .github/workflows/ci.yaml
+
       - name: 🔍 Filter paths
         uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
         id: filter
@@ -334,24 +345,28 @@ jobs:
           wg-server-private-key: ${{ secrets.WG_SERVER_PRIVATE_KEY }}
 
   heal-prod-on-failure:
-    name: 🩹 Heal Prod (restore main on merge-group failure)
+    name: 🩹 Heal Prod (restore main after unsuccessful merge-group deploy)
     needs: [changes, deploy-prod]
     # The deploy above runs on the SPECULATIVE merge ref, before the PR lands on
     # `main`. When it fails the merge group is ejected — but the OCI `latest` tag
     # (and the prod cluster reconciling it) is left pointing at that ejected
     # artifact, which never lands on `main`, so prod silently diverges from Git
     # until some future merge overwrites it (devantler-tech/platform#2026). This
-    # failure-path-only job re-deploys the current tip of `main` via the SAME
-    # composite, so the ejected artifact is immediately replaced by main's signed
-    # artifact and prod reconverges with Git. It has ZERO effect on a green merge
-    # (skipped) and does NOT relax the gate: deploy-prod's failure still fails
-    # `CI - Required Checks` below, so the PR is still ejected. It is intentionally
-    # NOT part of that aggregation — it is best-effort cleanup, not a merge gate.
+    # unsuccessful-path-only job re-deploys the current tip of `main` via the
+    # SAME composite, so the speculative artifact is immediately replaced by
+    # main's signed artifact and prod reconverges with Git. It covers explicit
+    # cancellation too: GitHub re-evaluates job conditions during normal
+    # cancellation, and `always()` keeps this cleanup eligible after the deploy
+    # job becomes cancelled. It has ZERO effect on a green merge (skipped) and
+    # does NOT relax the gate: deploy-prod's failure still fails `CI - Required
+    # Checks` below, so the PR is still ejected. It is intentionally NOT part of
+    # that aggregation — it is best-effort cleanup, not a merge gate.
     if: >-
       always() &&
       github.event_name == 'merge_group' &&
       needs.changes.outputs.k8s == 'true' &&
-      needs.deploy-prod.result == 'failure'
+      (needs.deploy-prod.result == 'failure' ||
+       needs.deploy-prod.result == 'cancelled')
     runs-on: ubuntu-latest
     environment: prod
     # Same shared lock as deploy-prod / cd.yaml so the heal can never run against

AGENTS.md

@@ -345,6 +345,14 @@ These conventions guide the autonomous **Daily AI Assistant** — and any agenti
 
 **Merge queue — `main` IS gated by a GitHub merge queue** (`Require merge queue` ruleset). Merge mechanics differ from non-queue repos: `gh pr merge --auto` *enqueues* (don't pass `--squash` — the queue sets the strategy), and `autoMergeRequest` stays `null` even while a PR is queued, so a queued PR can look un-queued in JSON. A queued PR runs the **`merge_group`** event of `ci.yaml`, whose `deploy-prod` job **deploys to the real prod cluster** — so a `merge_group` failure **evicts the PR from the queue**. **Root-cause a stall/kick-out before re-queuing** (per the monorepo contract *Merge policy → Merge-queue repos*): a PR that "was queued" but didn't merge has usually failed its `merge_group` run — pull it (`gh run list --event merge_group --json headBranch,conclusion` → `pr-<n>` → `gh run view --log-failed`) and diagnose. The `deploy-prod` step's **inline umami/coroot tenant provisioning** intermittently fails the gating verify on the Cilium mutual-auth first-packet drop (tracked in `#2337`); when that is the cause, re-queuing just re-hits it — advance the root-cause fix (e.g. `#2330` heal-on-failure) rather than looping the PR. Only a genuine one-off transient (runner OOM, network) warrants a clean re-queue.
 
+**Safe cancellation:** once a merge-group `deploy-prod` job enters the shared deploy composite, it
+may already have pushed the speculative ref to the mutable `latest` tag. Use only a normal workflow
+cancellation; the `always()` heal job treats the cancelled deploy as unsuccessful and restores the
+current tip of `main` after the production lock is released. Never force-cancel this workflow:
+GitHub's force-cancel endpoint bypasses conditions such as `always()` and can strand the speculative
+artifact. If a legacy/cancelled run did not execute `🩹 Heal Prod`, dispatch `CD` on `main` and
+verify that deployment before treating the production lane as clean.
+
 **Feature flags — four independent layers (feature-flag-first, monorepo#2059).** Land new behaviour **off**, validate it, then flip it on — using the right layer, coarsest first:
 1. **Runtime per-request flags → flagd + OpenFeature Operator** (`k8s/bases/infrastructure/controllers/openfeature-operator/`, `#2510`). Flag definitions live in Git as **`FeatureFlag` CRs** (`core.openfeature.dev/v1beta1`) reconciled by Flux; workloads opt in with the `openfeature.dev/enabled` + `openfeature.dev/featureflagsource` pod annotations. Prefer **flagd-proxy** sync (`provider: flagd-proxy` on the `FeatureFlagSource`) so pods need no cluster-wide API RBAC — and so Flux never fights the operator over the `flagd-kubernetes-sync` ClusterRoleBinding (that drift only happens under `provider: kubernetes`). A `FeatureFlag` CR belongs in the **`infrastructure` layer**, never the controllers layer (a CR can't share a Flux Kustomization with the controller that installs its CRD).
 2. **Version rollout / traffic shifting → Flagger** (already deployed): the release/canary toggle — "is this build safe to shift traffic to?", metric-analysed auto-rollback. Distinct from per-user flags; not a runtime flag.

go.mod

@@ -0,0 +1,3 @@
+module github.com/devantler-tech/platform
+
+go 1.25.12

scripts/validate-merge-group-heal/main.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+const expectedHealCondition = "always() && " +
+	"github.event_name == 'merge_group' && " +
+	"needs.changes.outputs.k8s == 'true' && " +
+	"(needs.deploy-prod.result == 'failure' || " +
+	"needs.deploy-prod.result == 'cancelled')"
+
+func validateWorkflowContract(workflow string) error {
+	healJob, ok := extractHealJob(workflow)
+	if !ok {
+		return errors.New("missing heal-prod-on-failure job")
+	}
+
+	requirements := []struct {
+		line        string
+		description string
+	}{
+		{line: "    needs: [changes, deploy-prod]", description: "deploy dependencies"},
+		{line: "      group: prod-deploy", description: "shared production lock"},
+		{line: "      cancel-in-progress: false", description: "non-preempting production lock"},
+		{line: "          ref: main", description: "current-main checkout"},
+	}
+	for _, requirement := range requirements {
+		if !containsExactLine(healJob, requirement.line) {
+			return fmt.Errorf("heal job is missing %s", requirement.description)
+		}
+	}
+
+	condition, ok := extractMultilineCondition(healJob)
+	if !ok {
+		return errors.New("heal job must use an explicit multiline condition")
+	}
+	if strings.Join(strings.Fields(condition), " ") != expectedHealCondition {
+		return errors.New(
+			"heal condition must cover exactly failed and cancelled deploys while excluding success",
+		)
+	}
+
+	return nil
+}
+
+func extractHealJob(workflow string) (string, bool) {
+	lines := strings.Split(workflow, "\n")
+	start := -1
+	for i, line := range lines {
+		if line == "  heal-prod-on-failure:" {
+			start = i + 1
+			break
+		}
+	}
+	if start < 0 {
+		return "", false
+	}
+
+	end := len(lines)
+	for i := start; i < len(lines); i++ {
+		line := lines[i]
+		if strings.HasPrefix(line, "  ") &&
+			!strings.HasPrefix(line, "   ") &&
+			strings.HasSuffix(line, ":") {
+			end = i
+			break
+		}
+	}
+
+	return strings.Join(lines[start:end], "\n"), true
+}
+
+func containsExactLine(block string, want string) bool {
+	for _, line := range strings.Split(block, "\n") {
+		if line == want {
+			return true
+		}
+	}
+	return false
+}
+
+func extractMultilineCondition(job string) (string, bool) {
+	lines := strings.Split(job, "\n")
+	for i, line := range lines {
+		if line != "    if: >-" {
+			continue
+		}
+
+		conditionLines := make([]string, 0, 5)
+		for _, conditionLine := range lines[i+1:] {
+			if !strings.HasPrefix(conditionLine, "      ") {
+				break
+			}
+			conditionLines = append(conditionLines, conditionLine)
+		}
+		if len(conditionLines) == 0 {
+			return "", false
+		}
+		return strings.Join(conditionLines, "\n"), true
+	}
+
+	return "", false
+}
+
+func run(workflowPath string, stdout io.Writer, stderr io.Writer) int {
+	workflow, err := os.ReadFile(workflowPath) //nolint:gosec // The explicit CLI path is the validator input.
+	if err != nil {
+		_, _ = fmt.Fprintf(stderr, "merge-group heal contract: read workflow: %v\n", err)
+		return 1
+	}
+
+	if err := validateWorkflowContract(string(workflow)); err != nil {
+		_, _ = fmt.Fprintf(stderr, "merge-group heal contract: %v\n", err)
+		return 1
+	}
+
+	_, _ = fmt.Fprintln(stdout, "Merge-group heal workflow contract passed.")
+	return 0
+}
+
+func runCLI(args []string, stdout io.Writer, stderr io.Writer) int {
+	if len(args) != 1 {
+		_, _ = fmt.Fprintln(stderr, "usage: validate-merge-group-heal <workflow-path>")
+		return 2
+	}
+	return run(args[0], stdout, stderr)
+}
+
+func main() {
+	os.Exit(runCLI(os.Args[1:], os.Stdout, os.Stderr))
+}