diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e479ca1b9..ecbf2bfc1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -53,6 +53,8 @@ jobs: - 'talos/**' - 'scripts/validate-naming.py' - 'scripts/validate-embedded-json.py' + - 'scripts/tests/test-cilium-bandwidth-manager-component.sh' + - 'tests/cilium-bandwidth-manager-bbr/**' - 'scripts/refresh-flux-ghcr-auth.sh' - 'scripts/ghcr-auth-lib.sh' - 'scripts/run-ksail-prod-with-pull-auth.sh' @@ -118,6 +120,14 @@ jobs: # scripts/validate-embedded-json.py (#2480). run: python3 scripts/validate-embedded-json.py + - name: 🚦 Validate default-off Cilium bandwidth manager + if: needs.changes.outputs.k8s == 'true' + # Render the committed and temporary opt-in states so the staged + # bandwidth-manager + BBR component cannot silently become active or + # regress the Hetzner WireGuard settings. Bash plus the kubectl bundled + # on the GitHub-hosted runner; no cluster or secrets required. + run: bash scripts/tests/test-cilium-bandwidth-manager-component.sh + - name: ⚙️ Setup KSail # Renovate-managed (datasource github-releases; grouped 'ksail' with the # deploy-prod / dr-rebuild pins). The validate step below renders diff --git a/k8s/providers/hetzner/infrastructure/controllers/cilium/components/bandwidth-manager-bbr/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/cilium/components/bandwidth-manager-bbr/kustomization.yaml new file mode 100644 index 000000000..cccd20dad --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/cilium/components/bandwidth-manager-bbr/kustomization.yaml @@ -0,0 +1,51 @@ +--- +# Default-off staging component for platform#2029 / platform#2607. +# +# Cilium's bandwidth manager installs fq qdiscs and EDT pacing, while BBR +# selects BBR congestion control for eligible egress traffic. The Talos 6.x +# kernel is newer than Cilium's Linux 5.18 minimum for BBR. This component is +# Hetzner-only because that is the production path where egress traverses the +# node datapath and where WireGuard is enabled. +# +# HOST-NAMESPACE GUARD: pod BBR also requires eBPF host routing. The current +# tunnel-mode datapath intentionally keeps bpf.masquerade off, so +# bbrHostNamespaceOnly stays true and limits BBR to hostNetwork workloads. +# Do not remove that guard until #2609 has merged and completed its own soak. +# +# DEVICE GUARD: production currently pins two heterogeneous private-NIC names +# (`enp7s0` on workers, `eth1` on control planes). Cilium requires every +# manually listed bandwidth-manager device name on every managed node. Do not +# activate this component until #2610 provides a homogeneous selection while +# preserving the private-NIC WireGuard and mutual-auth path. +# +# SAFETY: this component is staged, not activation-ready. Keep it unreferenced +# until both #2609 and #2610 are complete, then use a separate activation/soak. +# `rollOutCiliumPods: true` means enabling it rolls every Cilium agent one node +# at a time and replaces the node qdisc. Activate only in a low-traffic window; +# verify `cilium status --verbose` and unexpected Hubble drops after the roll. +# Roll back by removing the component reference, which restores the chart +# defaults in the next one-node-at-a-time roll. +# +# bpf.masquerade and netkit are deliberately out of scope and require their own +# independently validated slices. +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +patches: + - target: + group: helm.toolkit.fluxcd.io + version: v2 + kind: HelmRelease + name: cilium + namespace: kube-system + patch: | + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: cilium + namespace: kube-system + spec: + values: + bandwidthManager: + enabled: true + bbr: true + bbrHostNamespaceOnly: true diff --git a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml index 7e9d48c4e..751b10956 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml @@ -66,6 +66,12 @@ components: # silently-broken keda external-scaler and homepage Deployments. - ../../../../bases/components/helmrelease-drift-detection - ../../../../bases/components/helmrelease-flux-defaults + # Default-off Cilium bandwidth-manager + host-only BBR staging component + # (#2607). It is not activation-ready: #2609 must prove pod-BBR datapath + # prerequisites and #2610 must make device selection homogeneous first. + # Enabling replaces the node qdisc and rolls every Cilium agent; after both + # guards close, use a separate low-traffic activation. Rollback = re-comment. + # - cilium/components/bandwidth-manager-bbr/ patches: - path: openbao/patches/store-data-on-hcloud.yaml # Prod-only fail-closed WireGuard encryption (egress strict mode) — see file. diff --git a/scripts/tests/test-cilium-bandwidth-manager-component.sh b/scripts/tests/test-cilium-bandwidth-manager-component.sh new file mode 100755 index 000000000..8e10f95ee --- /dev/null +++ b/scripts/tests/test-cilium-bandwidth-manager-component.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash + +set -euo pipefail + +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +readonly root_dir +readonly controllers_dir="${root_dir}/k8s/providers/hetzner/infrastructure/controllers" +readonly opt_in_fixture="${root_dir}/tests/cilium-bandwidth-manager-bbr" +readonly ci_workflow="${root_dir}/.github/workflows/ci.yaml" + +fail() { + printf 'FAIL: %s\n' "$1" >&2 + exit 1 +} + +extract_cilium_release() { + awk ' + function reset_document() { + document = "" + is_helm_release = 0 + is_cilium = 0 + } + + function emit_if_cilium() { + if (!found && is_helm_release && is_cilium) { + printf "%s", document + found = 1 + } + reset_document() + } + + BEGIN { reset_document() } + /^---[[:space:]]*$/ { emit_if_cilium(); next } + { + if (!found) { + document = document $0 ORS + if ($0 ~ /^kind:[[:space:]]*HelmRelease[[:space:]]*$/) { + is_helm_release = 1 + } + if ($0 ~ /^ name:[[:space:]]*cilium[[:space:]]*$/) { + is_cilium = 1 + } + } + } + END { + if (!found) { + emit_if_cilium() + } + if (!found) { + exit 1 + } + } + ' +} + +require_text() { + local haystack="$1" + local needle="$2" + local description="$3" + + grep -Fq -- "$needle" <<<"${haystack}" || fail "${description}" +} + +reject_text() { + local haystack="$1" + local needle="$2" + local description="$3" + + if grep -Fq -- "$needle" <<<"${haystack}"; then + fail "${description}" + fi +} + +awk ' + /^ - name: 🚦 Validate default-off Cilium bandwidth manager$/ { + found_step = 1 + next + } + found_step && /^ if: needs\.changes\.outputs\.k8s == '\''true'\''$/ { + found_gate = 1 + exit + } + found_step && /^ - name:/ { + exit + } + END { + exit !(found_step && found_gate) + } +' "${ci_workflow}" || + fail 'the BBR workflow step must be gated to k8s changes' + +extractor_probe="$( + { + printf '%s\n' \ + 'apiVersion: helm.toolkit.fluxcd.io/v2' \ + 'kind: HelmRelease' \ + 'metadata:' \ + ' name: cilium' \ + '---' + for ((line = 0; line < 10000; line++)); do + printf '# trailing render content %05d\n' "${line}" + done + } | extract_cilium_release +)" || fail 'the Cilium release extractor must consume the complete render stream' +require_text \ + "${extractor_probe}" \ + 'kind: HelmRelease' \ + 'the Cilium release extractor must return the matching document' + +readonly controllers_kustomization="${controllers_dir}/kustomization.yaml" +grep -Fxq ' # - cilium/components/bandwidth-manager-bbr/' "${controllers_kustomization}" || + fail 'the production controllers overlay must retain the documented opt-in reference' +if grep -Fxq ' - cilium/components/bandwidth-manager-bbr/' "${controllers_kustomization}"; then + fail 'the production controllers overlay must keep the BBR component disabled by default' +fi + +default_release="$(kubectl kustomize "${controllers_dir}" | extract_cilium_release)" || + fail 'the default production controllers render has no Cilium HelmRelease' +reject_text \ + "${default_release}" \ + 'bandwidthManager:' \ + 'the default production render unexpectedly enables the bandwidth manager' + +opt_in_release="$( + kubectl kustomize "${opt_in_fixture}" --load-restrictor LoadRestrictionsNone | + extract_cilium_release +)" || fail 'the opt-in fixture render has no Cilium HelmRelease' + +require_text \ + "${opt_in_release}" \ + $'bandwidthManager:\n bbr: true\n bbrHostNamespaceOnly: true\n enabled: true' \ + 'the opt-in render must enable host-namespace-only BBR' +reject_text \ + "${opt_in_release}" \ + $'bpf:\n masquerade: true' \ + 'the opt-in render must not enable BPF masquerading' +require_text \ + "${opt_in_release}" \ + $'encryption:\n enabled: true\n nodeEncryption: false' \ + 'the opt-in render must preserve the production encryption settings' +require_text \ + "${opt_in_release}" \ + 'type: wireguard' \ + 'the opt-in render must preserve WireGuard encryption' + +printf 'PASS: Cilium bandwidth manager is default-off and the opt-in render preserves production guards\n' diff --git a/tests/cilium-bandwidth-manager-bbr/kustomization.yaml b/tests/cilium-bandwidth-manager-bbr/kustomization.yaml new file mode 100644 index 000000000..1248c925e --- /dev/null +++ b/tests/cilium-bandwidth-manager-bbr/kustomization.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../../k8s/providers/hetzner/infrastructure/controllers/ +components: + - ../../k8s/providers/hetzner/infrastructure/controllers/cilium/components/bandwidth-manager-bbr/