Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ jobs:
- 'talos/**'
- 'scripts/validate-naming.py'
- 'scripts/validate-embedded-json.py'
- 'scripts/tests/test-cilium-bandwidth-manager-component.sh'
- 'tests/cilium-bandwidth-manager-bbr/**'
- '.github/workflows/ci.yaml'
talos:
- 'talos/**'
Expand Down Expand Up @@ -75,6 +77,13 @@ jobs:
# scripts/validate-embedded-json.py (#2480).
run: python3 scripts/validate-embedded-json.py

- name: 🚦 Validate default-off Cilium bandwidth manager
# Render the committed and temporary opt-in states so the staged
# bandwidth-manager + BBR component cannot silently become active or
# regress the Hetzner WireGuard settings. Bash plus the kubectl bundled
# on the GitHub-hosted runner; no cluster or secrets required.
run: bash scripts/tests/test-cilium-bandwidth-manager-component.sh

- name: ⚙️ Setup KSail
# Renovate-managed (datasource github-releases; grouped 'ksail' with the
# deploy-prod / dr-rebuild pins). The validate step below renders
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
---
# Default-off staging component for platform#2029 / platform#2607.
#
# Cilium's bandwidth manager installs fq qdiscs and EDT pacing, while BBR
# selects BBR congestion control for eligible egress traffic. The Talos 6.x
# kernel is newer than Cilium's Linux 5.18 minimum for BBR. This component is
# Hetzner-only because that is the production path where egress traverses the
# node datapath and where WireGuard is enabled.
#
# HOST-NAMESPACE GUARD: pod BBR also requires eBPF host routing. The current
# tunnel-mode datapath intentionally keeps bpf.masquerade off, so
# bbrHostNamespaceOnly stays true and limits BBR to hostNetwork workloads.
# Do not remove that guard until #2609 has merged and completed its own soak.
#
# DEVICE GUARD: production currently pins two heterogeneous private-NIC names
# (`enp7s0` on workers, `eth1` on control planes). Cilium requires every
# manually listed bandwidth-manager device name on every managed node. Do not
# activate this component until #2610 provides a homogeneous selection while
# preserving the private-NIC WireGuard and mutual-auth path.
#
# SAFETY: this component is staged, not activation-ready. Keep it unreferenced
# until both #2609 and #2610 are complete, then use a separate activation/soak.
# `rollOutCiliumPods: true` means enabling it rolls every Cilium agent one node
# at a time and replaces the node qdisc. Activate only in a low-traffic window;
# verify `cilium status --verbose` and unexpected Hubble drops after the roll.
# Roll back by removing the component reference, which restores the chart
# defaults in the next one-node-at-a-time roll.
#
# bpf.masquerade and netkit are deliberately out of scope and require their own
# independently validated slices.
apiVersion: kustomize.config.k8s.io/v1alpha1
kind: Component
patches:
- target:
group: helm.toolkit.fluxcd.io
version: v2
kind: HelmRelease
name: cilium
namespace: kube-system
patch: |
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: cilium
namespace: kube-system
spec:
values:
bandwidthManager:
enabled: true
Comment thread
devantler marked this conversation as resolved.
bbr: true
Comment thread
devantler marked this conversation as resolved.
bbrHostNamespaceOnly: true
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,12 @@ components:
# silently-broken keda external-scaler and homepage Deployments.
- ../../../../bases/components/helmrelease-drift-detection
- ../../../../bases/components/helmrelease-flux-defaults
# Default-off Cilium bandwidth-manager + host-only BBR staging component
# (#2607). It is not activation-ready: #2609 must prove pod-BBR datapath
# prerequisites and #2610 must make device selection homogeneous first.
# Enabling replaces the node qdisc and rolls every Cilium agent; after both
# guards close, use a separate low-traffic activation. Rollback = re-comment.
# - cilium/components/bandwidth-manager-bbr/
patches:
- path: openbao/patches/store-data-on-hcloud.yaml
# Prod-only fail-closed WireGuard encryption (egress strict mode) — see file.
Expand Down
127 changes: 127 additions & 0 deletions scripts/tests/test-cilium-bandwidth-manager-component.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
#!/usr/bin/env bash

set -euo pipefail

root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
readonly root_dir
readonly controllers_dir="${root_dir}/k8s/providers/hetzner/infrastructure/controllers"
readonly opt_in_fixture="${root_dir}/tests/cilium-bandwidth-manager-bbr"

fail() {
printf 'FAIL: %s\n' "$1" >&2
exit 1
}

extract_cilium_release() {
awk '
function reset_document() {
document = ""
is_helm_release = 0
is_cilium = 0
}

function emit_if_cilium() {
if (!found && is_helm_release && is_cilium) {
printf "%s", document
found = 1
}
reset_document()
}

BEGIN { reset_document() }
/^---[[:space:]]*$/ { emit_if_cilium(); next }
{
if (!found) {
document = document $0 ORS
if ($0 ~ /^kind:[[:space:]]*HelmRelease[[:space:]]*$/) {
is_helm_release = 1
}
if ($0 ~ /^ name:[[:space:]]*cilium[[:space:]]*$/) {
is_cilium = 1
}
}
}
END {
if (!found) {
emit_if_cilium()
}
if (!found) {
exit 1
}
}
'
}

require_text() {
local haystack="$1"
local needle="$2"
local description="$3"

grep -Fq -- "$needle" <<<"${haystack}" || fail "${description}"
}

reject_text() {
local haystack="$1"
local needle="$2"
local description="$3"

if grep -Fq -- "$needle" <<<"${haystack}"; then
fail "${description}"
fi
}

extractor_probe="$(
{
printf '%s\n' \
'apiVersion: helm.toolkit.fluxcd.io/v2' \
'kind: HelmRelease' \
'metadata:' \
' name: cilium' \
'---'
for ((line = 0; line < 10000; line++)); do
printf '# trailing render content %05d\n' "${line}"
done
} | extract_cilium_release
)" || fail 'the Cilium release extractor must consume the complete render stream'
require_text \
"${extractor_probe}" \
'kind: HelmRelease' \
'the Cilium release extractor must return the matching document'

readonly controllers_kustomization="${controllers_dir}/kustomization.yaml"
grep -Fxq ' # - cilium/components/bandwidth-manager-bbr/' "${controllers_kustomization}" ||
fail 'the production controllers overlay must retain the documented opt-in reference'
if grep -Fxq ' - cilium/components/bandwidth-manager-bbr/' "${controllers_kustomization}"; then
fail 'the production controllers overlay must keep the BBR component disabled by default'
fi

default_release="$(kubectl kustomize "${controllers_dir}" | extract_cilium_release)" ||
Comment thread
devantler marked this conversation as resolved.
fail 'the default production controllers render has no Cilium HelmRelease'
reject_text \
"${default_release}" \
'bandwidthManager:' \
'the default production render unexpectedly enables the bandwidth manager'

opt_in_release="$(
kubectl kustomize "${opt_in_fixture}" --load-restrictor LoadRestrictionsNone |
extract_cilium_release
)" || fail 'the opt-in fixture render has no Cilium HelmRelease'

require_text \
"${opt_in_release}" \
$'bandwidthManager:\n bbr: true\n bbrHostNamespaceOnly: true\n enabled: true' \
'the opt-in render must enable host-namespace-only BBR'
reject_text \
"${opt_in_release}" \
$'bpf:\n masquerade: true' \
'the opt-in render must not enable BPF masquerading'
require_text \
"${opt_in_release}" \
$'encryption:\n enabled: true\n nodeEncryption: false' \
'the opt-in render must preserve the production encryption settings'
require_text \
"${opt_in_release}" \
'type: wireguard' \
'the opt-in render must preserve WireGuard encryption'

printf 'PASS: Cilium bandwidth manager is default-off and the opt-in render preserves production guards\n'
7 changes: 7 additions & 0 deletions tests/cilium-bandwidth-manager-bbr/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../k8s/providers/hetzner/infrastructure/controllers/
components:
- ../../k8s/providers/hetzner/infrastructure/controllers/cilium/components/bandwidth-manager-bbr/
Loading