From c6d188a77c72ff0c20182ed69aee8f52ae6b12ee Mon Sep 17 00:00:00 2001 From: Craig Ingram Date: Mon, 25 May 2026 16:59:47 -0400 Subject: [PATCH 1/3] Upgrade Varnish to 9.0.3 and document custom container images --- Dockerfile.controller | 19 +--- Dockerfile.exporter | 12 +-- Dockerfile.varnishd | 12 +-- Makefile | 9 +- README.md | 3 +- api/v1alpha1/varnishcluster_types.go | 6 +- config/samples/varnishcluster.yaml | 10 +- docker/install-varnish-9.sh | 47 ++++++++ docs/README.md | 3 +- docs/SUMMARY.md | 1 + docs/architecture.md | 2 +- docs/custom-images.md | 147 ++++++++++++++++++++++++++ docs/development.md | 12 ++- docs/operator-configuration.md | 2 + docs/varnish-cluster-configuration.md | 14 ++- docs/vcl-configuration.md | 2 +- 16 files changed, 254 insertions(+), 47 deletions(-) create mode 100644 docker/install-varnish-9.sh create mode 100644 docs/custom-images.md diff --git a/Dockerfile.controller b/Dockerfile.controller index 7927e4f9..bf432e68 100644 --- a/Dockerfile.controller +++ b/Dockerfile.controller @@ -1,4 +1,5 @@ ARG BUILDPLATFORM=linux/amd64 +ARG VARNISH_VERSION_NUMBER=9.0.3-1 FROM --platform=$BUILDPLATFORM golang:1.26-bookworm AS builder ENV DEBIAN_FRONTEND=noninteractive INSTALL_DIRECTORY=/usr/local/bin @@ -28,24 +29,14 @@ RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ ./cmd/varnish-controller/ -FROM --platform=$BUILDPLATFORM debian:trixie-slim AS binary - -RUN apt-get update && apt-get install -y --no-install-recommends varnish \ - && rm -rf /var/lib/apt/lists/* - - -# Build Final Varnish image FROM --platform=$BUILDPLATFORM debian:trixie-slim LABEL maintainer="Alex Lytvynenko , Tomash Sidei " -RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ - libc6 libedit2 libncursesw6 libtinfo6 libvarnishapi3 varnish \ - && rm -rf /var/lib/apt/lists/* \ - /etc/varnish/* \ - && mkdir -p /etc/varnish /var/lib/varnish \ - && chown -R varnish:varnish /etc/varnish /var/lib/varnish +COPY docker/install-varnish-9.sh /tmp/install-varnish-9.sh +RUN chmod +x /tmp/install-varnish-9.sh \ + && VARNISH_VERSION_NUMBER="${VARNISH_VERSION_NUMBER}" /tmp/install-varnish-9.sh tools \ + && rm /tmp/install-varnish-9.sh -COPY --from=binary /usr/bin/varnishadm /usr/bin/varnishstat /usr/bin/ COPY --from=builder /go/src/github.com/cin/varnish-operator/varnish-controller /varnish-controller RUN chown varnish:varnish /varnish-controller /usr/bin/varnishadm /usr/bin/varnishstat diff --git a/Dockerfile.exporter b/Dockerfile.exporter index 2bdc39f8..0dc3ca63 100644 --- a/Dockerfile.exporter +++ b/Dockerfile.exporter @@ -1,4 +1,5 @@ ARG BUILDPLATFORM=linux/amd64 +ARG VARNISH_VERSION_NUMBER=9.0.3-1 FROM --platform=$BUILDPLATFORM golang:1.26-bookworm AS builder ARG TARGETOS=linux ARG TARGETARCH=amd64 @@ -14,15 +15,14 @@ WORKDIR /src/prometheus_varnish_exporter RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /prometheus-varnish-exporter . + FROM --platform=$BUILDPLATFORM debian:trixie-slim LABEL maintainer="Alex Lytvynenko , Tomash Sidei " -# Install varnish so varnishstat and the varnish/vcache users match the varnishd image (VSM access). -RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ - libc6 libedit2 libncursesw6 libtinfo6 libvarnishapi3 varnish \ - && rm -rf /var/lib/apt/lists/* /etc/varnish/* \ - && mkdir -p /etc/varnish /var/lib/varnish \ - && chown -R varnish:varnish /etc/varnish /var/lib/varnish +COPY docker/install-varnish-9.sh /tmp/install-varnish-9.sh +RUN chmod +x /tmp/install-varnish-9.sh \ + && VARNISH_VERSION_NUMBER="${VARNISH_VERSION_NUMBER}" /tmp/install-varnish-9.sh tools \ + && rm /tmp/install-varnish-9.sh COPY --from=builder /prometheus-varnish-exporter /usr/bin/ RUN chown varnish:varnish /usr/bin/prometheus-varnish-exporter diff --git a/Dockerfile.varnishd b/Dockerfile.varnishd index d2521816..b9ad1789 100644 --- a/Dockerfile.varnishd +++ b/Dockerfile.varnishd @@ -1,13 +1,13 @@ ARG BUILDPLATFORM=linux/amd64 +ARG VARNISH_VERSION_NUMBER=9.0.3-1 FROM --platform=$BUILDPLATFORM debian:trixie-slim LABEL maintainer="Alex Lytvynenko , Tomash Sidei " -RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ - varnish \ - varnish-modules \ - && rm -rf /var/lib/apt/lists/* /etc/varnish/* \ - && chown -R varnish:varnish /etc/varnish /var/lib/varnish +COPY docker/install-varnish-9.sh /tmp/install-varnish-9.sh +RUN chmod +x /tmp/install-varnish-9.sh \ + && VARNISH_VERSION_NUMBER="${VARNISH_VERSION_NUMBER}" /tmp/install-varnish-9.sh minimal \ + && rm /tmp/install-varnish-9.sh USER varnish -ENTRYPOINT ["varnishd"] +ENTRYPOINT ["/usr/sbin/varnishd"] diff --git a/Makefile b/Makefile index bea2ca09..70978b4d 100644 --- a/Makefile +++ b/Makefile @@ -93,9 +93,12 @@ endif varnish-controller: fmt vet go build -o ${ROOT_DIR}bin/varnish-controller ${ROOT_DIR}cmd/varnish-controller/ +VARNISH_VERSION_NUMBER ?= 9.0.3-1 + # Build the docker image with varnishd itself and varnish modules docker-build-varnish: - docker build --platform ${PLATFORM} ${ROOT_DIR} -t ${VARNISH_IMG} -f Dockerfile.varnishd + docker build --platform ${PLATFORM} ${ROOT_DIR} -t ${VARNISH_IMG} -f Dockerfile.varnishd \ + --build-arg VARNISH_VERSION_NUMBER=${VARNISH_VERSION_NUMBER} docker-tag-push-varnish: ifndef PUBLISH @@ -108,7 +111,8 @@ endif # Build the docker image with varnish controller docker-build-varnish-controller: fmt vet - docker build --platform ${PLATFORM} ${ROOT_DIR} -t ${VARNISH_CONTROLLER_IMG} -f Dockerfile.controller + docker build --platform ${PLATFORM} ${ROOT_DIR} -t ${VARNISH_CONTROLLER_IMG} -f Dockerfile.controller \ + --build-arg VARNISH_VERSION_NUMBER=${VARNISH_VERSION_NUMBER} docker-tag-push-varnish-controller: ifndef PUBLISH @@ -123,6 +127,7 @@ endif PROMETHEUS_VARNISH_EXPORTER_VERSION ?= v1.8.3 docker-build-varnish-exporter: docker build --platform ${PLATFORM} ${ROOT_DIR} -t ${VARNISH_METRICS_IMG} -f Dockerfile.exporter \ + --build-arg VARNISH_VERSION_NUMBER=${VARNISH_VERSION_NUMBER} \ --build-arg PROMETHEUS_VARNISH_EXPORTER_VERSION=${PROMETHEUS_VARNISH_EXPORTER_VERSION} docker-tag-push-varnish-exporter: diff --git a/README.md b/README.md index 88e9efb1..3e6afd2c 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ The purpose of the project is to provide a convenient way to deploy and manage V Kubernetes version `>=1.29.0` is supported (see the operator bundle `minKubeVersion`). CI runs e2e against Kubernetes 1.34.3 and 1.35.1, and unit tests use envtest 1.36.0—see [docs/development.md](docs/development.md#kubernetes-versions-in-tests) for why those version numbers differ. -Varnish version 7.x is supported (container images ship Debian trixie packages, currently Varnish 7.7). +Varnish version 9.x is supported (container images ship Varnish 9.0.3 from [packages.varnish-software.com](https://packages.varnish-software.com/) on Debian trixie). Full documentation can be found [here](https://cin.github.io/varnish-operator/) @@ -34,4 +34,5 @@ The operator manages the whole lifecycle of the cluster: creating, deleting and ### Further reading * [QuickStart](https://cin.github.io/varnish-operator/quick-start.html) +* [Custom container images](https://cin.github.io/varnish-operator/custom-images.html) * [Contributing](https://cin.github.io/varnish-operator/development.html) diff --git a/api/v1alpha1/varnishcluster_types.go b/api/v1alpha1/varnishcluster_types.go index 9327db25..af5e698b 100644 --- a/api/v1alpha1/varnishcluster_types.go +++ b/api/v1alpha1/varnishcluster_types.go @@ -53,9 +53,9 @@ const ( VarnishSecretVolume = "secret" // VarnishWorkDir is the shared emptyDir mount for varnishd and varnishstat (must match -n). VarnishWorkDir = "/var/lib/varnish" - // VarnishRunAsUID and VarnishRunAsGID match the varnish system user from the Debian varnish package in our images. - VarnishRunAsUID = 997 - VarnishRunAsGID = 997 + // VarnishRunAsUID and VarnishRunAsGID match the varnish user from packages.varnish-software.com (Varnish 9 images). + VarnishRunAsUID = 1000 + VarnishRunAsGID = 1000 VarnishUpdateStrategyDelayedRollingUpdate = "DelayedRollingUpdate" diff --git a/config/samples/varnishcluster.yaml b/config/samples/varnishcluster.yaml index 6f9ba8f3..e45c0154 100644 --- a/config/samples/varnishcluster.yaml +++ b/config/samples/varnishcluster.yaml @@ -9,9 +9,11 @@ spec: # updateStrategy: # type: "OnDelete" #can be "OnDelete", "RollingUpdate" and "DelayedRollingUpdate" varnish: - # path to image + tag -# image: cinple/varnish:0.27.2 + # Optional: override images (tag = operator release, e.g. 0.38.0 or local—not Varnish 9.0.3). + # Sidecars default to varnish-controller: and varnish-metrics-exporter:. See docs/custom-images.md. +# image: cinple/varnish:0.38.0 # imagePullPolicy: Always +# imagePullSecret: regcred # Resources allocated to the Varnish pod through Kubernetes. It is strongly recommended that you specify resources, # since Varnish is an in-memory cache, and you do not want it restarting frequently. #resources: @@ -28,7 +30,7 @@ spec: # and defines the container's resources allocation. # controller: # # path to image + tag to override, by default it refers to varnish.image with "-controller" suffix image. -# image: cinple/varnish-controller:0.27.2 +# image: cinple/varnish-controller:0.38.0 # # imagePullPolicy controls how the varnish-controller image will be pulled for new containers # imagePullPolicy: Always # # Resources allocated to the Varnish controller container through Kubernetes. @@ -44,7 +46,7 @@ spec: # metricsExporter: # # path to image + tag to override, by default it refers to varnish.image with "-metrics-exporter" suffix image. -# image: cinple/varnish-metrics-exporter:0.27.2 +# image: cinple/varnish-metrics-exporter:0.38.0 # # imagePullPolicy controls how the varnish-metrics-exporter image will be pulled for new containers # imagePullPolicy: Always # # Resources allocated to the Varnish metrics exporter container through Kubernetes. diff --git a/docker/install-varnish-9.sh b/docker/install-varnish-9.sh new file mode 100644 index 00000000..249bfb4d --- /dev/null +++ b/docker/install-varnish-9.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# Install pinned Varnish 9.x from packages.varnish-software.com on Debian trixie. +# Usage: install-varnish-9.sh [minimal|tools] +# minimal — varnishd image: varnish + varnish-modules +# tools — controller/exporter: varnish CLI/libs only (no vmods) +set -ex + +MODE="${1:-minimal}" +VARNISH_VERSION_NUMBER="${VARNISH_VERSION_NUMBER:-9.0.3-1}" +REPO_FINGERPRINT="${REPO_FINGERPRINT:-694566269779DFAC975ED9BDD0525EAE838B3344}" + +. /etc/os-release +VARNISH_VERSION="${VARNISH_VERSION_NUMBER}~${VERSION_CODENAME}" + +export DEBIAN_FRONTEND=noninteractive +export DEBCONF_NONINTERACTIVE_SEEN=true + +apt-get update +apt-get install -y --no-install-recommends curl ca-certificates gnupg + +mkdir -p /etc/apt/keyrings +gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "${REPO_FINGERPRINT}" +gpg --batch --armor --export "${REPO_FINGERPRINT}" > /etc/apt/keyrings/varnish.gpg +echo "deb [signed-by=/etc/apt/keyrings/varnish.gpg] https://packages.varnish-software.com/varnish/${ID} ${VERSION_CODENAME} main" \ + > /etc/apt/sources.list.d/varnish.list + +apt-get update + +# Match official varnish/docker-varnish UID layout (not Debian stock 997). +adduser --uid 1000 --quiet --system --no-create-home --home /nonexistent --group varnish +adduser --uid 1001 --quiet --system --no-create-home --home /nonexistent --ingroup varnish vcache +adduser --uid 1002 --quiet --system --no-create-home --home /nonexistent --ingroup varnish varnishlog + +if [ "${MODE}" = "minimal" ]; then + PACKAGES="varnish=${VARNISH_VERSION} varnish-modules=${VARNISH_VERSION}" +else + PACKAGES="varnish=${VARNISH_VERSION}" +fi + +apt-get install -y --no-install-recommends ${PACKAGES} + +apt-mark hold varnish +rm -rf /var/lib/apt/lists/* /etc/varnish/* ~/.gnupg +mkdir -p /etc/varnish /var/lib/varnish +chown -R varnish:varnish /etc/varnish /var/lib/varnish +mkdir -p -m 1777 /var/lib/varnish/varnishd +chown varnish /var/lib/varnish/varnishd diff --git a/docs/README.md b/docs/README.md index b0e2e304..32dc7d47 100644 --- a/docs/README.md +++ b/docs/README.md @@ -40,7 +40,7 @@ spec: port: 80 ``` -See the [VarnishCluster configuration section](varnish-cluster-configuration.md) for more details about the `VarnishCluster` spec. +See the [VarnishCluster configuration section](varnish-cluster-configuration.md) for more details about the `VarnishCluster` spec. To use your own `varnish` / sidecar images, see [Custom container images](custom-images.md). ### VCL configuration @@ -59,6 +59,7 @@ See the [VCL files configuration](vcl-configuration.md) section for more details * [Quickstart](quick-start.md) * [VarnishCluster configuration](varnish-cluster-configuration.md) +* [Custom container images](custom-images.md) * [Varnish operator configuration](operator-configuration.md) * [VCL files configuration](vcl-configuration.md) * [Contribution](development.md) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 210b6967..eeb28002 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -6,6 +6,7 @@ * [Operator Configuration](operator-configuration.md) * [VarnishCluster](varnish-cluster.md) * [VarnishCluster Configuration](varnish-cluster-configuration.md) +* [Custom container images](custom-images.md) * [VCL Configuration](vcl-configuration.md) * [Monitoring](monitoring.md) * [Debugging Issues](debugging-issues.md) diff --git a/docs/architecture.md b/docs/architecture.md index 0094993f..a6aa49c0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -22,7 +22,7 @@ The containers share specific volumes for the varnish configuration and work dir ##### Varnish -The Varnish process itself. Varnish **7.x** is supported (images use Debian trixie packages, currently **7.7**). The operator doesn't support arbitrary Varnish images due to additional components needed for the operator to function. The container image is custom built with `varnish` and `varnish-modules` preinstalled. +The Varnish process itself. Varnish **9.x** is supported (default images install **9.0.3** from [packages.varnish-software.com](https://packages.varnish-software.com/) on Debian trixie, plus `varnish-modules`). You may set `spec.varnish.image` (and sidecar images) to your own registry, but all three pod images must stay compatible with the operator’s layout—see [Custom container images](custom-images.md). ##### Varnish-Controller diff --git a/docs/custom-images.md b/docs/custom-images.md new file mode 100644 index 00000000..144780dd --- /dev/null +++ b/docs/custom-images.md @@ -0,0 +1,147 @@ +# Custom container images + +Each `VarnishCluster` pod runs three containers (`varnish`, `varnish-controller`, `varnish-metrics-exporter`). You can point them at your own images via the CR spec. The operator does **not** validate image contents; images must satisfy the [compatibility requirements](#compatibility-requirements-for-custom-images) below or pods will fail readiness checks, VCL reloads, or metrics scraping. + +### Image tags vs Varnish version + +**Container image tags follow your operator release** (Helm `container.tag`, CI `VERSION`, e.g. `0.38.0` or `local`)—not the upstream Varnish semver. All three images for a given operator release are built and published together with that tag. + +The **Varnish daemon version** inside the `varnish` image (today **9.0.3** via `VARNISH_VERSION_NUMBER` at build time) is separate from the OCI tag. The controller and metrics exporter images are operator components; tag them like `varnish-controller:0.38.0`, not `varnish-metrics-exporter:9.0.3`. + +## Specifying images on `VarnishCluster` + +Set `spec.varnish.image` for the `varnishd` container. Sidecar images default from that name unless overridden explicitly. + +```yaml +spec: + varnish: + image: registry.example.com/acme/varnish:0.38.0 + imagePullPolicy: IfNotPresent + imagePullSecret: regcred # optional, applies to all containers in the pod + controller: + image: registry.example.com/acme/varnish-controller:0.38.0 + imagePullPolicy: IfNotPresent + metricsExporter: + image: registry.example.com/acme/varnish-metrics-exporter:0.38.0 + imagePullPolicy: IfNotPresent +``` + +| Field | Purpose | +| ----- | ------- | +| `spec.varnish.image` | `varnishd` container image (repository:tag). | +| `spec.varnish.imagePullPolicy` | Pull policy for the `varnish` container (default: `Always`). | +| `spec.varnish.imagePullSecret` | Secret used to pull images for the StatefulSet pod. | +| `spec.varnish.controller.image` | Controller sidecar; if empty, derived from `varnish.image` (see below). | +| `spec.varnish.metricsExporter.image` | Metrics sidecar; if empty, derived from `varnish.image` (see below). | + +### Default image names when fields are omitted + +If `spec.varnish.image` is **empty**, the operator uses a **coupled default** derived from the operator Deployment image (`CONTAINER_IMAGE` / Helm `container.image`): + +| Operator image | Default `varnish` image | Default controller | Default metrics exporter | +| -------------- | ----------------------- | ------------------ | ------------------------ | +| `registry.io/team/varnish-operator:1.2.0` | `registry.io/team/varnish:1.2.0` | `registry.io/team/varnish-controller:1.2.0` | `registry.io/team/varnish-metrics-exporter:1.2.0` | + +The repository path is taken from the operator image; the image name is replaced with `varnish`, and the **same tag** is reused. Controller and exporter names append `-controller` and `-metrics-exporter` to the repository name before the tag. + +If you set only `spec.varnish.image`, sidecars use that naming rule automatically: + +```yaml +spec: + varnish: + image: myregistry/varnish:0.38.0 + # controller -> myregistry/varnish-controller:0.38.0 + # metrics -> myregistry/varnish-metrics-exporter:0.38.0 +``` + +Override individual sidecars when your registry naming differs (keep the **same operator release tag** on all three unless you know the builds are paired): + +```yaml +spec: + varnish: + image: myregistry/varnish:0.38.0 + controller: + image: myregistry/varnish-ctrl:0.38.0 +``` + +After changing an image tag in a running cluster, set `spec.statefulSet.container.imagePullPolicy: Always` (on the embedded pod template, if you customize it) or delete pods so nodes pull new layers. + +## Compatibility requirements for custom images + +All three images must target the **same Varnish major/minor** (today: **9.0.3** from [packages.varnish-software.com](https://packages.varnish-software.com/) in this repository’s Dockerfiles). Mixing `varnishd` 9.x with sidecars built against 7.x will break `varnishadm`, `varnishstat`, and the Prometheus exporter. + +### `varnish` (`varnishd`) image + +| Requirement | Detail | +| ----------- | ------ | +| **Process** | `varnishd` on `PATH` or `/usr/sbin/varnishd` (StatefulSet does not override the image entrypoint). | +| **Vmods** | If you use the operator’s default VCL (`import var;`, `import directors;` in templates), install **`varnish-modules`** for your Varnish version. | +| **User / UID** | Run as non-root user **`varnish` with UID/GID 1000** (Varnish Software packages). The operator sets `runAsUser` / `runAsGroup` / `fsGroup` to **1000**. | +| **Directories** | Writable `/var/lib/varnish` (instance dir `-n`) and `/etc/varnish` (VCL mount). | +| **Args** | The operator injects `-F`, `-n /var/lib/varnish`, `-S /etc/varnish-secret/secret`, `-T 0.0.0.0:6082`, `-b 127.0.0.1:0`, `-a 0.0.0.0:`. Do not rely on overriding `-n`, `-f`, `-S`, `-T`, or `-b` via `spec.varnish.args`. | + +### `varnish-controller` image + +| Requirement | Detail | +| ----------- | ------ | +| **Binary** | `/varnish-controller` as entrypoint (built from this repo’s `cmd/varnish-controller`). | +| **CLI tools** | `varnishadm` and `varnishstat` compatible with the `varnishd` version. | +| **Libraries** | Matching `libvarnishapi` (e.g. `libvarnishapi.so.3` on 9.x). | +| **User** | UID **1000** (`varnish`), with read access to the shared workdir and secret volume. | + +### `varnish-metrics-exporter` image + +| Requirement | Detail | +| ----------- | ------ | +| **Binary** | `prometheus-varnish-exporter` (this repo builds [otto-de/prometheus_varnish_exporter](https://github.com/otto-de/prometheus_varnish_exporter) v1.8.3+). | +| **CLI tools** | `varnishstat` + matching `libvarnishapi`. | +| **User** | UID **1000** (`varnish`). | +| **Args** | Operator passes **`-n /var/lib/varnish`**; the image must support that flag. | + +### VCL and configuration (not in the image, but required) + +Custom images do not remove the need for compatible **VCL** in the ConfigMap (`spec.vcl`). Review [Varnish 9 upgrade notes](https://varnish-cache.org/docs/9.0/whats-new/upgrading-9.0.html) when moving from older versions. + +## What you cannot change via image alone + +These are fixed in the operator today; custom images must adapt to them (or you fork the operator): + +- **Security context**: `runAsUser` / `runAsGroup` / `fsGroup` **1000** on Varnish pods. +- **Volume layout**: shared `emptyDir` at `/var/lib/varnish`, VCL under `/etc/varnish`, admin secret at `/etc/varnish-secret/secret`. +- **Ports**: Varnish HTTP from `spec.service.port`, admin CLI on **6082**, metrics on **9131** (defaults). +- **Readiness probe**: `varnishadm -S … -T 127.0.0.1:6082 ping`. + +## Building images from this repository + +Reference Dockerfiles (Debian trixie + Varnish Software packages): + +| Image | Dockerfile | Install script | +| ----- | ---------- | -------------- | +| `varnish` | `Dockerfile.varnishd` | `docker/install-varnish-9.sh` (`minimal`: `varnish` + `varnish-modules`) | +| `varnish-controller` | `Dockerfile.controller` | `install-varnish-9.sh` (`tools`: CLI + libs) + Go binary | +| `varnish-metrics-exporter` | `Dockerfile.exporter` | `tools` + exporter build | + +```bash +make docker-build-pod REPO=myregistry VERSION=0.38.0 +# myregistry/varnish:0.38.0 +# myregistry/varnish-controller:0.38.0 +# myregistry/varnish-metrics-exporter:0.38.0 +``` + +Pin a different **Varnish package** inside the `varnish` image (does not change the OCI tag): + +```bash +make docker-build-varnish VARNISH_VERSION_NUMBER=9.0.3-1 REPO=myregistry VERSION=0.38.0 +``` + +Build args: `VERSION` (image tag / operator release), `VARNISH_VERSION_NUMBER` (Debian package pin, default `9.0.3-1`), `PROMETHEUS_VARNISH_EXPORTER_VERSION` (default `v1.8.3`). + +## Using upstream `library/varnish` images + +The official [Docker Hub `varnish` image](https://hub.docker.com/_/varnish) also uses UID **1000** and Varnish Software packages, but it is **not** tested as a drop-in for this operator: entrypoint scripts, bundled VCL, and optional vmods differ from the images built here. Prefer the Dockerfiles in this repo, or replicate their layout (users, paths, `varnishadm`/`varnishstat`, `varnish-modules`) and run `make e2e-tests` before production use. + +## Related documentation + +- [VarnishCluster configuration](varnish-cluster-configuration.md) — full spec field list +- [Development / local images](development.md) — kind, `make e2e-tests`, local tags +- [Architecture](architecture.md) — pod layout and component roles diff --git a/docs/development.md b/docs/development.md index 944c36f4..0caa43d0 100644 --- a/docs/development.md +++ b/docs/development.md @@ -127,7 +127,7 @@ This produces: | `cinple/varnish-controller:local` | `Dockerfile.controller` | | `cinple/varnish-metrics-exporter:local` | `Dockerfile.exporter` | -Override images in your `VarnishCluster`: +Override images in your `VarnishCluster` (see [Custom container images](custom-images.md) for defaults, naming rules, and what custom images must provide): ```yaml spec: @@ -141,9 +141,13 @@ spec: If you reuse the same tag, set `spec.statefulSet.container.imagePullPolicy: Always` and restart pods (or delete them) so Kubernetes pulls the new layers. -Varnish pod images (`varnish`, `varnish-controller`, `varnish-metrics-exporter`) are based on **Debian trixie** and ship **Varnish 7.x** from Debian packages. Rebuild all three together when upgrading Varnish. +Varnish pod images (`varnish`, `varnish-controller`, `varnish-metrics-exporter`) are based on **Debian trixie** and ship **Varnish 9.0.3** from [packages.varnish-software.com](https://packages.varnish-software.com/) (see `docker/install-varnish-9.sh`). Rebuild all three together when upgrading Varnish. Override the pin with build-arg `VARNISH_VERSION_NUMBER` (default `9.0.3-1`). -Those images run as the Debian **`varnish` user (UID/GID 997)**, not root. The StatefulSet sets `runAsNonRoot`, `runAsUser`/`runAsGroup` 997, drops capabilities, and uses `fsGroup` 997 on shared volumes so sidecars can read the Varnish workdir without a custom root user. +Those images run as the **`varnish` user (UID/GID 1000)** from the Varnish Software packages, not root. The StatefulSet sets `runAsNonRoot`, `runAsUser`/`runAsGroup` 1000, drops capabilities, and uses `fsGroup` 1000 on shared volumes so sidecars can read the Varnish workdir. + +```bash +docker build --build-arg VARNISH_VERSION_NUMBER=9.0.3-1 -f Dockerfile.varnishd . +``` The metrics exporter image accepts `PROMETHEUS_VARNISH_EXPORTER_VERSION` and `PROMETHEUS_VARNISH_EXPORTER_REPO` as build arguments (defaults: `v1.8.3` from [otto-de/prometheus_varnish_exporter](https://github.com/otto-de/prometheus_varnish_exporter)): @@ -152,7 +156,7 @@ docker build --build-arg PROMETHEUS_VARNISH_EXPORTER_VERSION=v1.8.3 \ -t my-exporter:local -f Dockerfile.exporter . ``` -When upgrading from Varnish 6 images, review custom VCL for [Varnish 7 changes](https://varnish-cache.org/docs/7.0/whats-new/upgrading-7.0.html) (notably PCRE2 regex behavior). +When upgrading from older Varnish images, review custom VCL for [Varnish 7](https://varnish-cache.org/docs/7.0/whats-new/upgrading-7.0.html) and [Varnish 9](https://varnish-cache.org/docs/9.0/whats-new/upgrading-9.0.html) release notes (PCRE2, removed APIs, etc.). Expect a cold cache after rollout; the default workdir is `emptyDir`. ## Code generation and manifests diff --git a/docs/operator-configuration.md b/docs/operator-configuration.md index 528c70f5..43535ab2 100644 --- a/docs/operator-configuration.md +++ b/docs/operator-configuration.md @@ -2,6 +2,8 @@ As the operator is packaged into [the Helm chart](https://helm.sh/docs/developing_charts/) the configuration is done by setting `values.yaml` overrides. +When a `VarnishCluster` does not set `spec.varnish.image`, workload images are derived from the operator image (`container.registry` / `container.repository` / `container.tag`)—for example `…/varnish:`, `…/varnish-controller:`, and `…/varnish-metrics-exporter:`. To override per cluster or run private builds, see [Custom container images](custom-images.md). + | Field | Description | Default | | ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------- | | `affinity` | [Affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) settings for operator pods | `optional` | diff --git a/docs/varnish-cluster-configuration.md b/docs/varnish-cluster-configuration.md index 0481f08a..194959f0 100644 --- a/docs/varnish-cluster-configuration.md +++ b/docs/varnish-cluster-configuration.md @@ -1,11 +1,17 @@ # VarnishCluster Configuration +## Container images + +Override the three workload images (`varnish`, `varnish-controller`, `varnish-metrics-exporter`) with `spec.varnish.image` and optional `spec.varnish.controller.image` / `spec.varnish.metricsExporter.image`. Image **tags match the operator release** (not the Varnish daemon version). If `varnish.image` is omitted, names are derived from the operator Deployment image (same registry/tag, with `varnish`, `varnish-controller`, and `varnish-metrics-exporter` image names). + +**Custom images must match operator expectations** (UID 1000, shared `/var/lib/varnish`, matching `varnishadm`/`varnishstat`/Varnish version, etc.). See **[Custom container images](custom-images.md)** for examples, defaults, and a full compatibility checklist. + | Field | Description | Is Required | | ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------- | | `affinity ` | [Affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) settings for the pods. It allows you to configure onto which nodes Varnish pods should prefer being scheduled. | `optional` | | `priorityClassName ` | [priorityClass](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass) settings for the pods. It allows you to set a PriorityClassName and thus set a priority to your pods, to avoid eviction. | `optional` | | `backend.namespaces ` | Namespace(s) to look for backend pods. By default - namespace the VarnishCluster is deployed to. | `required` | -| `backend.onlyReady ` | Include (`false`, by default) or exclude (`true`) backend pods from the VCL (.Backends template var). Alters `.Backends` template variable based on Kubernetes health checks (by default not ready pods are also included in VCL) instead of [Varnish health probes](https://varnish-cache.org/docs/7.7/reference/vcl-probe.html#backend-health-probes). | `optional` | +| `backend.onlyReady ` | Include (`false`, by default) or exclude (`true`) backend pods from the VCL (.Backends template var). Alters `.Backends` template variable based on Kubernetes health checks (by default not ready pods are also included in VCL) instead of [Varnish health probes](https://varnish-cache.org/docs/9.0/reference/vcl-probe.html#backend-health-probes). | `optional` | | `backend.port ` | The port of the backend pods being cached by Varnish. Can be port name or port number. | `required` | | `backend.selector ` | The selector used to identify the backend Pods. | `required` | | `backend.zoneBalancing ` | Controls Varnish backend topology aware routing which can assign weights to backends according to their geographical location. | `optional` | @@ -51,7 +57,7 @@ | `varnish.admAuth.key ` | The key from kubernetes secret which to use to collect data credentials for `varnishadm`. If the key is omitted, the cluster will use "secret" as the key. If the value associated to the key is empty, the cluster will generate a secret. | `optional` | | `varnish.args ` | Additional [Varnish daemon arguments](https://varnish-cache.org/docs/trunk/reference/varnishd.html#options) | `optional` | | `varnish.controller ` | An object that defines the configuration of a particular Varnish controller being deployed | `optional` | -| `varnish.controller.image ` | Path to the Varnish Controller image being used. If not defined uses `varnish.image`+`-controller` suffix. Something like `varnish-controller` | `optional` | +| `varnish.controller.image ` | Controller sidecar image. Empty → `-controller:` (see [custom-images.md](custom-images.md)) | `optional` | | `varnish.controller.imagePullPolicy ` | Image pull policy for the container. Default: `Always` | `optional` | | `varnish.controller.resources ` | [Resource requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) for Varnish controller container. | `optional` | | `varnish.envFrom ` | Injects an env var into the Varnish container from a ConfigMap or Secret. Useful if a value needs to be passed (securely in case of Secret) to the VCL files. So it can be read using [std.getenv()](https://varnish-cache.org/docs/trunk/reference/vmod_std.html#string-getenv-string-name). | `optional` | @@ -71,11 +77,11 @@ | `varnish.extraVolumeClaimTemplates[].spec ` | Spec for the [PersistentVolumeClaim](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#persistentvolumeclaimspec-v1-core) | `optional` | | `varnish.extraVolumeMounts ` | Additional [volume mounts](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#volumemount-v1-core) for the Varnish container | `optional` | | `varnish.extraVolumes ` | Additional [volumes](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#volume-v1-core) | `optional` | -| `varnish.image ` | Path to the Varnish image being used | `optional` | +| `varnish.image ` | `varnishd` container image (`repository:tag`). Empty → coupled default from operator image (see [custom-images.md](custom-images.md)) | `optional` | | `varnish.imagePullPolicy ` | Image pull policy for the Varnish container. Default: `Always` | `optional` | | `varnish.imagePullSecret ` | The name of the image pull secret to use to pull container images | `optional` | | `varnish.metricsExporter ` | An object that defines the configuration of a particular Varnish Prometheus metrics exporter being deployed | `optional` | -| `varnish.metricsExporter.image ` | Path to the Varnish Metrics exporter image being used. If not defined uses `varnish.image`+`-metrics-exporter` suffix. Something like `varnish-metrics-exporter` | `optional` | +| `varnish.metricsExporter.image ` | Metrics exporter image. Empty → `-metrics-exporter:` (see [custom-images.md](custom-images.md)) | `optional` | | `varnish.metricsExporter.imagePullPolicy ` | Image pull policy for the container. Default: `Always` | `optional` | | `varnish.metricsExporter.resources ` | [Resource requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) for Varnish metrics exporter container. | `optional` | | `varnish.resources ` | [Resource requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) for Varnish container. | `optional` | diff --git a/docs/vcl-configuration.md b/docs/vcl-configuration.md index 3f4b7444..08016ed7 100644 --- a/docs/vcl-configuration.md +++ b/docs/vcl-configuration.md @@ -125,7 +125,7 @@ The `.Backends` array can include both all backend nodes (default behavior) and It depends on which readiness checks you want to respect: * Kubernetes readiness checks - field set to `true`. In this case the `.Backends` template var in VCL will include only ready backend pods. Keep in mind that every backend state change will trigger a Varnish VCL config reload. -* Varnish readiness probes - field set to `false` or omitted. This way the `.Backends` array will include all scheduled pods, so it is strongly recommended to set [health checks](https://varnish-cache.org/docs/7.7/reference/vcl-probe.html#backend-health-probes) in your VCL configuration. Otherwise, Varnish could send traffic to not yet ready backends. +* Varnish readiness probes - field set to `false` or omitted. This way the `.Backends` array will include all scheduled pods, so it is strongly recommended to set [health checks](https://varnish-cache.org/docs/9.0/reference/vcl-probe.html#backend-health-probes) in your VCL configuration. Otherwise, Varnish could send traffic to not yet ready backends. You can also combine approaches and use both Kubernetes and Varnish health probes. From 52abdf3bd4d7ad9b36f53f8993e3b5b2facc229f Mon Sep 17 00:00:00 2001 From: Craig Ingram Date: Mon, 25 May 2026 17:37:30 -0400 Subject: [PATCH 2/3] Added test around default VCL to ensure it works with Varnish 9 --- docs/development.md | 17 ++ hack/create_dev_cluster.sh | 80 +++++++- hack/delete_dev_cluster.sh | 8 +- .../varnishcluster_configmap_test.go | 4 + tests/default_vcl_test.go | 184 ++++++++++++++++++ 5 files changed, 284 insertions(+), 9 deletions(-) create mode 100644 tests/default_vcl_test.go diff --git a/docs/development.md b/docs/development.md index 0caa43d0..b4d27051 100644 --- a/docs/development.md +++ b/docs/development.md @@ -158,6 +158,23 @@ docker build --build-arg PROMETHEUS_VARNISH_EXPORTER_VERSION=v1.8.3 \ When upgrading from older Varnish images, review custom VCL for [Varnish 7](https://varnish-cache.org/docs/7.0/whats-new/upgrading-7.0.html) and [Varnish 9](https://varnish-cache.org/docs/9.0/whats-new/upgrading-9.0.html) release notes (PCRE2, removed APIs, etc.). Expect a cold cache after rollout; the default workdir is `emptyDir`. +### Default VCL on a dev cluster + +`./hack/create_dev_cluster.sh -v` creates a `VarnishCluster` that references `vcl-config` / `entrypoint.vcl`. If that ConfigMap does not exist, the operator seeds the [default VCL](../pkg/varnishcluster/controller/varnishcluster_default_vcl.go) (`import var`, `directors` round-robin backends, `/heartbeat`, `/liveness`, `X-Varnish-Cache`). + +Manual smoke test (after `-b` and `-v`): + +```bash +export KUBECONFIG=./e2e-tests-kubeconfig +kubectl port-forward -n varnish-cluster svc/varnishcluster-example 8080:80 + +curl -s -o /dev/null -w "%{http_code}\n" http://127.0.0.1:8080/heartbeat # 200 +curl -s -o /dev/null -w "%{http_code}\n" http://127.0.0.1:8080/liveness # 200 +curl -sI http://127.0.0.1:8080/ | grep -i x-varnish-cache # MISS then HIT +``` + +Automated coverage: `go test ./tests -ginkgo.focus="operator default VCL"` (requires `make e2e-tests` or an equivalent cluster). + ## Code generation and manifests ```bash diff --git a/hack/create_dev_cluster.sh b/hack/create_dev_cluster.sh index 02fd0df6..d440807b 100755 --- a/hack/create_dev_cluster.sh +++ b/hack/create_dev_cluster.sh @@ -42,6 +42,41 @@ if ! which helm >/dev/null; then exit 1 fi +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +KUBECONFIG_FILE="${ROOT_DIR}/e2e-tests-kubeconfig" + +function e2e_kubeconfig_valid() { + kubectl config current-context --kubeconfig="${KUBECONFIG_FILE}" >/dev/null 2>&1 +} + +function refresh_e2e_kubeconfig_from_kind() { + if kind get clusters 2>/dev/null | grep -qx "${cluster_name}"; then + kind get kubeconfig --name "${cluster_name}" > "${KUBECONFIG_FILE}" + return 0 + fi + return 1 +} + +function use_e2e_kubeconfig() { + if [[ -f "${KUBECONFIG_FILE}" ]] && ! e2e_kubeconfig_valid; then + echo "warning: ${KUBECONFIG_FILE} is empty or stale; refreshing from kind cluster ${cluster_name}..." >&2 + rm -f "${KUBECONFIG_FILE}" + fi + if [[ ! -f "${KUBECONFIG_FILE}" ]]; then + if ! refresh_e2e_kubeconfig_from_kind; then + echo "error: no valid kubeconfig at ${KUBECONFIG_FILE} and kind cluster '${cluster_name}' is not running." >&2 + echo "Run: ${SCRIPT_DIR}/create_dev_cluster.sh # without -v or -b" >&2 + exit 1 + fi + fi + if ! e2e_kubeconfig_valid; then + echo "error: ${KUBECONFIG_FILE} is not a usable kubeconfig." >&2 + exit 1 + fi + export KUBECONFIG="${KUBECONFIG_FILE}" +} + varnish_namespace="varnish-operator" cluster_name="e2e-tests" repo="cinple" @@ -66,14 +101,15 @@ Creates a dev cluster and varnish-operator install -n|--namespace | namespace -p|--platform | platform (not validated so know which build you're calling) -r|--repo | CR repository --b|--backends | create backends +-b|--backends | create nginx backends (requires cluster; run script without -v/-b first) -s|--skip-docker-build | skip docker build --v|--create-varnishcluster | create varnish cluster +-v|--create-varnishcluster | create sample VarnishCluster (requires cluster; run script without -v/-b first) -x|--ignore-podman | ignore podman's presence ! } function default_vc_namespace { + use_e2e_kubeconfig if [[ "$varnish_namespace" == "varnish-operator" ]]; then if [ "$(kubectl get namespace --no-headers | grep varnish-cluster | wc -l | xargs echo -n)" -eq 0 ]; then kubectl create namespace varnish-cluster @@ -82,6 +118,21 @@ function default_vc_namespace { fi } +function load_local_images_into_kind() { + use_e2e_kubeconfig + if ! kind get clusters 2>/dev/null | grep -qx "${cluster_name}"; then + echo "error: kind cluster '${cluster_name}' not found." >&2 + exit 1 + fi + for image in "${workload_images[@]}"; do + if ! docker image inspect "${image}" >/dev/null 2>&1; then + echo "error: local image ${image} not found. Run ${SCRIPT_DIR}/create_dev_cluster.sh (without -v/-b) to build images first." >&2 + exit 1 + fi + kind load docker-image --name "${cluster_name}" "${image}" + done +} + function create_nginx_backends { if [ "$dry_run" = true ]; then echo "dry-run: would otherwise be installing nginx" @@ -98,6 +149,7 @@ function create_varnishcluster { fi default_vc_namespace + load_local_images_into_kind cat < /dev/null 2>&1 - kind create cluster --name $cluster_name --image "${kind_node_image}" --kubeconfig ./e2e-tests-kubeconfig - export KUBECONFIG=./e2e-tests-kubeconfig + kind create cluster --name $cluster_name --image "${kind_node_image}" --kubeconfig "${KUBECONFIG_FILE}" fi +use_e2e_kubeconfig + if [ "$(kubectl get namespace --no-headers | grep varnish-operator | wc -l | xargs echo -n)" -eq 0 ]; then kubectl create ns $varnish_namespace fi @@ -202,7 +268,7 @@ if [ "$skip_docker_build" = false ]; then fi for image in "${images[@]}"; do - kind load docker-image -n $cluster_name $image + kind load docker-image --name "${cluster_name}" "${image}" done helm install varnish-operator varnish-operator --namespace=$varnish_namespace --wait --set container.imagePullPolicy=Never --set container.image=$container_image diff --git a/hack/delete_dev_cluster.sh b/hack/delete_dev_cluster.sh index 101933aa..d46e2241 100755 --- a/hack/delete_dev_cluster.sh +++ b/hack/delete_dev_cluster.sh @@ -1,5 +1,9 @@ #!/bin/bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +KUBECONFIG_FILE="${ROOT_DIR}/e2e-tests-kubeconfig" + cluster_name="e2e-tests" function usage { @@ -22,6 +26,6 @@ done kind delete cluster --name $cluster_name > /dev/null 2>&1 -if [[ -f ./e2e-tests-kubeconfig ]]; then - rm -f ./e2e-tests-kubeconfig +if [[ -f "${KUBECONFIG_FILE}" ]]; then + rm -f "${KUBECONFIG_FILE}" fi diff --git a/pkg/varnishcluster/controller/varnishcluster_configmap_test.go b/pkg/varnishcluster/controller/varnishcluster_configmap_test.go index 7f890af2..13135b38 100644 --- a/pkg/varnishcluster/controller/varnishcluster_configmap_test.go +++ b/pkg/varnishcluster/controller/varnishcluster_configmap_test.go @@ -63,6 +63,10 @@ var _ = Describe("the ConfigMap", func() { cmLabels := vclabels.CombinedComponentLabels(newVC, vcapi.VarnishComponentVCLFileConfigMap) Expect(cm.Labels).To(Equal(cmLabels)) + Expect(cm.Data).To(HaveKey("entrypoint.vcl")) + Expect(cm.Data).To(HaveKey("backends.vcl.tmpl")) + Expect(cm.Data["entrypoint.vcl"]).To(ContainSubstring("return (ok);")) + Expect(cm.Data["entrypoint.vcl"]).To(ContainSubstring("return (hash);")) }) }) diff --git a/tests/default_vcl_test.go b/tests/default_vcl_test.go new file mode 100644 index 00000000..090f3c8a --- /dev/null +++ b/tests/default_vcl_test.go @@ -0,0 +1,184 @@ +package tests + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "time" + + vcapi "github.com/cin/varnish-operator/api/v1alpha1" + + "github.com/gogo/protobuf/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Exercises the operator-seeded ConfigMap (entrypoint.vcl + backends.vcl.tmpl), not a user-supplied VCL. +var _ = Describe("operator default VCL", func() { + vcNamespace := "default" + vcName := "default-vcl-test" + configMapName := "e2e-operator-default-vcl" + objMeta := metav1.ObjectMeta{ + Namespace: vcNamespace, + Name: vcName, + } + backendResponse := "DEFAULT-VCL-BACKEND" + backendLabels := map[string]string{"app": "default-vcl-backend"} + backendDeploymentName := "default-vcl-backend" + varnishPodLabels := map[string]string{ + vcapi.LabelVarnishOwner: vcName, + vcapi.LabelVarnishComponent: vcapi.VarnishComponentVarnish, + } + + backendsDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: backendDeploymentName, + Namespace: vcNamespace, + Labels: backendLabels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: proto.Int32(1), + Selector: &metav1.LabelSelector{ + MatchLabels: backendLabels, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: backendLabels, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "backend", + Image: "hashicorp/http-echo", + Ports: []v1.ContainerPort{ + { + Name: "web", + Protocol: v1.ProtocolTCP, + ContainerPort: 5678, + }, + }, + Args: []string{fmt.Sprintf("-text=%s", backendResponse)}, + }, + }, + }, + }, + }, + } + + backendPort := intstr.FromInt(5678) + vc := &vcapi.VarnishCluster{ + ObjectMeta: objMeta, + Spec: vcapi.VarnishClusterSpec{ + Backend: &vcapi.VarnishClusterBackend{ + Selector: backendLabels, + Port: &backendPort, + }, + Service: &vcapi.VarnishClusterService{ + Port: proto.Int32(9090), + }, + Varnish: &vcapi.VarnishClusterVarnish{ + ImagePullPolicy: v1.PullNever, + Controller: &vcapi.VarnishClusterVarnishController{ + ImagePullPolicy: v1.PullNever, + }, + MetricsExporter: &vcapi.VarnishClusterVarnishMetricsExporter{ + ImagePullPolicy: v1.PullNever, + }, + }, + VCL: &vcapi.VarnishClusterVCL{ + ConfigMapName: proto.String(configMapName), + EntrypointFileName: proto.String("entrypoint.vcl"), + }, + }, + } + + AfterEach(func() { + By("deleting created resources") + Expect(k8sClient.DeleteAllOf(context.Background(), &vcapi.VarnishCluster{}, client.InNamespace(vcNamespace))).To(Succeed()) + Expect(k8sClient.DeleteAllOf(context.Background(), &appsv1.Deployment{}, client.InNamespace(vcNamespace), client.MatchingLabels(backendLabels))).To(Succeed()) + Expect(k8sClient.DeleteAllOf(context.Background(), &v1.ConfigMap{}, client.InNamespace(vcNamespace), client.MatchingLabels(map[string]string{ + vcapi.LabelVarnishOwner: vcName, + }))).To(Succeed()) + waitForPodsTermination(vcNamespace, varnishPodLabels) + waitForPodsTermination(vcNamespace, backendLabels) + waitUntilVarnishClusterRemoved(vcName, vcNamespace) + }) + + It("seeds ConfigMap VCL and serves traffic on Varnish 9", func() { + Expect(k8sClient.Create(context.Background(), backendsDeployment)).To(Succeed()) + Expect(k8sClient.Create(context.Background(), vc)).To(Succeed()) + + cm := &v1.ConfigMap{} + Eventually(func() error { + return k8sClient.Get(context.Background(), types.NamespacedName{ + Name: configMapName, Namespace: vcNamespace, + }, cm) + }, time.Minute, time.Second*2).Should(Succeed()) + + By("operator created default VCL files") + Expect(cm.Data).To(HaveKey("entrypoint.vcl")) + Expect(cm.Data).To(HaveKey("backends.vcl.tmpl")) + entrypoint := cm.Data["entrypoint.vcl"] + Expect(entrypoint).To(ContainSubstring("return (ok);")) + Expect(entrypoint).To(ContainSubstring("return (hash);")) + Expect(entrypoint).To(ContainSubstring("import var;")) + Expect(entrypoint).To(ContainSubstring(`include "backends.vcl"`)) + Expect(cm.Data["backends.vcl.tmpl"]).To(ContainSubstring("import directors;")) + + By("backend pods become ready") + waitForPodsReadiness(vcNamespace, backendLabels) + By("varnish pods become ready") + waitForPodsReadiness(vcNamespace, varnishPodLabels) + + pf := portForwardPod(vcNamespace, varnishPodLabels, []string{"6081:6081"}) + defer pf.Close() + + By("default /heartbeat endpoint") + Eventually(func() (int, error) { + resp, err := http.Get("http://localhost:6081/heartbeat") + if err != nil { + return 0, err + } + defer func() { _ = resp.Body.Close() }() + return resp.StatusCode, nil + }, time.Minute, time.Second*2).Should(Equal(200)) + + By("default /liveness endpoint with healthy backend") + Eventually(func() (int, error) { + resp, err := http.Get("http://localhost:6081/liveness") + if err != nil { + return 0, err + } + defer func() { _ = resp.Body.Close() }() + return resp.StatusCode, nil + }, time.Minute, time.Second*2).Should(Equal(200)) + + By("cached backend response with X-Varnish-Cache") + var resp *http.Response + Eventually(func() (int, error) { + var err error + resp, err = http.Get("http://localhost:6081/cached-path") + if err != nil { + return 0, err + } + return resp.StatusCode, nil + }, time.Second*30, time.Second*2).Should(Equal(200)) + Expect(resp.Header.Get("X-Varnish-Cache")).To(Equal("MISS")) + body, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(strings.TrimSpace(string(body))).To(Equal(backendResponse)) + + resp, err = http.Get("http://localhost:6081/cached-path") + Expect(err).NotTo(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(200)) + Expect(resp.Header.Get("X-Varnish-Cache")).To(Equal("HIT")) + }) +}) From 8707287dc3118b35646dbac8e7dd9628e051e008 Mon Sep 17 00:00:00 2001 From: Craig Ingram Date: Mon, 25 May 2026 19:24:25 -0400 Subject: [PATCH 3/3] Fixing unit test flakiness (sp) --- pkg/varnishcluster/controller/suite_test.go | 12 ++++++++++++ .../controller/varnishcluster_configmap_test.go | 7 ++++--- .../controller/varnishcluster_secret_test.go | 8 +++++++- .../controller/varnishcluster_statefulset.go | 12 ++++++------ 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/pkg/varnishcluster/controller/suite_test.go b/pkg/varnishcluster/controller/suite_test.go index 0bcbef3a..abd5805a 100644 --- a/pkg/varnishcluster/controller/suite_test.go +++ b/pkg/varnishcluster/controller/suite_test.go @@ -166,6 +166,13 @@ func StartTestManager(mgr manager.Manager) chan struct{} { return stop } +func waitUntilSecretRemoved(name, namespace string) { + Eventually(func() bool { + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, &v1.Secret{}) + return errors.IsNotFound(err) + }, time.Second*5).Should(BeTrue()) +} + // As the test control plane doesn't support garbage collection, this function is used to clean up resources // Designed to not fail if the resource is not found func CleanUpCreatedResources(vcName, vcNamespace string) { @@ -228,7 +235,12 @@ func CleanUpCreatedResources(vcName, vcNamespace string) { Expect(err).To(BeNil()) err = k8sClient.DeleteAllOf(context.Background(), &apps.StatefulSet{}, client.InNamespace(vcNamespace)) Expect(err).To(BeNil()) + _ = k8sClient.Delete(context.Background(), &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: names.VarnishSecret(vcName), Namespace: vcNamespace}, + }) + waitUntilSecretRemoved(names.VarnishSecret(vcName), vcNamespace) err = k8sClient.DeleteAllOf(context.Background(), &v1.Secret{}, client.InNamespace(vcNamespace)) Expect(err).To(haveNoErrorOrNotFoundError) + waitUntilSecretRemoved(names.VarnishSecret(vcName), vcNamespace) _ = k8sClient.Delete(context.Background(), &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: "test-priorityclass"}}) } diff --git a/pkg/varnishcluster/controller/varnishcluster_configmap_test.go b/pkg/varnishcluster/controller/varnishcluster_configmap_test.go index 13135b38..93220744 100644 --- a/pkg/varnishcluster/controller/varnishcluster_configmap_test.go +++ b/pkg/varnishcluster/controller/varnishcluster_configmap_test.go @@ -63,10 +63,11 @@ var _ = Describe("the ConfigMap", func() { cmLabels := vclabels.CombinedComponentLabels(newVC, vcapi.VarnishComponentVCLFileConfigMap) Expect(cm.Labels).To(Equal(cmLabels)) - Expect(cm.Data).To(HaveKey("entrypoint.vcl")) + Expect(cm.Data).To(HaveKey(*newVC.Spec.VCL.EntrypointFileName)) Expect(cm.Data).To(HaveKey("backends.vcl.tmpl")) - Expect(cm.Data["entrypoint.vcl"]).To(ContainSubstring("return (ok);")) - Expect(cm.Data["entrypoint.vcl"]).To(ContainSubstring("return (hash);")) + entrypoint := cm.Data[*newVC.Spec.VCL.EntrypointFileName] + Expect(entrypoint).To(ContainSubstring("return (ok);")) + Expect(entrypoint).To(ContainSubstring("return (hash);")) }) }) diff --git a/pkg/varnishcluster/controller/varnishcluster_secret_test.go b/pkg/varnishcluster/controller/varnishcluster_secret_test.go index d74e76f0..e19a3c07 100644 --- a/pkg/varnishcluster/controller/varnishcluster_secret_test.go +++ b/pkg/varnishcluster/controller/varnishcluster_secret_test.go @@ -45,6 +45,12 @@ var _ = Describe("the varnish secret", func() { } secretName := types.NamespacedName{Name: names.VarnishSecret(vc.Name), Namespace: vcNamespace} + BeforeEach(func() { + _ = k8sClient.Delete(context.Background(), &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: secretName.Name, Namespace: secretName.Namespace}, + }) + waitUntilSecretRemoved(secretName.Name, secretName.Namespace) + }) AfterEach(func() { CleanUpCreatedResources(vcName, vcNamespace) }) @@ -168,7 +174,7 @@ var _ = Describe("the varnish secret", func() { }) }) - Context("when the secret already exists and has empty password", func() { + Context("when the secret already exists with other keys and empty varnish password", func() { It("should update the password and do not touch another key", func() { customSecret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/varnishcluster/controller/varnishcluster_statefulset.go b/pkg/varnishcluster/controller/varnishcluster_statefulset.go index 34fbe2a8..2d704326 100644 --- a/pkg/varnishcluster/controller/varnishcluster_statefulset.go +++ b/pkg/varnishcluster/controller/varnishcluster_statefulset.go @@ -295,12 +295,12 @@ func (r *ReconcileVarnishCluster) reconcileStatefulSet(ctx context.Context, inst SecurityContext: &v1.PodSecurityContext{ FSGroup: proto.Int64(vcapi.VarnishRunAsGID), }, - ServiceAccountName: names.ServiceAccount(instance.Name), - NodeSelector: instance.Spec.NodeSelector, - Affinity: instance.Spec.Affinity, - PriorityClassName: instance.Spec.PriorityClassName, - Tolerations: instance.Spec.Tolerations, - RestartPolicy: v1.RestartPolicyAlways, + ServiceAccountName: names.ServiceAccount(instance.Name), + NodeSelector: instance.Spec.NodeSelector, + Affinity: instance.Spec.Affinity, + PriorityClassName: instance.Spec.PriorityClassName, + Tolerations: instance.Spec.Tolerations, + RestartPolicy: v1.RestartPolicyAlways, }, }, },