diff --git a/Dockerfile.controller b/Dockerfile.controller index 7927e4f..bf432e6 100644 --- a/Dockerfile.controller +++ b/Dockerfile.controller @@ -1,4 +1,5 @@ ARG BUILDPLATFORM=linux/amd64 +ARG VARNISH_VERSION_NUMBER=9.0.3-1 FROM --platform=$BUILDPLATFORM golang:1.26-bookworm AS builder ENV DEBIAN_FRONTEND=noninteractive INSTALL_DIRECTORY=/usr/local/bin @@ -28,24 +29,14 @@ RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ ./cmd/varnish-controller/ -FROM --platform=$BUILDPLATFORM debian:trixie-slim AS binary - -RUN apt-get update && apt-get install -y --no-install-recommends varnish \ - && rm -rf /var/lib/apt/lists/* - - -# Build Final Varnish image FROM --platform=$BUILDPLATFORM debian:trixie-slim LABEL maintainer="Alex Lytvynenko , Tomash Sidei " -RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ - libc6 libedit2 libncursesw6 libtinfo6 libvarnishapi3 varnish \ - && rm -rf /var/lib/apt/lists/* \ - /etc/varnish/* \ - && mkdir -p /etc/varnish /var/lib/varnish \ - && chown -R varnish:varnish /etc/varnish /var/lib/varnish +COPY docker/install-varnish-9.sh /tmp/install-varnish-9.sh +RUN chmod +x /tmp/install-varnish-9.sh \ + && VARNISH_VERSION_NUMBER="${VARNISH_VERSION_NUMBER}" /tmp/install-varnish-9.sh tools \ + && rm /tmp/install-varnish-9.sh -COPY --from=binary /usr/bin/varnishadm /usr/bin/varnishstat /usr/bin/ COPY --from=builder /go/src/github.com/cin/varnish-operator/varnish-controller /varnish-controller RUN chown varnish:varnish /varnish-controller /usr/bin/varnishadm /usr/bin/varnishstat diff --git a/Dockerfile.exporter b/Dockerfile.exporter index 2bdc39f..0dc3ca6 100644 --- a/Dockerfile.exporter +++ b/Dockerfile.exporter @@ -1,4 +1,5 @@ ARG BUILDPLATFORM=linux/amd64 +ARG VARNISH_VERSION_NUMBER=9.0.3-1 FROM --platform=$BUILDPLATFORM golang:1.26-bookworm AS builder ARG TARGETOS=linux ARG TARGETARCH=amd64 @@ -14,15 +15,14 @@ WORKDIR /src/prometheus_varnish_exporter RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /prometheus-varnish-exporter . + FROM --platform=$BUILDPLATFORM debian:trixie-slim LABEL maintainer="Alex Lytvynenko , Tomash Sidei " -# Install varnish so varnishstat and the varnish/vcache users match the varnishd image (VSM access). -RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ - libc6 libedit2 libncursesw6 libtinfo6 libvarnishapi3 varnish \ - && rm -rf /var/lib/apt/lists/* /etc/varnish/* \ - && mkdir -p /etc/varnish /var/lib/varnish \ - && chown -R varnish:varnish /etc/varnish /var/lib/varnish +COPY docker/install-varnish-9.sh /tmp/install-varnish-9.sh +RUN chmod +x /tmp/install-varnish-9.sh \ + && VARNISH_VERSION_NUMBER="${VARNISH_VERSION_NUMBER}" /tmp/install-varnish-9.sh tools \ + && rm /tmp/install-varnish-9.sh COPY --from=builder /prometheus-varnish-exporter /usr/bin/ RUN chown varnish:varnish /usr/bin/prometheus-varnish-exporter diff --git a/Dockerfile.varnishd b/Dockerfile.varnishd index d252181..b9ad178 100644 --- a/Dockerfile.varnishd +++ b/Dockerfile.varnishd @@ -1,13 +1,13 @@ ARG BUILDPLATFORM=linux/amd64 +ARG VARNISH_VERSION_NUMBER=9.0.3-1 FROM --platform=$BUILDPLATFORM debian:trixie-slim LABEL maintainer="Alex Lytvynenko , Tomash Sidei " -RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ - varnish \ - varnish-modules \ - && rm -rf /var/lib/apt/lists/* /etc/varnish/* \ - && chown -R varnish:varnish /etc/varnish /var/lib/varnish +COPY docker/install-varnish-9.sh /tmp/install-varnish-9.sh +RUN chmod +x /tmp/install-varnish-9.sh \ + && VARNISH_VERSION_NUMBER="${VARNISH_VERSION_NUMBER}" /tmp/install-varnish-9.sh minimal \ + && rm /tmp/install-varnish-9.sh USER varnish -ENTRYPOINT ["varnishd"] +ENTRYPOINT ["/usr/sbin/varnishd"] diff --git a/Makefile b/Makefile index bea2ca0..70978b4 100644 --- a/Makefile +++ b/Makefile @@ -93,9 +93,12 @@ endif varnish-controller: fmt vet go build -o ${ROOT_DIR}bin/varnish-controller ${ROOT_DIR}cmd/varnish-controller/ +VARNISH_VERSION_NUMBER ?= 9.0.3-1 + # Build the docker image with varnishd itself and varnish modules docker-build-varnish: - docker build --platform ${PLATFORM} ${ROOT_DIR} -t ${VARNISH_IMG} -f Dockerfile.varnishd + docker build --platform ${PLATFORM} ${ROOT_DIR} -t ${VARNISH_IMG} -f Dockerfile.varnishd \ + --build-arg VARNISH_VERSION_NUMBER=${VARNISH_VERSION_NUMBER} docker-tag-push-varnish: ifndef PUBLISH @@ -108,7 +111,8 @@ endif # Build the docker image with varnish controller docker-build-varnish-controller: fmt vet - docker build --platform ${PLATFORM} ${ROOT_DIR} -t ${VARNISH_CONTROLLER_IMG} -f Dockerfile.controller + docker build --platform ${PLATFORM} ${ROOT_DIR} -t ${VARNISH_CONTROLLER_IMG} -f Dockerfile.controller \ + --build-arg VARNISH_VERSION_NUMBER=${VARNISH_VERSION_NUMBER} docker-tag-push-varnish-controller: ifndef PUBLISH @@ -123,6 +127,7 @@ endif PROMETHEUS_VARNISH_EXPORTER_VERSION ?= v1.8.3 docker-build-varnish-exporter: docker build --platform ${PLATFORM} ${ROOT_DIR} -t ${VARNISH_METRICS_IMG} -f Dockerfile.exporter \ + --build-arg VARNISH_VERSION_NUMBER=${VARNISH_VERSION_NUMBER} \ --build-arg PROMETHEUS_VARNISH_EXPORTER_VERSION=${PROMETHEUS_VARNISH_EXPORTER_VERSION} docker-tag-push-varnish-exporter: diff --git a/README.md b/README.md index 88e9efb..3e6afd2 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ The purpose of the project is to provide a convenient way to deploy and manage V Kubernetes version `>=1.29.0` is supported (see the operator bundle `minKubeVersion`). CI runs e2e against Kubernetes 1.34.3 and 1.35.1, and unit tests use envtest 1.36.0—see [docs/development.md](docs/development.md#kubernetes-versions-in-tests) for why those version numbers differ. -Varnish version 7.x is supported (container images ship Debian trixie packages, currently Varnish 7.7). +Varnish version 9.x is supported (container images ship Varnish 9.0.3 from [packages.varnish-software.com](https://packages.varnish-software.com/) on Debian trixie). Full documentation can be found [here](https://cin.github.io/varnish-operator/) @@ -34,4 +34,5 @@ The operator manages the whole lifecycle of the cluster: creating, deleting and ### Further reading * [QuickStart](https://cin.github.io/varnish-operator/quick-start.html) +* [Custom container images](https://cin.github.io/varnish-operator/custom-images.html) * [Contributing](https://cin.github.io/varnish-operator/development.html) diff --git a/api/v1alpha1/varnishcluster_types.go b/api/v1alpha1/varnishcluster_types.go index 9327db2..af5e698 100644 --- a/api/v1alpha1/varnishcluster_types.go +++ b/api/v1alpha1/varnishcluster_types.go @@ -53,9 +53,9 @@ const ( VarnishSecretVolume = "secret" // VarnishWorkDir is the shared emptyDir mount for varnishd and varnishstat (must match -n). VarnishWorkDir = "/var/lib/varnish" - // VarnishRunAsUID and VarnishRunAsGID match the varnish system user from the Debian varnish package in our images. - VarnishRunAsUID = 997 - VarnishRunAsGID = 997 + // VarnishRunAsUID and VarnishRunAsGID match the varnish user from packages.varnish-software.com (Varnish 9 images). + VarnishRunAsUID = 1000 + VarnishRunAsGID = 1000 VarnishUpdateStrategyDelayedRollingUpdate = "DelayedRollingUpdate" diff --git a/config/samples/varnishcluster.yaml b/config/samples/varnishcluster.yaml index 6f9ba8f..e45c015 100644 --- a/config/samples/varnishcluster.yaml +++ b/config/samples/varnishcluster.yaml @@ -9,9 +9,11 @@ spec: # updateStrategy: # type: "OnDelete" #can be "OnDelete", "RollingUpdate" and "DelayedRollingUpdate" varnish: - # path to image + tag -# image: cinple/varnish:0.27.2 + # Optional: override images (tag = operator release, e.g. 0.38.0 or local—not Varnish 9.0.3). + # Sidecars default to varnish-controller: and varnish-metrics-exporter:. See docs/custom-images.md. +# image: cinple/varnish:0.38.0 # imagePullPolicy: Always +# imagePullSecret: regcred # Resources allocated to the Varnish pod through Kubernetes. It is strongly recommended that you specify resources, # since Varnish is an in-memory cache, and you do not want it restarting frequently. #resources: @@ -28,7 +30,7 @@ spec: # and defines the container's resources allocation. # controller: # # path to image + tag to override, by default it refers to varnish.image with "-controller" suffix image. -# image: cinple/varnish-controller:0.27.2 +# image: cinple/varnish-controller:0.38.0 # # imagePullPolicy controls how the varnish-controller image will be pulled for new containers # imagePullPolicy: Always # # Resources allocated to the Varnish controller container through Kubernetes. @@ -44,7 +46,7 @@ spec: # metricsExporter: # # path to image + tag to override, by default it refers to varnish.image with "-metrics-exporter" suffix image. -# image: cinple/varnish-metrics-exporter:0.27.2 +# image: cinple/varnish-metrics-exporter:0.38.0 # # imagePullPolicy controls how the varnish-metrics-exporter image will be pulled for new containers # imagePullPolicy: Always # # Resources allocated to the Varnish metrics exporter container through Kubernetes. diff --git a/docker/install-varnish-9.sh b/docker/install-varnish-9.sh new file mode 100644 index 0000000..249bfb4 --- /dev/null +++ b/docker/install-varnish-9.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# Install pinned Varnish 9.x from packages.varnish-software.com on Debian trixie. +# Usage: install-varnish-9.sh [minimal|tools] +# minimal — varnishd image: varnish + varnish-modules +# tools — controller/exporter: varnish CLI/libs only (no vmods) +set -ex + +MODE="${1:-minimal}" +VARNISH_VERSION_NUMBER="${VARNISH_VERSION_NUMBER:-9.0.3-1}" +REPO_FINGERPRINT="${REPO_FINGERPRINT:-694566269779DFAC975ED9BDD0525EAE838B3344}" + +. /etc/os-release +VARNISH_VERSION="${VARNISH_VERSION_NUMBER}~${VERSION_CODENAME}" + +export DEBIAN_FRONTEND=noninteractive +export DEBCONF_NONINTERACTIVE_SEEN=true + +apt-get update +apt-get install -y --no-install-recommends curl ca-certificates gnupg + +mkdir -p /etc/apt/keyrings +gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "${REPO_FINGERPRINT}" +gpg --batch --armor --export "${REPO_FINGERPRINT}" > /etc/apt/keyrings/varnish.gpg +echo "deb [signed-by=/etc/apt/keyrings/varnish.gpg] https://packages.varnish-software.com/varnish/${ID} ${VERSION_CODENAME} main" \ + > /etc/apt/sources.list.d/varnish.list + +apt-get update + +# Match official varnish/docker-varnish UID layout (not Debian stock 997). +adduser --uid 1000 --quiet --system --no-create-home --home /nonexistent --group varnish +adduser --uid 1001 --quiet --system --no-create-home --home /nonexistent --ingroup varnish vcache +adduser --uid 1002 --quiet --system --no-create-home --home /nonexistent --ingroup varnish varnishlog + +if [ "${MODE}" = "minimal" ]; then + PACKAGES="varnish=${VARNISH_VERSION} varnish-modules=${VARNISH_VERSION}" +else + PACKAGES="varnish=${VARNISH_VERSION}" +fi + +apt-get install -y --no-install-recommends ${PACKAGES} + +apt-mark hold varnish +rm -rf /var/lib/apt/lists/* /etc/varnish/* ~/.gnupg +mkdir -p /etc/varnish /var/lib/varnish +chown -R varnish:varnish /etc/varnish /var/lib/varnish +mkdir -p -m 1777 /var/lib/varnish/varnishd +chown varnish /var/lib/varnish/varnishd diff --git a/docs/README.md b/docs/README.md index b0e2e30..32dc7d4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -40,7 +40,7 @@ spec: port: 80 ``` -See the [VarnishCluster configuration section](varnish-cluster-configuration.md) for more details about the `VarnishCluster` spec. +See the [VarnishCluster configuration section](varnish-cluster-configuration.md) for more details about the `VarnishCluster` spec. To use your own `varnish` / sidecar images, see [Custom container images](custom-images.md). ### VCL configuration @@ -59,6 +59,7 @@ See the [VCL files configuration](vcl-configuration.md) section for more details * [Quickstart](quick-start.md) * [VarnishCluster configuration](varnish-cluster-configuration.md) +* [Custom container images](custom-images.md) * [Varnish operator configuration](operator-configuration.md) * [VCL files configuration](vcl-configuration.md) * [Contribution](development.md) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 210b696..eeb2800 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -6,6 +6,7 @@ * [Operator Configuration](operator-configuration.md) * [VarnishCluster](varnish-cluster.md) * [VarnishCluster Configuration](varnish-cluster-configuration.md) +* [Custom container images](custom-images.md) * [VCL Configuration](vcl-configuration.md) * [Monitoring](monitoring.md) * [Debugging Issues](debugging-issues.md) diff --git a/docs/architecture.md b/docs/architecture.md index 0094993..a6aa49c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -22,7 +22,7 @@ The containers share specific volumes for the varnish configuration and work dir ##### Varnish -The Varnish process itself. Varnish **7.x** is supported (images use Debian trixie packages, currently **7.7**). The operator doesn't support arbitrary Varnish images due to additional components needed for the operator to function. The container image is custom built with `varnish` and `varnish-modules` preinstalled. +The Varnish process itself. Varnish **9.x** is supported (default images install **9.0.3** from [packages.varnish-software.com](https://packages.varnish-software.com/) on Debian trixie, plus `varnish-modules`). You may set `spec.varnish.image` (and sidecar images) to your own registry, but all three pod images must stay compatible with the operator’s layout—see [Custom container images](custom-images.md). ##### Varnish-Controller diff --git a/docs/custom-images.md b/docs/custom-images.md new file mode 100644 index 0000000..144780d --- /dev/null +++ b/docs/custom-images.md @@ -0,0 +1,147 @@ +# Custom container images + +Each `VarnishCluster` pod runs three containers (`varnish`, `varnish-controller`, `varnish-metrics-exporter`). You can point them at your own images via the CR spec. The operator does **not** validate image contents; images must satisfy the [compatibility requirements](#compatibility-requirements-for-custom-images) below or pods will fail readiness checks, VCL reloads, or metrics scraping. + +### Image tags vs Varnish version + +**Container image tags follow your operator release** (Helm `container.tag`, CI `VERSION`, e.g. `0.38.0` or `local`)—not the upstream Varnish semver. All three images for a given operator release are built and published together with that tag. + +The **Varnish daemon version** inside the `varnish` image (today **9.0.3** via `VARNISH_VERSION_NUMBER` at build time) is separate from the OCI tag. The controller and metrics exporter images are operator components; tag them like `varnish-controller:0.38.0`, not `varnish-metrics-exporter:9.0.3`. + +## Specifying images on `VarnishCluster` + +Set `spec.varnish.image` for the `varnishd` container. Sidecar images default from that name unless overridden explicitly. + +```yaml +spec: + varnish: + image: registry.example.com/acme/varnish:0.38.0 + imagePullPolicy: IfNotPresent + imagePullSecret: regcred # optional, applies to all containers in the pod + controller: + image: registry.example.com/acme/varnish-controller:0.38.0 + imagePullPolicy: IfNotPresent + metricsExporter: + image: registry.example.com/acme/varnish-metrics-exporter:0.38.0 + imagePullPolicy: IfNotPresent +``` + +| Field | Purpose | +| ----- | ------- | +| `spec.varnish.image` | `varnishd` container image (repository:tag). | +| `spec.varnish.imagePullPolicy` | Pull policy for the `varnish` container (default: `Always`). | +| `spec.varnish.imagePullSecret` | Secret used to pull images for the StatefulSet pod. | +| `spec.varnish.controller.image` | Controller sidecar; if empty, derived from `varnish.image` (see below). | +| `spec.varnish.metricsExporter.image` | Metrics sidecar; if empty, derived from `varnish.image` (see below). | + +### Default image names when fields are omitted + +If `spec.varnish.image` is **empty**, the operator uses a **coupled default** derived from the operator Deployment image (`CONTAINER_IMAGE` / Helm `container.image`): + +| Operator image | Default `varnish` image | Default controller | Default metrics exporter | +| -------------- | ----------------------- | ------------------ | ------------------------ | +| `registry.io/team/varnish-operator:1.2.0` | `registry.io/team/varnish:1.2.0` | `registry.io/team/varnish-controller:1.2.0` | `registry.io/team/varnish-metrics-exporter:1.2.0` | + +The repository path is taken from the operator image; the image name is replaced with `varnish`, and the **same tag** is reused. Controller and exporter names append `-controller` and `-metrics-exporter` to the repository name before the tag. + +If you set only `spec.varnish.image`, sidecars use that naming rule automatically: + +```yaml +spec: + varnish: + image: myregistry/varnish:0.38.0 + # controller -> myregistry/varnish-controller:0.38.0 + # metrics -> myregistry/varnish-metrics-exporter:0.38.0 +``` + +Override individual sidecars when your registry naming differs (keep the **same operator release tag** on all three unless you know the builds are paired): + +```yaml +spec: + varnish: + image: myregistry/varnish:0.38.0 + controller: + image: myregistry/varnish-ctrl:0.38.0 +``` + +After changing an image tag in a running cluster, set `spec.statefulSet.container.imagePullPolicy: Always` (on the embedded pod template, if you customize it) or delete pods so nodes pull new layers. + +## Compatibility requirements for custom images + +All three images must target the **same Varnish major/minor** (today: **9.0.3** from [packages.varnish-software.com](https://packages.varnish-software.com/) in this repository’s Dockerfiles). Mixing `varnishd` 9.x with sidecars built against 7.x will break `varnishadm`, `varnishstat`, and the Prometheus exporter. + +### `varnish` (`varnishd`) image + +| Requirement | Detail | +| ----------- | ------ | +| **Process** | `varnishd` on `PATH` or `/usr/sbin/varnishd` (StatefulSet does not override the image entrypoint). | +| **Vmods** | If you use the operator’s default VCL (`import var;`, `import directors;` in templates), install **`varnish-modules`** for your Varnish version. | +| **User / UID** | Run as non-root user **`varnish` with UID/GID 1000** (Varnish Software packages). The operator sets `runAsUser` / `runAsGroup` / `fsGroup` to **1000**. | +| **Directories** | Writable `/var/lib/varnish` (instance dir `-n`) and `/etc/varnish` (VCL mount). | +| **Args** | The operator injects `-F`, `-n /var/lib/varnish`, `-S /etc/varnish-secret/secret`, `-T 0.0.0.0:6082`, `-b 127.0.0.1:0`, `-a 0.0.0.0:`. Do not rely on overriding `-n`, `-f`, `-S`, `-T`, or `-b` via `spec.varnish.args`. | + +### `varnish-controller` image + +| Requirement | Detail | +| ----------- | ------ | +| **Binary** | `/varnish-controller` as entrypoint (built from this repo’s `cmd/varnish-controller`). | +| **CLI tools** | `varnishadm` and `varnishstat` compatible with the `varnishd` version. | +| **Libraries** | Matching `libvarnishapi` (e.g. `libvarnishapi.so.3` on 9.x). | +| **User** | UID **1000** (`varnish`), with read access to the shared workdir and secret volume. | + +### `varnish-metrics-exporter` image + +| Requirement | Detail | +| ----------- | ------ | +| **Binary** | `prometheus-varnish-exporter` (this repo builds [otto-de/prometheus_varnish_exporter](https://github.com/otto-de/prometheus_varnish_exporter) v1.8.3+). | +| **CLI tools** | `varnishstat` + matching `libvarnishapi`. | +| **User** | UID **1000** (`varnish`). | +| **Args** | Operator passes **`-n /var/lib/varnish`**; the image must support that flag. | + +### VCL and configuration (not in the image, but required) + +Custom images do not remove the need for compatible **VCL** in the ConfigMap (`spec.vcl`). Review [Varnish 9 upgrade notes](https://varnish-cache.org/docs/9.0/whats-new/upgrading-9.0.html) when moving from older versions. + +## What you cannot change via image alone + +These are fixed in the operator today; custom images must adapt to them (or you fork the operator): + +- **Security context**: `runAsUser` / `runAsGroup` / `fsGroup` **1000** on Varnish pods. +- **Volume layout**: shared `emptyDir` at `/var/lib/varnish`, VCL under `/etc/varnish`, admin secret at `/etc/varnish-secret/secret`. +- **Ports**: Varnish HTTP from `spec.service.port`, admin CLI on **6082**, metrics on **9131** (defaults). +- **Readiness probe**: `varnishadm -S … -T 127.0.0.1:6082 ping`. + +## Building images from this repository + +Reference Dockerfiles (Debian trixie + Varnish Software packages): + +| Image | Dockerfile | Install script | +| ----- | ---------- | -------------- | +| `varnish` | `Dockerfile.varnishd` | `docker/install-varnish-9.sh` (`minimal`: `varnish` + `varnish-modules`) | +| `varnish-controller` | `Dockerfile.controller` | `install-varnish-9.sh` (`tools`: CLI + libs) + Go binary | +| `varnish-metrics-exporter` | `Dockerfile.exporter` | `tools` + exporter build | + +```bash +make docker-build-pod REPO=myregistry VERSION=0.38.0 +# myregistry/varnish:0.38.0 +# myregistry/varnish-controller:0.38.0 +# myregistry/varnish-metrics-exporter:0.38.0 +``` + +Pin a different **Varnish package** inside the `varnish` image (does not change the OCI tag): + +```bash +make docker-build-varnish VARNISH_VERSION_NUMBER=9.0.3-1 REPO=myregistry VERSION=0.38.0 +``` + +Build args: `VERSION` (image tag / operator release), `VARNISH_VERSION_NUMBER` (Debian package pin, default `9.0.3-1`), `PROMETHEUS_VARNISH_EXPORTER_VERSION` (default `v1.8.3`). + +## Using upstream `library/varnish` images + +The official [Docker Hub `varnish` image](https://hub.docker.com/_/varnish) also uses UID **1000** and Varnish Software packages, but it is **not** tested as a drop-in for this operator: entrypoint scripts, bundled VCL, and optional vmods differ from the images built here. Prefer the Dockerfiles in this repo, or replicate their layout (users, paths, `varnishadm`/`varnishstat`, `varnish-modules`) and run `make e2e-tests` before production use. + +## Related documentation + +- [VarnishCluster configuration](varnish-cluster-configuration.md) — full spec field list +- [Development / local images](development.md) — kind, `make e2e-tests`, local tags +- [Architecture](architecture.md) — pod layout and component roles diff --git a/docs/development.md b/docs/development.md index 944c36f..b4d2705 100644 --- a/docs/development.md +++ b/docs/development.md @@ -127,7 +127,7 @@ This produces: | `cinple/varnish-controller:local` | `Dockerfile.controller` | | `cinple/varnish-metrics-exporter:local` | `Dockerfile.exporter` | -Override images in your `VarnishCluster`: +Override images in your `VarnishCluster` (see [Custom container images](custom-images.md) for defaults, naming rules, and what custom images must provide): ```yaml spec: @@ -141,9 +141,13 @@ spec: If you reuse the same tag, set `spec.statefulSet.container.imagePullPolicy: Always` and restart pods (or delete them) so Kubernetes pulls the new layers. -Varnish pod images (`varnish`, `varnish-controller`, `varnish-metrics-exporter`) are based on **Debian trixie** and ship **Varnish 7.x** from Debian packages. Rebuild all three together when upgrading Varnish. +Varnish pod images (`varnish`, `varnish-controller`, `varnish-metrics-exporter`) are based on **Debian trixie** and ship **Varnish 9.0.3** from [packages.varnish-software.com](https://packages.varnish-software.com/) (see `docker/install-varnish-9.sh`). Rebuild all three together when upgrading Varnish. Override the pin with build-arg `VARNISH_VERSION_NUMBER` (default `9.0.3-1`). -Those images run as the Debian **`varnish` user (UID/GID 997)**, not root. The StatefulSet sets `runAsNonRoot`, `runAsUser`/`runAsGroup` 997, drops capabilities, and uses `fsGroup` 997 on shared volumes so sidecars can read the Varnish workdir without a custom root user. +Those images run as the **`varnish` user (UID/GID 1000)** from the Varnish Software packages, not root. The StatefulSet sets `runAsNonRoot`, `runAsUser`/`runAsGroup` 1000, drops capabilities, and uses `fsGroup` 1000 on shared volumes so sidecars can read the Varnish workdir. + +```bash +docker build --build-arg VARNISH_VERSION_NUMBER=9.0.3-1 -f Dockerfile.varnishd . +``` The metrics exporter image accepts `PROMETHEUS_VARNISH_EXPORTER_VERSION` and `PROMETHEUS_VARNISH_EXPORTER_REPO` as build arguments (defaults: `v1.8.3` from [otto-de/prometheus_varnish_exporter](https://github.com/otto-de/prometheus_varnish_exporter)): @@ -152,7 +156,24 @@ docker build --build-arg PROMETHEUS_VARNISH_EXPORTER_VERSION=v1.8.3 \ -t my-exporter:local -f Dockerfile.exporter . ``` -When upgrading from Varnish 6 images, review custom VCL for [Varnish 7 changes](https://varnish-cache.org/docs/7.0/whats-new/upgrading-7.0.html) (notably PCRE2 regex behavior). +When upgrading from older Varnish images, review custom VCL for [Varnish 7](https://varnish-cache.org/docs/7.0/whats-new/upgrading-7.0.html) and [Varnish 9](https://varnish-cache.org/docs/9.0/whats-new/upgrading-9.0.html) release notes (PCRE2, removed APIs, etc.). Expect a cold cache after rollout; the default workdir is `emptyDir`. + +### Default VCL on a dev cluster + +`./hack/create_dev_cluster.sh -v` creates a `VarnishCluster` that references `vcl-config` / `entrypoint.vcl`. If that ConfigMap does not exist, the operator seeds the [default VCL](../pkg/varnishcluster/controller/varnishcluster_default_vcl.go) (`import var`, `directors` round-robin backends, `/heartbeat`, `/liveness`, `X-Varnish-Cache`). + +Manual smoke test (after `-b` and `-v`): + +```bash +export KUBECONFIG=./e2e-tests-kubeconfig +kubectl port-forward -n varnish-cluster svc/varnishcluster-example 8080:80 + +curl -s -o /dev/null -w "%{http_code}\n" http://127.0.0.1:8080/heartbeat # 200 +curl -s -o /dev/null -w "%{http_code}\n" http://127.0.0.1:8080/liveness # 200 +curl -sI http://127.0.0.1:8080/ | grep -i x-varnish-cache # MISS then HIT +``` + +Automated coverage: `go test ./tests -ginkgo.focus="operator default VCL"` (requires `make e2e-tests` or an equivalent cluster). ## Code generation and manifests diff --git a/docs/operator-configuration.md b/docs/operator-configuration.md index 528c70f..43535ab 100644 --- a/docs/operator-configuration.md +++ b/docs/operator-configuration.md @@ -2,6 +2,8 @@ As the operator is packaged into [the Helm chart](https://helm.sh/docs/developing_charts/) the configuration is done by setting `values.yaml` overrides. +When a `VarnishCluster` does not set `spec.varnish.image`, workload images are derived from the operator image (`container.registry` / `container.repository` / `container.tag`)—for example `…/varnish:`, `…/varnish-controller:`, and `…/varnish-metrics-exporter:`. To override per cluster or run private builds, see [Custom container images](custom-images.md). + | Field | Description | Default | | ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------- | | `affinity` | [Affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) settings for operator pods | `optional` | diff --git a/docs/varnish-cluster-configuration.md b/docs/varnish-cluster-configuration.md index 0481f08..194959f 100644 --- a/docs/varnish-cluster-configuration.md +++ b/docs/varnish-cluster-configuration.md @@ -1,11 +1,17 @@ # VarnishCluster Configuration +## Container images + +Override the three workload images (`varnish`, `varnish-controller`, `varnish-metrics-exporter`) with `spec.varnish.image` and optional `spec.varnish.controller.image` / `spec.varnish.metricsExporter.image`. Image **tags match the operator release** (not the Varnish daemon version). If `varnish.image` is omitted, names are derived from the operator Deployment image (same registry/tag, with `varnish`, `varnish-controller`, and `varnish-metrics-exporter` image names). + +**Custom images must match operator expectations** (UID 1000, shared `/var/lib/varnish`, matching `varnishadm`/`varnishstat`/Varnish version, etc.). See **[Custom container images](custom-images.md)** for examples, defaults, and a full compatibility checklist. + | Field | Description | Is Required | | ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------- | | `affinity ` | [Affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) settings for the pods. It allows you to configure onto which nodes Varnish pods should prefer being scheduled. | `optional` | | `priorityClassName ` | [priorityClass](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass) settings for the pods. It allows you to set a PriorityClassName and thus set a priority to your pods, to avoid eviction. | `optional` | | `backend.namespaces ` | Namespace(s) to look for backend pods. By default - namespace the VarnishCluster is deployed to. | `required` | -| `backend.onlyReady ` | Include (`false`, by default) or exclude (`true`) backend pods from the VCL (.Backends template var). Alters `.Backends` template variable based on Kubernetes health checks (by default not ready pods are also included in VCL) instead of [Varnish health probes](https://varnish-cache.org/docs/7.7/reference/vcl-probe.html#backend-health-probes). | `optional` | +| `backend.onlyReady ` | Include (`false`, by default) or exclude (`true`) backend pods from the VCL (.Backends template var). Alters `.Backends` template variable based on Kubernetes health checks (by default not ready pods are also included in VCL) instead of [Varnish health probes](https://varnish-cache.org/docs/9.0/reference/vcl-probe.html#backend-health-probes). | `optional` | | `backend.port ` | The port of the backend pods being cached by Varnish. Can be port name or port number. | `required` | | `backend.selector ` | The selector used to identify the backend Pods. | `required` | | `backend.zoneBalancing ` | Controls Varnish backend topology aware routing which can assign weights to backends according to their geographical location. | `optional` | @@ -51,7 +57,7 @@ | `varnish.admAuth.key ` | The key from kubernetes secret which to use to collect data credentials for `varnishadm`. If the key is omitted, the cluster will use "secret" as the key. If the value associated to the key is empty, the cluster will generate a secret. | `optional` | | `varnish.args ` | Additional [Varnish daemon arguments](https://varnish-cache.org/docs/trunk/reference/varnishd.html#options) | `optional` | | `varnish.controller ` | An object that defines the configuration of a particular Varnish controller being deployed | `optional` | -| `varnish.controller.image ` | Path to the Varnish Controller image being used. If not defined uses `varnish.image`+`-controller` suffix. Something like `varnish-controller` | `optional` | +| `varnish.controller.image ` | Controller sidecar image. Empty → `-controller:` (see [custom-images.md](custom-images.md)) | `optional` | | `varnish.controller.imagePullPolicy ` | Image pull policy for the container. Default: `Always` | `optional` | | `varnish.controller.resources ` | [Resource requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) for Varnish controller container. | `optional` | | `varnish.envFrom ` | Injects an env var into the Varnish container from a ConfigMap or Secret. Useful if a value needs to be passed (securely in case of Secret) to the VCL files. So it can be read using [std.getenv()](https://varnish-cache.org/docs/trunk/reference/vmod_std.html#string-getenv-string-name). | `optional` | @@ -71,11 +77,11 @@ | `varnish.extraVolumeClaimTemplates[].spec ` | Spec for the [PersistentVolumeClaim](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#persistentvolumeclaimspec-v1-core) | `optional` | | `varnish.extraVolumeMounts ` | Additional [volume mounts](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#volumemount-v1-core) for the Varnish container | `optional` | | `varnish.extraVolumes ` | Additional [volumes](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#volume-v1-core) | `optional` | -| `varnish.image ` | Path to the Varnish image being used | `optional` | +| `varnish.image ` | `varnishd` container image (`repository:tag`). Empty → coupled default from operator image (see [custom-images.md](custom-images.md)) | `optional` | | `varnish.imagePullPolicy ` | Image pull policy for the Varnish container. Default: `Always` | `optional` | | `varnish.imagePullSecret ` | The name of the image pull secret to use to pull container images | `optional` | | `varnish.metricsExporter ` | An object that defines the configuration of a particular Varnish Prometheus metrics exporter being deployed | `optional` | -| `varnish.metricsExporter.image ` | Path to the Varnish Metrics exporter image being used. If not defined uses `varnish.image`+`-metrics-exporter` suffix. Something like `varnish-metrics-exporter` | `optional` | +| `varnish.metricsExporter.image ` | Metrics exporter image. Empty → `-metrics-exporter:` (see [custom-images.md](custom-images.md)) | `optional` | | `varnish.metricsExporter.imagePullPolicy ` | Image pull policy for the container. Default: `Always` | `optional` | | `varnish.metricsExporter.resources ` | [Resource requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) for Varnish metrics exporter container. | `optional` | | `varnish.resources ` | [Resource requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) for Varnish container. | `optional` | diff --git a/docs/vcl-configuration.md b/docs/vcl-configuration.md index 3f4b744..08016ed 100644 --- a/docs/vcl-configuration.md +++ b/docs/vcl-configuration.md @@ -125,7 +125,7 @@ The `.Backends` array can include both all backend nodes (default behavior) and It depends on which readiness checks you want to respect: * Kubernetes readiness checks - field set to `true`. In this case the `.Backends` template var in VCL will include only ready backend pods. Keep in mind that every backend state change will trigger a Varnish VCL config reload. -* Varnish readiness probes - field set to `false` or omitted. This way the `.Backends` array will include all scheduled pods, so it is strongly recommended to set [health checks](https://varnish-cache.org/docs/7.7/reference/vcl-probe.html#backend-health-probes) in your VCL configuration. Otherwise, Varnish could send traffic to not yet ready backends. +* Varnish readiness probes - field set to `false` or omitted. This way the `.Backends` array will include all scheduled pods, so it is strongly recommended to set [health checks](https://varnish-cache.org/docs/9.0/reference/vcl-probe.html#backend-health-probes) in your VCL configuration. Otherwise, Varnish could send traffic to not yet ready backends. You can also combine approaches and use both Kubernetes and Varnish health probes. diff --git a/hack/create_dev_cluster.sh b/hack/create_dev_cluster.sh index 02fd0df..d440807 100755 --- a/hack/create_dev_cluster.sh +++ b/hack/create_dev_cluster.sh @@ -42,6 +42,41 @@ if ! which helm >/dev/null; then exit 1 fi +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +KUBECONFIG_FILE="${ROOT_DIR}/e2e-tests-kubeconfig" + +function e2e_kubeconfig_valid() { + kubectl config current-context --kubeconfig="${KUBECONFIG_FILE}" >/dev/null 2>&1 +} + +function refresh_e2e_kubeconfig_from_kind() { + if kind get clusters 2>/dev/null | grep -qx "${cluster_name}"; then + kind get kubeconfig --name "${cluster_name}" > "${KUBECONFIG_FILE}" + return 0 + fi + return 1 +} + +function use_e2e_kubeconfig() { + if [[ -f "${KUBECONFIG_FILE}" ]] && ! e2e_kubeconfig_valid; then + echo "warning: ${KUBECONFIG_FILE} is empty or stale; refreshing from kind cluster ${cluster_name}..." >&2 + rm -f "${KUBECONFIG_FILE}" + fi + if [[ ! -f "${KUBECONFIG_FILE}" ]]; then + if ! refresh_e2e_kubeconfig_from_kind; then + echo "error: no valid kubeconfig at ${KUBECONFIG_FILE} and kind cluster '${cluster_name}' is not running." >&2 + echo "Run: ${SCRIPT_DIR}/create_dev_cluster.sh # without -v or -b" >&2 + exit 1 + fi + fi + if ! e2e_kubeconfig_valid; then + echo "error: ${KUBECONFIG_FILE} is not a usable kubeconfig." >&2 + exit 1 + fi + export KUBECONFIG="${KUBECONFIG_FILE}" +} + varnish_namespace="varnish-operator" cluster_name="e2e-tests" repo="cinple" @@ -66,14 +101,15 @@ Creates a dev cluster and varnish-operator install -n|--namespace | namespace -p|--platform | platform (not validated so know which build you're calling) -r|--repo | CR repository --b|--backends | create backends +-b|--backends | create nginx backends (requires cluster; run script without -v/-b first) -s|--skip-docker-build | skip docker build --v|--create-varnishcluster | create varnish cluster +-v|--create-varnishcluster | create sample VarnishCluster (requires cluster; run script without -v/-b first) -x|--ignore-podman | ignore podman's presence ! } function default_vc_namespace { + use_e2e_kubeconfig if [[ "$varnish_namespace" == "varnish-operator" ]]; then if [ "$(kubectl get namespace --no-headers | grep varnish-cluster | wc -l | xargs echo -n)" -eq 0 ]; then kubectl create namespace varnish-cluster @@ -82,6 +118,21 @@ function default_vc_namespace { fi } +function load_local_images_into_kind() { + use_e2e_kubeconfig + if ! kind get clusters 2>/dev/null | grep -qx "${cluster_name}"; then + echo "error: kind cluster '${cluster_name}' not found." >&2 + exit 1 + fi + for image in "${workload_images[@]}"; do + if ! docker image inspect "${image}" >/dev/null 2>&1; then + echo "error: local image ${image} not found. Run ${SCRIPT_DIR}/create_dev_cluster.sh (without -v/-b) to build images first." >&2 + exit 1 + fi + kind load docker-image --name "${cluster_name}" "${image}" + done +} + function create_nginx_backends { if [ "$dry_run" = true ]; then echo "dry-run: would otherwise be installing nginx" @@ -98,6 +149,7 @@ function create_varnishcluster { fi default_vc_namespace + load_local_images_into_kind cat < /dev/null 2>&1 - kind create cluster --name $cluster_name --image "${kind_node_image}" --kubeconfig ./e2e-tests-kubeconfig - export KUBECONFIG=./e2e-tests-kubeconfig + kind create cluster --name $cluster_name --image "${kind_node_image}" --kubeconfig "${KUBECONFIG_FILE}" fi +use_e2e_kubeconfig + if [ "$(kubectl get namespace --no-headers | grep varnish-operator | wc -l | xargs echo -n)" -eq 0 ]; then kubectl create ns $varnish_namespace fi @@ -202,7 +268,7 @@ if [ "$skip_docker_build" = false ]; then fi for image in "${images[@]}"; do - kind load docker-image -n $cluster_name $image + kind load docker-image --name "${cluster_name}" "${image}" done helm install varnish-operator varnish-operator --namespace=$varnish_namespace --wait --set container.imagePullPolicy=Never --set container.image=$container_image diff --git a/hack/delete_dev_cluster.sh b/hack/delete_dev_cluster.sh index 101933a..d46e224 100755 --- a/hack/delete_dev_cluster.sh +++ b/hack/delete_dev_cluster.sh @@ -1,5 +1,9 @@ #!/bin/bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +KUBECONFIG_FILE="${ROOT_DIR}/e2e-tests-kubeconfig" + cluster_name="e2e-tests" function usage { @@ -22,6 +26,6 @@ done kind delete cluster --name $cluster_name > /dev/null 2>&1 -if [[ -f ./e2e-tests-kubeconfig ]]; then - rm -f ./e2e-tests-kubeconfig +if [[ -f "${KUBECONFIG_FILE}" ]]; then + rm -f "${KUBECONFIG_FILE}" fi diff --git a/pkg/varnishcluster/controller/suite_test.go b/pkg/varnishcluster/controller/suite_test.go index 0bcbef3..abd5805 100644 --- a/pkg/varnishcluster/controller/suite_test.go +++ b/pkg/varnishcluster/controller/suite_test.go @@ -166,6 +166,13 @@ func StartTestManager(mgr manager.Manager) chan struct{} { return stop } +func waitUntilSecretRemoved(name, namespace string) { + Eventually(func() bool { + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, &v1.Secret{}) + return errors.IsNotFound(err) + }, time.Second*5).Should(BeTrue()) +} + // As the test control plane doesn't support garbage collection, this function is used to clean up resources // Designed to not fail if the resource is not found func CleanUpCreatedResources(vcName, vcNamespace string) { @@ -228,7 +235,12 @@ func CleanUpCreatedResources(vcName, vcNamespace string) { Expect(err).To(BeNil()) err = k8sClient.DeleteAllOf(context.Background(), &apps.StatefulSet{}, client.InNamespace(vcNamespace)) Expect(err).To(BeNil()) + _ = k8sClient.Delete(context.Background(), &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: names.VarnishSecret(vcName), Namespace: vcNamespace}, + }) + waitUntilSecretRemoved(names.VarnishSecret(vcName), vcNamespace) err = k8sClient.DeleteAllOf(context.Background(), &v1.Secret{}, client.InNamespace(vcNamespace)) Expect(err).To(haveNoErrorOrNotFoundError) + waitUntilSecretRemoved(names.VarnishSecret(vcName), vcNamespace) _ = k8sClient.Delete(context.Background(), &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: "test-priorityclass"}}) } diff --git a/pkg/varnishcluster/controller/varnishcluster_configmap_test.go b/pkg/varnishcluster/controller/varnishcluster_configmap_test.go index 7f890af..9322074 100644 --- a/pkg/varnishcluster/controller/varnishcluster_configmap_test.go +++ b/pkg/varnishcluster/controller/varnishcluster_configmap_test.go @@ -63,6 +63,11 @@ var _ = Describe("the ConfigMap", func() { cmLabels := vclabels.CombinedComponentLabels(newVC, vcapi.VarnishComponentVCLFileConfigMap) Expect(cm.Labels).To(Equal(cmLabels)) + Expect(cm.Data).To(HaveKey(*newVC.Spec.VCL.EntrypointFileName)) + Expect(cm.Data).To(HaveKey("backends.vcl.tmpl")) + entrypoint := cm.Data[*newVC.Spec.VCL.EntrypointFileName] + Expect(entrypoint).To(ContainSubstring("return (ok);")) + Expect(entrypoint).To(ContainSubstring("return (hash);")) }) }) diff --git a/pkg/varnishcluster/controller/varnishcluster_secret_test.go b/pkg/varnishcluster/controller/varnishcluster_secret_test.go index d74e76f..e19a3c0 100644 --- a/pkg/varnishcluster/controller/varnishcluster_secret_test.go +++ b/pkg/varnishcluster/controller/varnishcluster_secret_test.go @@ -45,6 +45,12 @@ var _ = Describe("the varnish secret", func() { } secretName := types.NamespacedName{Name: names.VarnishSecret(vc.Name), Namespace: vcNamespace} + BeforeEach(func() { + _ = k8sClient.Delete(context.Background(), &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: secretName.Name, Namespace: secretName.Namespace}, + }) + waitUntilSecretRemoved(secretName.Name, secretName.Namespace) + }) AfterEach(func() { CleanUpCreatedResources(vcName, vcNamespace) }) @@ -168,7 +174,7 @@ var _ = Describe("the varnish secret", func() { }) }) - Context("when the secret already exists and has empty password", func() { + Context("when the secret already exists with other keys and empty varnish password", func() { It("should update the password and do not touch another key", func() { customSecret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/varnishcluster/controller/varnishcluster_statefulset.go b/pkg/varnishcluster/controller/varnishcluster_statefulset.go index 34fbe2a..2d70432 100644 --- a/pkg/varnishcluster/controller/varnishcluster_statefulset.go +++ b/pkg/varnishcluster/controller/varnishcluster_statefulset.go @@ -295,12 +295,12 @@ func (r *ReconcileVarnishCluster) reconcileStatefulSet(ctx context.Context, inst SecurityContext: &v1.PodSecurityContext{ FSGroup: proto.Int64(vcapi.VarnishRunAsGID), }, - ServiceAccountName: names.ServiceAccount(instance.Name), - NodeSelector: instance.Spec.NodeSelector, - Affinity: instance.Spec.Affinity, - PriorityClassName: instance.Spec.PriorityClassName, - Tolerations: instance.Spec.Tolerations, - RestartPolicy: v1.RestartPolicyAlways, + ServiceAccountName: names.ServiceAccount(instance.Name), + NodeSelector: instance.Spec.NodeSelector, + Affinity: instance.Spec.Affinity, + PriorityClassName: instance.Spec.PriorityClassName, + Tolerations: instance.Spec.Tolerations, + RestartPolicy: v1.RestartPolicyAlways, }, }, }, diff --git a/tests/default_vcl_test.go b/tests/default_vcl_test.go new file mode 100644 index 0000000..090f3c8 --- /dev/null +++ b/tests/default_vcl_test.go @@ -0,0 +1,184 @@ +package tests + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "time" + + vcapi "github.com/cin/varnish-operator/api/v1alpha1" + + "github.com/gogo/protobuf/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Exercises the operator-seeded ConfigMap (entrypoint.vcl + backends.vcl.tmpl), not a user-supplied VCL. +var _ = Describe("operator default VCL", func() { + vcNamespace := "default" + vcName := "default-vcl-test" + configMapName := "e2e-operator-default-vcl" + objMeta := metav1.ObjectMeta{ + Namespace: vcNamespace, + Name: vcName, + } + backendResponse := "DEFAULT-VCL-BACKEND" + backendLabels := map[string]string{"app": "default-vcl-backend"} + backendDeploymentName := "default-vcl-backend" + varnishPodLabels := map[string]string{ + vcapi.LabelVarnishOwner: vcName, + vcapi.LabelVarnishComponent: vcapi.VarnishComponentVarnish, + } + + backendsDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: backendDeploymentName, + Namespace: vcNamespace, + Labels: backendLabels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: proto.Int32(1), + Selector: &metav1.LabelSelector{ + MatchLabels: backendLabels, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: backendLabels, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "backend", + Image: "hashicorp/http-echo", + Ports: []v1.ContainerPort{ + { + Name: "web", + Protocol: v1.ProtocolTCP, + ContainerPort: 5678, + }, + }, + Args: []string{fmt.Sprintf("-text=%s", backendResponse)}, + }, + }, + }, + }, + }, + } + + backendPort := intstr.FromInt(5678) + vc := &vcapi.VarnishCluster{ + ObjectMeta: objMeta, + Spec: vcapi.VarnishClusterSpec{ + Backend: &vcapi.VarnishClusterBackend{ + Selector: backendLabels, + Port: &backendPort, + }, + Service: &vcapi.VarnishClusterService{ + Port: proto.Int32(9090), + }, + Varnish: &vcapi.VarnishClusterVarnish{ + ImagePullPolicy: v1.PullNever, + Controller: &vcapi.VarnishClusterVarnishController{ + ImagePullPolicy: v1.PullNever, + }, + MetricsExporter: &vcapi.VarnishClusterVarnishMetricsExporter{ + ImagePullPolicy: v1.PullNever, + }, + }, + VCL: &vcapi.VarnishClusterVCL{ + ConfigMapName: proto.String(configMapName), + EntrypointFileName: proto.String("entrypoint.vcl"), + }, + }, + } + + AfterEach(func() { + By("deleting created resources") + Expect(k8sClient.DeleteAllOf(context.Background(), &vcapi.VarnishCluster{}, client.InNamespace(vcNamespace))).To(Succeed()) + Expect(k8sClient.DeleteAllOf(context.Background(), &appsv1.Deployment{}, client.InNamespace(vcNamespace), client.MatchingLabels(backendLabels))).To(Succeed()) + Expect(k8sClient.DeleteAllOf(context.Background(), &v1.ConfigMap{}, client.InNamespace(vcNamespace), client.MatchingLabels(map[string]string{ + vcapi.LabelVarnishOwner: vcName, + }))).To(Succeed()) + waitForPodsTermination(vcNamespace, varnishPodLabels) + waitForPodsTermination(vcNamespace, backendLabels) + waitUntilVarnishClusterRemoved(vcName, vcNamespace) + }) + + It("seeds ConfigMap VCL and serves traffic on Varnish 9", func() { + Expect(k8sClient.Create(context.Background(), backendsDeployment)).To(Succeed()) + Expect(k8sClient.Create(context.Background(), vc)).To(Succeed()) + + cm := &v1.ConfigMap{} + Eventually(func() error { + return k8sClient.Get(context.Background(), types.NamespacedName{ + Name: configMapName, Namespace: vcNamespace, + }, cm) + }, time.Minute, time.Second*2).Should(Succeed()) + + By("operator created default VCL files") + Expect(cm.Data).To(HaveKey("entrypoint.vcl")) + Expect(cm.Data).To(HaveKey("backends.vcl.tmpl")) + entrypoint := cm.Data["entrypoint.vcl"] + Expect(entrypoint).To(ContainSubstring("return (ok);")) + Expect(entrypoint).To(ContainSubstring("return (hash);")) + Expect(entrypoint).To(ContainSubstring("import var;")) + Expect(entrypoint).To(ContainSubstring(`include "backends.vcl"`)) + Expect(cm.Data["backends.vcl.tmpl"]).To(ContainSubstring("import directors;")) + + By("backend pods become ready") + waitForPodsReadiness(vcNamespace, backendLabels) + By("varnish pods become ready") + waitForPodsReadiness(vcNamespace, varnishPodLabels) + + pf := portForwardPod(vcNamespace, varnishPodLabels, []string{"6081:6081"}) + defer pf.Close() + + By("default /heartbeat endpoint") + Eventually(func() (int, error) { + resp, err := http.Get("http://localhost:6081/heartbeat") + if err != nil { + return 0, err + } + defer func() { _ = resp.Body.Close() }() + return resp.StatusCode, nil + }, time.Minute, time.Second*2).Should(Equal(200)) + + By("default /liveness endpoint with healthy backend") + Eventually(func() (int, error) { + resp, err := http.Get("http://localhost:6081/liveness") + if err != nil { + return 0, err + } + defer func() { _ = resp.Body.Close() }() + return resp.StatusCode, nil + }, time.Minute, time.Second*2).Should(Equal(200)) + + By("cached backend response with X-Varnish-Cache") + var resp *http.Response + Eventually(func() (int, error) { + var err error + resp, err = http.Get("http://localhost:6081/cached-path") + if err != nil { + return 0, err + } + return resp.StatusCode, nil + }, time.Second*30, time.Second*2).Should(Equal(200)) + Expect(resp.Header.Get("X-Varnish-Cache")).To(Equal("MISS")) + body, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(strings.TrimSpace(string(body))).To(Equal(backendResponse)) + + resp, err = http.Get("http://localhost:6081/cached-path") + Expect(err).NotTo(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(200)) + Expect(resp.Header.Get("X-Varnish-Cache")).To(Equal("HIT")) + }) +})