From bf751760771480bb24c4b0caea4566b1e77aaa18 Mon Sep 17 00:00:00 2001 From: hzmi Date: Tue, 20 Oct 2026 20:41:49 +0000 Subject: [PATCH] Automate cloudflared updates with multi-arch smoke tests Checks for new cloudflared releases every 6 hours and opens a pull request bumping CLOUDFLARED_VERSION, plus GOVERSION when the release needs a newer Go. Every platform is tested by running a hello world quick tunnel and requesting it, and updates that pass are merged after 24 hours unless someone objects. --- .github/scripts/smoke-test.sh | 108 +++++++++++++++ .github/scripts/update-versions.sh | 109 +++++++++++++++ .github/workflows/docker-build.yml | 56 ++++++++ .github/workflows/update-cloudflared.yml | 168 +++++++++++++++++++++++ 4 files changed, 441 insertions(+) create mode 100755 .github/scripts/smoke-test.sh create mode 100755 .github/scripts/update-versions.sh create mode 100644 .github/workflows/update-cloudflared.yml diff --git a/.github/scripts/smoke-test.sh b/.github/scripts/smoke-test.sh new file mode 100755 index 0000000..bba3f20 --- /dev/null +++ b/.github/scripts/smoke-test.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Usage: smoke-test.sh IMAGE [EXPECTED_VERSION] +# +# Runs IMAGE as a quick tunnel to cloudflared's built-in hello world server and +# requests it several times through trycloudflare.com, the same check that's +# done by hand before releasing a new version. +# +# Set PLATFORM to run an image for another architecture under QEMU, and +# QEMU_CPU to choose which CPU QEMU emulates. +set -euo pipefail + +image=$1 +expected=${2:-} +requests=${REQUESTS:-5} +name=cloudflared-smoke-test-$$ +url= + +# Without an init, a crash under QEMU leaves it hung as PID 1 instead of exiting +run_args=(--init) +if [ -n "${PLATFORM:-}" ]; then + run_args+=(--platform "$PLATFORM") +fi +if [ -n "${QEMU_CPU:-}" ]; then + run_args+=(-e "QEMU_CPU=$QEMU_CPU") +fi + +trap 'docker rm -f "$name" >/dev/null 2>&1 || true' EXIT + +fail() { + echo "::error::$*" + echo "--- cloudflared logs" + docker logs "$name" 2>&1 || true + exit 1 +} + +running() { [ "$(docker inspect -f '{{.State.Running}}' "$name" 2>/dev/null)" = true ]; } + +start_tunnel() { + local logs + docker rm -f "$name" >/dev/null 2>&1 || true + docker run -d --name "$name" "${run_args[@]}" "$image" --hello-world >/dev/null + for _ in $(seq 60); do + logs=$(docker logs "$name" 2>&1) + url=$(sed -n -E 's/.*\|[[:space:]]+(https:\/\/[a-z0-9-]+\.trycloudflare\.com).*/\1/p' <<<"$logs" | tail -n1) + if [ -n "$url" ] && grep -q 'Registered tunnel connection' <<<"$logs"; then + return 0 + fi + running || break + sleep 1 + done + echo "$logs" | tail -n 20 + return 1 +} + +# Resolve over DoH at first, so a lookup made before the record existed isn't +# cached. Lookups of new hostnames are flaky, so once one works, reuse its IP. +resolve=(--doh-url https://1.1.1.1/dns-query) + +request() { + local out ip + out=$(curl -fsS --max-time 15 "${resolve[@]}" -w '\n%{remote_ip}' "$url") || return 1 + ip=${out##*$'\n'} + if [[ $ip == *:* ]]; then + ip="[$ip]" + fi + resolve=(--resolve "${url#https://}:443:$ip") + grep -q 'Congrats! You created a tunnel!' <<<"$out" +} + +if [ -n "$expected" ]; then + if ! version=$(docker run --rm "${run_args[@]}" "$image" version 2>&1); then + echo "$version" + echo "::error::cloudflared crashed" + exit 1 + fi + echo "$version" + if [[ $version != *"cloudflared version $expected "* ]]; then + echo "::error::Expected cloudflared version $expected" + exit 1 + fi +fi + +# Quick tunnels are best effort, so give creating one a few tries +for attempt in 1 2 3; do + start_tunnel && break + echo "Couldn't start a quick tunnel (attempt $attempt)" + if [ "$attempt" -eq 3 ]; then + fail "Couldn't start a quick tunnel" + fi + sleep $((attempt * 10)) +done +echo "Tunnel up at $url" + +# New hostnames can take a moment to resolve, but once one request gets +# through, the rest all have to as well +deadline=$((SECONDS + 90)) +until request; do + [ "$SECONDS" -lt "$deadline" ] || fail "$url never became reachable" + sleep 3 +done +echo "Request 1/$requests OK" +for i in $(seq 2 "$requests"); do + request || fail "Request $i/$requests to $url failed" + echo "Request $i/$requests OK" +done + +running || fail "cloudflared exited during the test" +echo "Smoke test passed" diff --git a/.github/scripts/update-versions.sh b/.github/scripts/update-versions.sh new file mode 100755 index 0000000..bafebdb --- /dev/null +++ b/.github/scripts/update-versions.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Updates docker-bake.hcl to the latest cloudflared release, and Go to the +# newest golang alpine image that can build it. Results are printed as +# key=value lines, and also written to $GITHUB_OUTPUT when running in Actions. +# +# Set TARGET_VERSION to update to a specific release instead of the latest. +# Needs an authenticated gh, curl, jq and docker buildx. +set -euo pipefail + +cd "$(dirname "$0")/../.." + +version_re='^[0-9]{4}\.[0-9]+\.[0-9]+$' +go_re='^[0-9]+\.[0-9]+(\.[0-9]+)?$' + +# Bake variables can be overridden from the environment, so make sure none are +bake_args() { + env -u CLOUDFLARED_VERSION -u GOVERSION -u ALPINEVERSION \ + docker buildx bake --print --progress=quiet default | jq -c '.target.default.args' +} + +max_version() { printf '%s\n' "$@" | sort -V | tail -n1; } + +minor() { cut -d. -f1,2 <<<"$1"; } + +set_bake_var() { + sed -i -E "/^variable \"$1\" \{/,/^\}/ s/(default[[:space:]]*=[[:space:]]*)\"[^\"]*\"/\1\"$2\"/" docker-bake.hcl +} + +output() { + echo "$1=$2" + if [ -n "${GITHUB_OUTPUT:-}" ]; then + echo "$1=$2" >>"$GITHUB_OUTPUT" + fi +} + +args=$(bake_args) +old_version=$(jq -r .VERSION <<<"$args") +old_go=$(jq -r .GOVERSION <<<"$args") +alpine=$(jq -r .ALPINEVERSION <<<"$args") + +new_version=${TARGET_VERSION:-$(gh api repos/cloudflare/cloudflared/releases/latest --jq .tag_name)} +if ! [[ $new_version =~ $version_re ]]; then + echo "Unexpected cloudflared version: $new_version" >&2 + exit 1 +fi + +if [ "$new_version" = "$old_version" ]; then + echo "Already at cloudflared $old_version" + output changed false + exit 0 +fi +if [ -z "${TARGET_VERSION:-}" ] && [ "$(max_version "$old_version" "$new_version")" = "$old_version" ]; then + echo "Latest release $new_version is older than $old_version, not downgrading" + output changed false + exit 0 +fi + +# The go and toolchain lines in go.mod are the oldest Go that can build this +# release. golang images set GOTOOLCHAIN=local, so they won't fetch a newer one. +go_mod=$(gh api -H 'Accept: application/vnd.github.raw' "repos/cloudflare/cloudflared/contents/go.mod?ref=$new_version") +mapfile -t required < <(awk '$1 == "go" || $1 == "toolchain" { sub(/^go/, "", $2); print $2 }' <<<"$go_mod" | grep -E "$go_re") +if [ "${#required[@]}" -eq 0 ]; then + echo "Couldn't find a Go version in cloudflared $new_version's go.mod" >&2 + exit 1 +fi +required_go=$(max_version "${required[@]}") +go_minor=$(max_version "$(minor "$old_go")" "$(minor "$required_go")") + +# Images are published a while after Go releases, so use what's on Docker Hub +tags= +url="https://hub.docker.com/v2/namespaces/library/repositories/golang/tags?name=-alpine$alpine&page_size=100" +while [ "$url" != null ]; do + page=$(curl -fsS --retry 3 "$url") + tags+=$(jq -r '.results[].name' <<<"$page")$'\n' + url=$(jq -r .next <<<"$page") +done +mapfile -t candidates < <(sed -n -E "s/^(${go_minor//./\\.}\.[0-9]+)-alpine${alpine//./\\.}\$/\1/p" <<<"$tags") +if [ "${#candidates[@]}" -eq 0 ]; then + echo "No golang:$go_minor.x-alpine$alpine image exists yet, ALPINEVERSION may need bumping" >&2 + exit 1 +fi +new_go=$(max_version "${candidates[@]}" "$old_go") +if [ "$(max_version "$new_go" "$required_go")" != "$new_go" ]; then + echo "cloudflared $new_version needs Go $required_go, but the newest golang:$go_minor.x-alpine$alpine is $new_go" >&2 + exit 1 +fi + +set_bake_var CLOUDFLARED_VERSION "$new_version" +set_bake_var GOVERSION "$new_go" +# Builds without bake use the Dockerfile's default, so keep it on the same minor +sed -i -E "s/^ARG GOVERSION=.*/ARG GOVERSION=$(minor "$new_go")/" Dockerfile + +args=$(bake_args) +if [ "$(jq -r .VERSION <<<"$args")" != "$new_version" ] || [ "$(jq -r .GOVERSION <<<"$args")" != "$new_go" ]; then + echo "docker-bake.hcl didn't update as expected: $args" >&2 + exit 1 +fi + +title="Upgrade cloudflared to $new_version" +if [ "$new_go" != "$old_go" ]; then + title+=", Go to $new_go" +fi + +output changed true +output old_version "$old_version" +output version "$new_version" +output old_go "$old_go" +output go "$new_go" +output title "$title" diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index f434031..d586f6f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -3,6 +3,7 @@ name: docker-build on: push: pull_request: + workflow_dispatch: jobs: multi-arch-push: @@ -39,3 +40,58 @@ jobs: set: | *.cache-from=type=gha *.cache-to=type=gha,mode=max + + smoke-test: + name: smoke-test (${{ matrix.platform }}) + runs-on: ${{ matrix.runner || 'ubuntu-latest' }} + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + - platform: linux/386 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + # Emulate the oldest CPUs each image is meant for: Raspberry Pi 2 + # for v7, and Pi 1/Zero for v6 so ARMv7-only instructions get caught + - platform: linux/arm/v7 + qemu: true + qemu-cpu: cortex-a7 + - platform: linux/arm/v6 + qemu: true + qemu-cpu: arm1176 + - platform: linux/s390x + qemu: true + - platform: linux/ppc64le + qemu: true + - platform: linux/riscv64 + qemu: true + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Set up QEMU + if: ${{ matrix.qemu }} + uses: docker/setup-qemu-action@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - + name: Bake image + uses: docker/bake-action@v6 + with: + source: . + load: true + set: | + *.platform=${{ matrix.platform }} + *.tags=cloudflared:smoke-test + *.cache-from=type=gha + - + name: Run hello world tunnel + env: + PLATFORM: ${{ matrix.platform }} + QEMU_CPU: ${{ matrix.qemu-cpu }} + run: | + version=$(docker buildx bake --print --progress=quiet | jq -r .target.default.args.VERSION) + .github/scripts/smoke-test.sh cloudflared:smoke-test "$version" diff --git a/.github/workflows/update-cloudflared.yml b/.github/workflows/update-cloudflared.yml new file mode 100644 index 0000000..74fc6d0 --- /dev/null +++ b/.github/workflows/update-cloudflared.yml @@ -0,0 +1,168 @@ +name: update-cloudflared + +on: + schedule: + - cron: '17 */6 * * *' + workflow_dispatch: + inputs: + version: + description: cloudflared release to update to, if not the latest + required: false + +permissions: + contents: write + pull-requests: write + actions: write + +concurrency: update-cloudflared + +env: + BRANCH: update-cloudflared + BOT_EMAIL: 41898282+github-actions[bot]@users.noreply.github.com + # How long a passing update waits for objections before it's merged + GRACE_HOURS: 24 + +jobs: + merge: + runs-on: ubuntu-latest + steps: + - + name: Merge tested update + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + OWNER: ${{ github.repository_owner }} + run: | + pr=$(gh pr list --head "$BRANCH" --state open --json number,isCrossRepository --jq '[.[] | select(.isCrossRepository | not)][0].number // empty') + if [ -z "$pr" ]; then + exit 0 + fi + + info=$(gh pr view "$pr" --json headRefOid,labels,commits) + if [ "$(jq '[.commits[].authors[] | select(.email != env.BOT_EMAIL)] | length' <<<"$info")" != 0 ]; then + echo "::notice::#$pr has commits from someone else, so it needs merging by hand" + exit 0 + fi + if jq -e '.labels | any(.name == "hold")' <<<"$info" >/dev/null; then + echo "::notice::#$pr is on hold" + exit 0 + fi + sha=$(jq -r .headRefOid <<<"$info") + pushed=$(jq -r '.commits[-1].committedDate' <<<"$info") + + run=$(gh api "repos/$GH_REPO/actions/workflows/docker-build.yml/runs?branch=$BRANCH&event=workflow_dispatch&head_sha=$sha&per_page=1" --jq '.workflow_runs[0] // empty') + if [ -z "$run" ]; then + echo "No build of $sha found, starting one" + gh workflow run docker-build.yml --ref "$BRANCH" + exit 0 + fi + url=$(jq -r .html_url <<<"$run") + if [ "$(jq -r .status <<<"$run")" != completed ]; then + echo "The build of $sha is still running: $url" + exit 0 + fi + if [ "$(jq -r .conclusion <<<"$run")" != success ]; then + # Quick tunnels are best effort, so give a failure one retry + if [ "$(jq -r .run_attempt <<<"$run")" = 1 ]; then + echo "The build of $sha failed, retrying it: $url" + gh run rerun "$(jq -r .id <<<"$run")" --failed + exit 0 + fi + marker="" + if ! gh pr view "$pr" --json comments --jq '.comments[].body' | grep -qF "$marker"; then + gh pr comment "$pr" --body "@$OWNER The build or smoke test failed, so this won't be merged automatically: $url $marker" + fi + exit 0 + fi + + age=$(($(date +%s) - $(date -d "$pushed" +%s))) + if [ "$age" -lt $((GRACE_HOURS * 3600)) ]; then + echo "#$pr passed, it'll be merged $GRACE_HOURS hours after its last update" + exit 0 + fi + gh pr merge "$pr" --squash --match-head-commit "$sha" + # Merges made with GITHUB_TOKEN don't trigger the push build, so publish by hand + gh workflow run docker-build.yml --ref main + gh pr comment "$pr" --body "Passed and had no objections for $GRACE_HOURS hours, so merged automatically. [Publishing]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/workflows/docker-build.yml?query=branch%3Amain)" + + update: + needs: merge + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v4 + with: + ref: main + - + name: Update versions + id: update + env: + GH_TOKEN: ${{ github.token }} + TARGET_VERSION: ${{ inputs.version }} + run: .github/scripts/update-versions.sh + - + name: Open pull request + if: ${{ steps.update.outputs.changed == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + OLD_VERSION: ${{ steps.update.outputs.old_version }} + VERSION: ${{ steps.update.outputs.version }} + OLD_GO: ${{ steps.update.outputs.old_go }} + GO: ${{ steps.update.outputs.go }} + TITLE: ${{ steps.update.outputs.title }} + run: | + # --head also matches branches of the same name in forks + own='[.[] | select(.isCrossRepository | not)]' + + pr=$(gh pr list --head "$BRANCH" --state open --json number,isCrossRepository --jq "${own}[0].number // empty") + if [ -n "$pr" ]; then + # Once someone else pushes to the branch, e.g. to fix the build, it's + # theirs and this workflow stops force-pushing over it + others=$(gh pr view "$pr" --json commits --jq '[.commits[].authors[] | select(.email != env.BOT_EMAIL)] | length') + if [ "$others" != 0 ]; then + echo "::notice::#$pr has commits from someone else, leaving it alone" + exit 0 + fi + git fetch --quiet --depth=1 origin "$BRANCH" + if git diff --quiet FETCH_HEAD -- docker-bake.hcl Dockerfile; then + echo "#$pr is already up to date" + exit 0 + fi + elif gh pr list --head "$BRANCH" --state closed --limit 20 --json title,isCrossRepository --jq "${own}[].title" | + grep -qE "^Upgrade cloudflared to ${VERSION//./\\.}(,|$)"; then + echo "::notice::The update to $VERSION was closed without merging, not reopening it" + exit 0 + fi + + git config user.name 'github-actions[bot]' + git config user.email "$BOT_EMAIL" + git switch -C "$BRANCH" + git commit -am "$TITLE" + git push --force origin "$BRANCH" + + go="\`$OLD_GO\` (unchanged)" + if [ "$GO" != "$OLD_GO" ]; then + go="\`$OLD_GO\` → \`$GO\`" + fi + body=$(cat <