Skip to content

Commit 26f83c1

Browse files
cgwalterscgwalters
committed
staged-images: Add chunkah-staged bootc base image builds
Add infrastructure to build rechunked bootc base images using chunkah. These 'staged' images mirror upstream fedora-bootc and centos-bootc, strip /sysroot (ostree data), and rechunk with content-based layers for optimal layer reuse across updates. Closes: #151 Assisted-by: OpenCode (Claude Opus 4) Signed-off-by: Colin Walters <walters@verbum.org>
1 parent 86f10ae commit 26f83c1

7 files changed

Lines changed: 465 additions & 0 deletions

File tree

‎.github/workflows/build-staged-images.yml‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
name: Build chunkah-staged bootc base images
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths:
7+
- 'staged-images/**'
8+
- '.github/workflows/build-staged-images.yml'
9+
schedule:
10+
# Rebuild weekly to pick up upstream base image updates
11+
- cron: '0 6 * * 1'
12+
workflow_dispatch:
13+
14+
concurrency:
15+
group: ${{ github.workflow }}-${{ github.ref }}
16+
cancel-in-progress: true
17+
18+
env:
19+
REGISTRY: ghcr.io
20+
21+
jobs:
22+
# Read sources.json and generate matrices for downstream jobs.
23+
generate-matrix:
24+
name: Generate matrix
25+
runs-on: ubuntu-24.04
26+
outputs:
27+
mirror: ${{ steps.matrix.outputs.mirror }}
28+
build: ${{ steps.matrix.outputs.build }}
29+
manifest: ${{ steps.matrix.outputs.manifest }}
30+
steps:
31+
- uses: actions/checkout@v6
32+
- uses: extractions/setup-just@v3
33+
- id: matrix
34+
run: |
35+
echo "mirror=$(just staged-images/ci-mirror-matrix)" >> "$GITHUB_OUTPUT"
36+
echo "build=$(just staged-images/ci-matrix)" >> "$GITHUB_OUTPUT"
37+
echo "manifest=$(just staged-images/ci-manifest-matrix)" >> "$GITHUB_OUTPUT"
38+
39+
# Mirror upstream source images to GHCR so we have our own copy.
40+
# Upstream registries (quay.io) may delete old manifests, breaking
41+
# digest-pinned pulls.
42+
mirror:
43+
name: Mirror ${{ matrix.name }}:${{ matrix.tag }}
44+
needs: generate-matrix
45+
runs-on: ubuntu-24.04
46+
permissions:
47+
contents: read
48+
packages: write
49+
strategy:
50+
fail-fast: false
51+
matrix: ${{ fromJson(needs.generate-matrix.outputs.mirror) }}
52+
steps:
53+
- uses: actions/checkout@v6
54+
- uses: bootc-dev/actions/bootc-ubuntu-setup@main
55+
- name: Log in to GHCR
56+
run: |
57+
echo "${{ secrets.GITHUB_TOKEN }}" | \
58+
podman login -u "${{ github.actor }}" --password-stdin ${{ env.REGISTRY }}
59+
- name: Mirror image
60+
run: just staged-images/mirror ${{ matrix.name }}-${{ matrix.tag }}
61+
env:
62+
REGISTRY_OWNER: ${{ github.repository_owner }}
63+
64+
build:
65+
name: Build ${{ matrix.name }}:${{ matrix.tag }} (${{ matrix.arch }})
66+
needs: [generate-matrix, mirror]
67+
if: ${{ !cancelled() }}
68+
runs-on: ${{ matrix.runner }}
69+
permissions:
70+
contents: read
71+
packages: write
72+
strategy:
73+
fail-fast: false
74+
matrix: ${{ fromJson(needs.generate-matrix.outputs.build) }}
75+
steps:
76+
- uses: actions/checkout@v6
77+
- uses: bootc-dev/actions/bootc-ubuntu-setup@main
78+
- name: Log in to GHCR
79+
run: |
80+
echo "${{ secrets.GITHUB_TOKEN }}" | \
81+
podman login -u "${{ github.actor }}" --password-stdin ${{ env.REGISTRY }}
82+
- name: Build staged image
83+
run: just staged-images/build ${{ matrix.image_key }}
84+
env:
85+
SOURCE_FROM_MIRROR: "1"
86+
REGISTRY_OWNER: ${{ github.repository_owner }}
87+
- name: Push by digest
88+
id: push
89+
run: |
90+
digest=$(just staged-images/push ${{ matrix.image_key }} ${{ matrix.arch }})
91+
echo "digest=${digest}" >> "$GITHUB_OUTPUT"
92+
env:
93+
REGISTRY_OWNER: ${{ github.repository_owner }}
94+
- name: Upload digest artifact
95+
run: |
96+
mkdir -p "${{ runner.temp }}/digests"
97+
echo "${{ steps.push.outputs.digest }}" > "${{ runner.temp }}/digests/${{ matrix.arch }}"
98+
- uses: actions/upload-artifact@v7
99+
with:
100+
name: staged-digests-${{ matrix.name }}-${{ matrix.tag }}-${{ matrix.arch }}
101+
path: ${{ runner.temp }}/digests/*
102+
if-no-files-found: error
103+
retention-days: 1
104+
105+
manifest:
106+
name: Manifest ${{ matrix.name }}:${{ matrix.tag }}
107+
needs: [generate-matrix, build]
108+
if: ${{ !cancelled() }}
109+
runs-on: ubuntu-24.04
110+
permissions:
111+
contents: read
112+
packages: write
113+
strategy:
114+
fail-fast: false
115+
matrix: ${{ fromJson(needs.generate-matrix.outputs.manifest) }}
116+
steps:
117+
- uses: bootc-dev/actions/bootc-ubuntu-setup@main
118+
- uses: bootc-dev/actions/create-manifest@main
119+
with:
120+
image: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ matrix.name }}
121+
tags: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ matrix.name }}:${{ matrix.tag }}
122+
artifact-pattern: staged-digests-${{ matrix.name }}-${{ matrix.tag }}-*
123+
registry-login-env: 'false'

‎renovate-shared-config.json‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,18 @@
5757
"# renovate: datasource=(?<datasource>[a-z-]+) depName=(?<depName>[^\\s]+)\\n\\s*(?:export )?\\w*VERSION=(?<currentValue>v?\\S+)"
5858
]
5959
},
60+
// Container image digest pinning in JSON config files
61+
// Matches patterns like:
62+
// "_renovate": "datasource=docker depName=quay.io/fedora/fedora-bootc",
63+
// ...
64+
// "source": "quay.io/fedora/fedora-bootc:43@sha256:abc123..."
65+
{
66+
"customType": "regex",
67+
"managerFilePatterns": ["**/sources.json"],
68+
"matchStrings": [
69+
"\"_renovate\":\\s*\"datasource=(?<datasource>docker) depName=(?<depName>[^\"]+)\"[^}]*\"source\":\\s*\"\\S+:(?<currentValue>[^@\\s\"]+)@(?<currentDigest>sha256:[a-f0-9]+)\""
70+
]
71+
},
6072
// Git refs (commit SHA) tracking in Justfiles and YAML workflows
6173
// Justfile example:
6274
// # renovate: datasource=git-refs depName=https://github.com/org/repo branch=main
@@ -134,6 +146,22 @@
134146
"groupName": "Docker",
135147
"enabled": true
136148
},
149+
// Group staged bootc base image digest updates separately
150+
//
151+
// These are the upstream source images for chunkah-staged builds.
152+
// Digest updates trigger a rebuild of the staged images, so they
153+
// get their own PR. Must come after the Docker group rule so it
154+
// takes precedence (Renovate applies all matching rules in order,
155+
// later rules win).
156+
{
157+
"description": ["Staged bootc base image digest updates"],
158+
"matchManagers": ["custom.regex"],
159+
"matchDepNames": [
160+
"quay.io/fedora/fedora-bootc",
161+
"quay.io/centos-bootc/centos-bootc"
162+
],
163+
"groupName": "staged-images"
164+
},
137165
// bcvk gets its own group so it isn't blocked by the weekly schedule
138166
// applied to other Docker group members. Without this, the Docker group
139167
// PR can only be created on Sundays (when all deps are in-schedule),

‎staged-images/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
source-config.json
2+
out.ociarchive

‎staged-images/Containerfile‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Rechunk a bootc base image with chunkah
2+
#
3+
# Takes an upstream bootc image (fedora-bootc or centos-bootc), strips
4+
# the /sysroot (ostree data), and rechunks using chunkah for optimal
5+
# layer reuse across updates. The result is a "staged" base image
6+
# suitable for use as a FROM base in bootc development.
7+
#
8+
# Usage: just staged-images/build fedora-bootc-44
9+
10+
ARG SOURCE_IMAGE
11+
ARG CHUNKAH=quay.io/jlebon/chunkah:latest
12+
ARG MAX_LAYERS=128
13+
14+
FROM ${SOURCE_IMAGE} AS source
15+
16+
FROM ${CHUNKAH} AS chunkah
17+
ARG MAX_LAYERS
18+
RUN --mount=type=bind,target=/run/src,rw \
19+
--mount=from=source,target=/chunkah,ro \
20+
chunkah build \
21+
--config /run/src/source-config.json \
22+
--prune /sysroot/ \
23+
--max-layers "${MAX_LAYERS}" \
24+
--label ostree.commit- \
25+
--label ostree.final-diffid- \
26+
> /run/src/out.ociarchive
27+
28+
FROM oci-archive:out.ociarchive

‎staged-images/Justfile‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
_sources := justfile_directory() / "sources.py"
2+
registry := env("REGISTRY", "ghcr.io")
3+
registry_owner := env("REGISTRY_OWNER", "bootc-dev")
4+
5+
# List available staged images
6+
list:
7+
@python3 "{{_sources}}" list
8+
9+
# Mirror an upstream source image to our registry.
10+
# Usage: just staged-images/mirror fedora-bootc-43
11+
mirror image:
12+
#!/bin/bash
13+
set -euo pipefail
14+
name=$(python3 "{{_sources}}" field {{image}} name)
15+
tag=$(python3 "{{_sources}}" field {{image}} tag)
16+
ref=$(python3 "{{_sources}}" mirror-ref {{image}})
17+
dest="{{registry}}/{{registry_owner}}/${name}-source:${tag}"
18+
echo "Mirroring ${ref} -> ${dest}"
19+
skopeo copy --all --retry-times 3 "docker://${ref}" "docker://${dest}"
20+
echo "Mirrored ${dest}"
21+
22+
# Build a staged image locally.
23+
# Usage: just staged-images/build fedora-bootc-43
24+
# Set SOURCE_FROM_MIRROR=1 to pull from registry mirror instead of upstream.
25+
build image:
26+
#!/bin/bash
27+
set -euo pipefail
28+
name=$(python3 "{{_sources}}" field {{image}} name)
29+
tag=$(python3 "{{_sources}}" field {{image}} tag)
30+
src=$(python3 "{{_sources}}" field {{image}} source)
31+
staged_name="${name}-staged"
32+
if [ "${SOURCE_FROM_MIRROR:-}" = "1" ]; then
33+
src="{{registry}}/{{registry_owner}}/${name}-source:${tag}"
34+
fi
35+
echo "=== Pulling source image ==="
36+
podman pull "${src}"
37+
echo "=== Writing source config ==="
38+
podman inspect "${src}" > "{{justfile_directory()}}/source-config.json"
39+
echo "=== Building ${staged_name}:${tag} ==="
40+
# -v is needed for buildah < 1.44 (see containers/buildah#5952)
41+
buildah build --skip-unused-stages=false \
42+
-v "{{justfile_directory()}}:/run/src" --security-opt=label=disable \
43+
--build-arg SOURCE_IMAGE="${src}" \
44+
--build-arg MAX_LAYERS=128 \
45+
-f "{{justfile_directory()}}/Containerfile" \
46+
-t "localhost/${staged_name}:${tag}" \
47+
"{{justfile_directory()}}"
48+
echo "=== Verifying ==="
49+
echo "Labels:"
50+
podman inspect "localhost/${staged_name}:${tag}" | jq '.[0].Config.Labels'
51+
echo "Layer count:"
52+
podman inspect "localhost/${staged_name}:${tag}" | jq '.[0].RootFS.Layers | length'
53+
echo "Built localhost/${staged_name}:${tag}"
54+
55+
# Build all staged images
56+
build-all:
57+
#!/bin/bash
58+
set -euo pipefail
59+
for image in $(python3 "{{_sources}}" list); do
60+
just {{justfile_directory()}}/build "$image"
61+
done
62+
63+
# Push a built staged image by digest, print only the digest to stdout.
64+
# Usage: just staged-images/push fedora-bootc-43 amd64
65+
push image arch="":
66+
#!/bin/bash
67+
set -euo pipefail
68+
name=$(python3 "{{_sources}}" field {{image}} name)
69+
tag=$(python3 "{{_sources}}" field {{image}} tag)
70+
staged_name="${name}-staged"
71+
arch="{{arch}}"
72+
if [ -z "$arch" ]; then
73+
arch=$(podman info --format '{{{{.Host.Arch}}')
74+
fi
75+
dest="{{registry}}/{{registry_owner}}/${staged_name}"
76+
# Use a per-arch tag to avoid collisions when pushing in parallel
77+
push_tag="${tag}-${arch}"
78+
podman tag "localhost/${staged_name}:${tag}" "${dest}:${push_tag}" >&2
79+
digestfile=$(mktemp)
80+
podman push --retry 3 --digestfile "${digestfile}" "${dest}:${push_tag}" >&2
81+
digest=$(cat "${digestfile}")
82+
rm -f "${digestfile}"
83+
echo "${digest}"
84+
85+
# Generate GHA matrices (used by CI workflow)
86+
[private]
87+
ci-matrix:
88+
@python3 "{{_sources}}" ci-matrix
89+
[private]
90+
ci-mirror-matrix:
91+
@python3 "{{_sources}}" ci-mirror-matrix
92+
[private]
93+
ci-manifest-matrix:
94+
@python3 "{{_sources}}" ci-manifest-matrix

‎staged-images/sources.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
[
2+
{
3+
"_renovate": "datasource=docker depName=quay.io/fedora/fedora-bootc",
4+
"name": "fedora-bootc",
5+
"tag": "43",
6+
"source": "quay.io/fedora/fedora-bootc:43@sha256:dca83fb0b030b529394a129e82f0f75913d0b21f9da304b7ac048b1f12e48932"
7+
},
8+
{
9+
"_renovate": "datasource=docker depName=quay.io/fedora/fedora-bootc",
10+
"name": "fedora-bootc",
11+
"tag": "44",
12+
"source": "quay.io/fedora/fedora-bootc:44@sha256:4abd97abb04f600ba7a0eda950fd1d241d43bf472fd9fdd85e87a6be6cd5b1d1"
13+
},
14+
{
15+
"_renovate": "datasource=docker depName=quay.io/centos-bootc/centos-bootc",
16+
"name": "centos-bootc",
17+
"tag": "stream9",
18+
"source": "quay.io/centos-bootc/centos-bootc:stream9@sha256:2b96ee6f7157ed79c0f3db4d2c56686444e61733cabddc6229711c6a6b5dd673"
19+
},
20+
{
21+
"_renovate": "datasource=docker depName=quay.io/centos-bootc/centos-bootc",
22+
"name": "centos-bootc",
23+
"tag": "stream10",
24+
"source": "quay.io/centos-bootc/centos-bootc:stream10@sha256:43d2199dc2b147905ff3970b77957ac4775554b5ee6973abb577243a3fa28614"
25+
}
26+
]

0 commit comments

Comments
 (0)