Skip to content

Commit 5f65dde

Browse files
authored
staged-images: Add chunkah-staged bootc base image builds (#152)
Add infrastructure to build rechunked bootc base images using chunkah. These 'staged' images mirror upstream fedora-bootc and centos-bootc, strip /sysroot (ostree data), and rechunk with content-based layers for optimal layer reuse across updates. Source image digests live in sources.json, with a Renovate custom regex manager to bump them automatically. A Python helper script (sources.py) handles all JSON queries and GHA matrix generation so the Justfile stays readable. Local usage: just staged-images/list just staged-images/build fedora-bootc-44 The CI workflow has three phases: mirror — skopeo copy --all from quay.io to GHCR so we own the copies (upstream deletes old manifests). build — per-arch (amd64 + arm64) chunkah builds on native runners, pushed by digest. manifest — assemble per-arch digests into multi-arch manifest lists using bootc-dev/actions/create-manifest. Target images: - ghcr.io/bootc-dev/fedora-bootc-staged:43 - ghcr.io/bootc-dev/fedora-bootc-staged:44 - ghcr.io/bootc-dev/centos-bootc-staged:stream9 - ghcr.io/bootc-dev/centos-bootc-staged:stream10 Closes: #151 Assisted-by: OpenCode (Claude Opus 4)
1 parent 86f10ae commit 5f65dde

6 files changed

Lines changed: 308 additions & 0 deletions

File tree

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
name: Build chunkah-staged bootc base images
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths:
7+
- 'staged-images/**'
8+
- '.github/workflows/build-staged-images.yml'
9+
schedule:
10+
# Rebuild weekly to pick up upstream base image updates
11+
- cron: '0 6 * * 1'
12+
workflow_dispatch:
13+
14+
concurrency:
15+
group: ${{ github.workflow }}-${{ github.ref }}
16+
cancel-in-progress: true
17+
18+
env:
19+
REGISTRY: ghcr.io
20+
21+
jobs:
22+
# Read sources.json and generate matrices for downstream jobs.
23+
generate-matrix:
24+
name: Generate matrix
25+
runs-on: ubuntu-24.04
26+
outputs:
27+
mirror: ${{ steps.matrix.outputs.mirror }}
28+
build: ${{ steps.matrix.outputs.build }}
29+
manifest: ${{ steps.matrix.outputs.manifest }}
30+
steps:
31+
- uses: actions/checkout@v6
32+
- uses: extractions/setup-just@v3
33+
- id: matrix
34+
run: |
35+
echo "mirror=$(just staged-images/ci-mirror-matrix)" >> "$GITHUB_OUTPUT"
36+
echo "build=$(just staged-images/ci-matrix)" >> "$GITHUB_OUTPUT"
37+
echo "manifest=$(just staged-images/ci-manifest-matrix)" >> "$GITHUB_OUTPUT"
38+
39+
# Mirror upstream source images to GHCR so we have our own copy.
40+
# Upstream registries (quay.io) may delete old manifests, breaking
41+
# digest-pinned pulls.
42+
mirror:
43+
name: Mirror ${{ matrix.name }}:${{ matrix.tag }}
44+
needs: generate-matrix
45+
runs-on: ubuntu-24.04
46+
permissions:
47+
contents: read
48+
packages: write
49+
strategy:
50+
fail-fast: false
51+
matrix: ${{ fromJson(needs.generate-matrix.outputs.mirror) }}
52+
steps:
53+
- uses: actions/checkout@v6
54+
- uses: bootc-dev/actions/bootc-ubuntu-setup@main
55+
- name: Log in to GHCR
56+
run: |
57+
echo "${{ secrets.GITHUB_TOKEN }}" | \
58+
podman login -u "${{ github.actor }}" --password-stdin ${{ env.REGISTRY }}
59+
- name: Mirror image
60+
run: just staged-images/mirror ${{ matrix.name }}-${{ matrix.tag }}
61+
env:
62+
REGISTRY_OWNER: ${{ github.repository_owner }}
63+
64+
build:
65+
name: Build ${{ matrix.name }}:${{ matrix.tag }} (${{ matrix.arch }})
66+
needs: [generate-matrix, mirror]
67+
if: ${{ !cancelled() }}
68+
runs-on: ${{ matrix.runner }}
69+
permissions:
70+
contents: read
71+
packages: write
72+
strategy:
73+
fail-fast: false
74+
matrix: ${{ fromJson(needs.generate-matrix.outputs.build) }}
75+
steps:
76+
- uses: actions/checkout@v6
77+
- uses: bootc-dev/actions/bootc-ubuntu-setup@main
78+
- name: Log in to GHCR
79+
run: |
80+
echo "${{ secrets.GITHUB_TOKEN }}" | \
81+
podman login -u "${{ github.actor }}" --password-stdin ${{ env.REGISTRY }}
82+
- name: Build staged image
83+
run: just staged-images/build ${{ matrix.image_key }}
84+
env:
85+
SOURCE_FROM_MIRROR: "1"
86+
REGISTRY_OWNER: ${{ github.repository_owner }}
87+
- name: Push by digest
88+
id: push
89+
run: |
90+
digest=$(just staged-images/push ${{ matrix.image_key }} ${{ matrix.arch }})
91+
echo "digest=${digest}" >> "$GITHUB_OUTPUT"
92+
env:
93+
REGISTRY_OWNER: ${{ github.repository_owner }}
94+
- name: Upload digest artifact
95+
run: |
96+
mkdir -p "${{ runner.temp }}/digests"
97+
echo "${{ steps.push.outputs.digest }}" > "${{ runner.temp }}/digests/${{ matrix.arch }}"
98+
- uses: actions/upload-artifact@v7
99+
with:
100+
name: staged-digests-${{ matrix.name }}-${{ matrix.tag }}-${{ matrix.arch }}
101+
path: ${{ runner.temp }}/digests/*
102+
if-no-files-found: error
103+
retention-days: 1
104+
105+
manifest:
106+
name: Manifest ${{ matrix.name }}:${{ matrix.tag }}
107+
needs: [generate-matrix, build]
108+
if: ${{ !cancelled() }}
109+
runs-on: ubuntu-24.04
110+
permissions:
111+
contents: read
112+
packages: write
113+
strategy:
114+
fail-fast: false
115+
matrix: ${{ fromJson(needs.generate-matrix.outputs.manifest) }}
116+
steps:
117+
- uses: bootc-dev/actions/bootc-ubuntu-setup@main
118+
- uses: bootc-dev/actions/create-manifest@main
119+
with:
120+
image: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ matrix.name }}
121+
tags: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ matrix.name }}:${{ matrix.tag }}
122+
artifact-pattern: staged-digests-${{ matrix.name }}-${{ matrix.tag }}-*
123+
registry-login-env: 'false'

renovate-shared-config.json

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,18 @@
5757
"# renovate: datasource=(?<datasource>[a-z-]+) depName=(?<depName>[^\\s]+)\\n\\s*(?:export )?\\w*VERSION=(?<currentValue>v?\\S+)"
5858
]
5959
},
60+
// Container image digest pinning in JSON config files
61+
// Matches patterns like:
62+
// "_renovate": "datasource=docker depName=quay.io/fedora/fedora-bootc",
63+
// ...
64+
// "source": "quay.io/fedora/fedora-bootc:43@sha256:abc123..."
65+
{
66+
"customType": "regex",
67+
"managerFilePatterns": ["**/sources.json"],
68+
"matchStrings": [
69+
"\"_renovate\":\\s*\"datasource=(?<datasource>docker) depName=(?<depName>[^\"]+)\"[^}]*\"source\":\\s*\"\\S+:(?<currentValue>[^@\\s\"]+)@(?<currentDigest>sha256:[a-f0-9]+)\""
70+
]
71+
},
6072
// Git refs (commit SHA) tracking in Justfiles and YAML workflows
6173
// Justfile example:
6274
// # renovate: datasource=git-refs depName=https://github.com/org/repo branch=main
@@ -134,6 +146,22 @@
134146
"groupName": "Docker",
135147
"enabled": true
136148
},
149+
// Group staged bootc base image digest updates separately
150+
//
151+
// These are the upstream source images for chunkah-staged builds.
152+
// Digest updates trigger a rebuild of the staged images, so they
153+
// get their own PR. Must come after the Docker group rule so it
154+
// takes precedence (Renovate applies all matching rules in order,
155+
// later rules win).
156+
{
157+
"description": ["Staged bootc base image digest updates"],
158+
"matchManagers": ["custom.regex"],
159+
"matchDepNames": [
160+
"quay.io/fedora/fedora-bootc",
161+
"quay.io/centos-bootc/centos-bootc"
162+
],
163+
"groupName": "staged-images"
164+
},
137165
// bcvk gets its own group so it isn't blocked by the weekly schedule
138166
// applied to other Docker group members. Without this, the Docker group
139167
// PR can only be created on Sundays (when all deps are in-schedule),

staged-images/.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
source-config.json
2+
out.ociarchive

staged-images/Containerfile

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Rechunk a bootc base image with chunkah
2+
#
3+
# Takes an upstream bootc image (fedora-bootc or centos-bootc), strips
4+
# the /sysroot (ostree data), and rechunks using chunkah for optimal
5+
# layer reuse across updates. The result is a "staged" base image
6+
# suitable for use as a FROM base in bootc development.
7+
#
8+
# Usage: just staged-images/build fedora-bootc-44
9+
10+
ARG SOURCE_IMAGE
11+
ARG CHUNKAH=quay.io/jlebon/chunkah:latest
12+
ARG MAX_LAYERS=128
13+
14+
FROM ${SOURCE_IMAGE} AS source
15+
16+
FROM ${CHUNKAH} AS chunkah
17+
ARG MAX_LAYERS
18+
RUN --mount=type=bind,target=/run/src,rw \
19+
--mount=from=source,target=/chunkah,ro \
20+
chunkah build \
21+
--config /run/src/source-config.json \
22+
--prune /sysroot/ \
23+
--max-layers "${MAX_LAYERS}" \
24+
--label ostree.commit- \
25+
--label ostree.final-diffid- \
26+
> /run/src/out.ociarchive
27+
28+
FROM oci-archive:out.ociarchive

staged-images/Justfile

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
_sources := justfile_directory() / "sources.json"
2+
registry := env("REGISTRY", "ghcr.io")
3+
registry_owner := env("REGISTRY_OWNER", "bootc-dev")
4+
5+
# Look up a field from sources.json by image key (e.g. fedora-bootc-43)
6+
[private]
7+
_field image field:
8+
@jq -re --arg n "{{image}}" '.[] | select(.name + "-" + .tag == $n) | .{{field}}' "{{_sources}}"
9+
10+
# List available staged images
11+
list:
12+
@jq -r '.[] | .name + "-" + .tag' "{{_sources}}"
13+
14+
# Mirror an upstream source image to our registry.
15+
# Usage: just staged-images/mirror fedora-bootc-43
16+
mirror image:
17+
#!/bin/bash
18+
set -euo pipefail
19+
name=$(just {{justfile_directory()}}/_field {{image}} name)
20+
tag=$(just {{justfile_directory()}}/_field {{image}} tag)
21+
src=$(just {{justfile_directory()}}/_field {{image}} source)
22+
dest="{{registry}}/{{registry_owner}}/${name}-source:${tag}"
23+
# skopeo doesn't support tag@digest, use digest-only form
24+
src_by_digest="${src%%:*}@${src##*@}"
25+
echo "Mirroring ${src_by_digest} -> ${dest}"
26+
skopeo copy --all --retry-times 3 "docker://${src_by_digest}" "docker://${dest}"
27+
echo "Mirrored ${dest}"
28+
29+
# Build a staged image locally.
30+
# Usage: just staged-images/build fedora-bootc-43
31+
# Set SOURCE_FROM_MIRROR=1 to pull from registry mirror instead of upstream.
32+
build image:
33+
#!/bin/bash
34+
set -euo pipefail
35+
name=$(just {{justfile_directory()}}/_field {{image}} name)
36+
tag=$(just {{justfile_directory()}}/_field {{image}} tag)
37+
src=$(just {{justfile_directory()}}/_field {{image}} source)
38+
staged_name="${name}-staged"
39+
if [ "${SOURCE_FROM_MIRROR:-}" = "1" ]; then
40+
src="{{registry}}/{{registry_owner}}/${name}-source:${tag}"
41+
fi
42+
echo "=== Pulling source image ==="
43+
podman pull "${src}"
44+
echo "=== Writing source config ==="
45+
podman inspect "${src}" > "{{justfile_directory()}}/source-config.json"
46+
echo "=== Building ${staged_name}:${tag} ==="
47+
# -v is needed for buildah < 1.44 (see containers/buildah#5952)
48+
buildah build --skip-unused-stages=false \
49+
-v "{{justfile_directory()}}:/run/src" --security-opt=label=disable \
50+
--build-arg SOURCE_IMAGE="${src}" \
51+
--build-arg MAX_LAYERS=128 \
52+
-f "{{justfile_directory()}}/Containerfile" \
53+
-t "localhost/${staged_name}:${tag}" \
54+
"{{justfile_directory()}}"
55+
echo "=== Verifying ==="
56+
echo "Labels:"
57+
podman inspect "localhost/${staged_name}:${tag}" | jq '.[0].Config.Labels'
58+
echo "Layer count:"
59+
podman inspect "localhost/${staged_name}:${tag}" | jq '.[0].RootFS.Layers | length'
60+
echo "Built localhost/${staged_name}:${tag}"
61+
62+
# Build all staged images
63+
build-all:
64+
#!/bin/bash
65+
set -euo pipefail
66+
for image in $(jq -r '.[] | .name + "-" + .tag' "{{_sources}}"); do
67+
just {{justfile_directory()}}/build "$image"
68+
done
69+
70+
# Push a built staged image by digest, print only the digest to stdout.
71+
# Usage: just staged-images/push fedora-bootc-43 amd64
72+
push image arch="":
73+
#!/bin/bash
74+
set -euo pipefail
75+
name=$(just {{justfile_directory()}}/_field {{image}} name)
76+
tag=$(just {{justfile_directory()}}/_field {{image}} tag)
77+
staged_name="${name}-staged"
78+
arch="{{arch}}"
79+
if [ -z "$arch" ]; then
80+
arch=$(podman info --format '{{{{.Host.Arch}}')
81+
fi
82+
dest="{{registry}}/{{registry_owner}}/${staged_name}"
83+
# Use a per-arch tag to avoid collisions when pushing in parallel
84+
push_tag="${tag}-${arch}"
85+
podman tag "localhost/${staged_name}:${tag}" "${dest}:${push_tag}" >&2
86+
digestfile=$(mktemp)
87+
podman push --retry 3 --digestfile "${digestfile}" "${dest}:${push_tag}" >&2
88+
digest=$(cat "${digestfile}")
89+
rm -f "${digestfile}"
90+
echo "${digest}"
91+
92+
# Generate GHA matrices from sources.json (used by CI workflow)
93+
[private]
94+
ci-matrix:
95+
@jq -c '[.[] | . as $img | {name: ($img.name + "-staged"), tag: $img.tag, image_key: ($img.name + "-" + $img.tag), arch: "amd64", runner: "ubuntu-24.04"}, {name: ($img.name + "-staged"), tag: $img.tag, image_key: ($img.name + "-" + $img.tag), arch: "arm64", runner: "ubuntu-24.04-arm"}] | {include: .}' "{{_sources}}"
96+
[private]
97+
ci-mirror-matrix:
98+
@jq -c '[.[] | {name: .name, tag: .tag, source: .source, mirror_name: (.name + "-source")}] | {include: .}' "{{_sources}}"
99+
[private]
100+
ci-manifest-matrix:
101+
@jq -c '[.[] | {name: (.name + "-staged"), tag: .tag}] | {include: .}' "{{_sources}}"

staged-images/sources.json

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
[
2+
{
3+
"_renovate": "datasource=docker depName=quay.io/fedora/fedora-bootc",
4+
"name": "fedora-bootc",
5+
"tag": "43",
6+
"source": "quay.io/fedora/fedora-bootc:43@sha256:dca83fb0b030b529394a129e82f0f75913d0b21f9da304b7ac048b1f12e48932"
7+
},
8+
{
9+
"_renovate": "datasource=docker depName=quay.io/fedora/fedora-bootc",
10+
"name": "fedora-bootc",
11+
"tag": "44",
12+
"source": "quay.io/fedora/fedora-bootc:44@sha256:4abd97abb04f600ba7a0eda950fd1d241d43bf472fd9fdd85e87a6be6cd5b1d1"
13+
},
14+
{
15+
"_renovate": "datasource=docker depName=quay.io/centos-bootc/centos-bootc",
16+
"name": "centos-bootc",
17+
"tag": "stream9",
18+
"source": "quay.io/centos-bootc/centos-bootc:stream9@sha256:2b96ee6f7157ed79c0f3db4d2c56686444e61733cabddc6229711c6a6b5dd673"
19+
},
20+
{
21+
"_renovate": "datasource=docker depName=quay.io/centos-bootc/centos-bootc",
22+
"name": "centos-bootc",
23+
"tag": "stream10",
24+
"source": "quay.io/centos-bootc/centos-bootc:stream10@sha256:43d2199dc2b147905ff3970b77957ac4775554b5ee6973abb577243a3fa28614"
25+
}
26+
]

0 commit comments

Comments
 (0)