Skip to content

Commit 433f020

Browse files
LockedThreadLockedThread
authored
Merge pull request #70 from LockedThread/feat/speedup-cicd
Feat/speedup cicd
2 parents c58060d + cc7daae commit 433f020

5 files changed

Lines changed: 209 additions & 36 deletions

File tree

‎.github/workflows/build-and-validate.yaml‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ on:
44
pull_request:
55
branches: [main, development]
66

7+
env:
8+
REGISTRY: ghcr.io
9+
PYTHON_BASE_IMAGE_NAME: ${{ github.repository }}/python-base
10+
711
jobs:
812
build-and-validate:
913
strategy:
@@ -12,20 +16,41 @@ jobs:
1216
include:
1317
- platform: linux/amd64
1418
runner: ubuntu-latest
19+
arch: amd64
1520
- platform: linux/arm64
1621
runner: ubuntu-24.04-arm
22+
arch: arm64
1723
runs-on: ${{ matrix.runner }}
1824
permissions:
1925
contents: read
26+
packages: read
2027
steps:
2128
- name: Checkout
2229
uses: actions/checkout@v6
2330

2431
- name: Set up Docker Buildx
2532
uses: docker/setup-buildx-action@v3
2633

34+
- name: Log into registry ${{ env.REGISTRY }}
35+
uses: docker/login-action@v3
36+
with:
37+
registry: ${{ env.REGISTRY }}
38+
username: ${{ github.actor }}
39+
password: ${{ secrets.GITHUB_TOKEN }}
40+
41+
- name: Get Python version from Dockerfile
42+
id: python
43+
run: |
44+
VERSION=$(grep -oP 'ARG PYTHON_VERSION=\K[0-9.]+' Dockerfile.python-base)
45+
echo "version=${VERSION}" >> $GITHUB_OUTPUT
46+
2747
- name: Build image (${{ matrix.platform }})
28-
run: docker build --platform ${{ matrix.platform }} -t gtsam_docker:latest .
48+
run: |
49+
LOWER_IMAGE_NAME=$(echo "${{ env.PYTHON_BASE_IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]')
50+
docker build \
51+
--platform ${{ matrix.platform }} \
52+
--build-arg PYTHON_BASE_IMAGE=${{ env.REGISTRY }}/${LOWER_IMAGE_NAME}:${{ steps.python.outputs.version }}-trixie-${{ matrix.arch }} \
53+
-t gtsam_docker:latest .
2954
3055
- name: Validate image (PlanarSLAM example)
3156
run: |

‎.github/workflows/build-python-base.yaml‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
name: Build Python Base Image
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
python_version:
7+
description: "Python version to build"
8+
required: true
9+
default: "3.11.2"
10+
push:
11+
paths:
12+
- "Dockerfile.python-base"
13+
branches:
14+
- main
15+
- development
16+
17+
env:
18+
REGISTRY: ghcr.io
19+
IMAGE_NAME: ${{ github.repository }}/python-base
20+
21+
jobs:
22+
build:
23+
strategy:
24+
matrix:
25+
platform: ["linux/amd64", "linux/arm64"]
26+
runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
27+
permissions:
28+
contents: read
29+
packages: write
30+
steps:
31+
- name: Checkout
32+
uses: actions/checkout@v6
33+
34+
- name: Set up Docker Buildx
35+
uses: docker/setup-buildx-action@v3
36+
37+
- name: Log into registry ${{ env.REGISTRY }}
38+
uses: docker/login-action@v3
39+
with:
40+
registry: ${{ env.REGISTRY }}
41+
username: ${{ github.actor }}
42+
password: ${{ secrets.GITHUB_TOKEN }}
43+
44+
- name: Set Python version
45+
id: python
46+
run: |
47+
# Use input if provided, otherwise extract from Dockerfile.python-base
48+
if [ -n "${{ inputs.python_version }}" ]; then
49+
echo "version=${{ inputs.python_version }}" >> $GITHUB_OUTPUT
50+
else
51+
VERSION=$(grep -oP 'ARG PYTHON_VERSION=\K[0-9.]+' Dockerfile.python-base)
52+
echo "version=${VERSION}" >> $GITHUB_OUTPUT
53+
fi
54+
55+
- name: Extract Docker metadata
56+
id: meta
57+
uses: docker/metadata-action@v5
58+
with:
59+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
60+
tags: |
61+
type=raw,value=${{ steps.python.outputs.version }}-trixie-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
62+
63+
- name: Build and push Python base image for ${{ matrix.platform }}
64+
uses: docker/build-push-action@v6
65+
with:
66+
context: .
67+
file: Dockerfile.python-base
68+
push: true
69+
provenance: false
70+
platforms: ${{ matrix.platform }}
71+
tags: ${{ steps.meta.outputs.tags }}
72+
labels: ${{ steps.meta.outputs.labels }}
73+
build-args: |
74+
PYTHON_VERSION=${{ steps.python.outputs.version }}
75+
76+
manifest:
77+
needs: build
78+
runs-on: ubuntu-latest
79+
permissions:
80+
contents: read
81+
packages: write
82+
steps:
83+
- name: Checkout
84+
uses: actions/checkout@v6
85+
86+
- name: Log into registry ${{ env.REGISTRY }}
87+
uses: docker/login-action@v3
88+
with:
89+
registry: ${{ env.REGISTRY }}
90+
username: ${{ github.actor }}
91+
password: ${{ secrets.GITHUB_TOKEN }}
92+
93+
- name: Set Python version
94+
id: python
95+
run: |
96+
if [ -n "${{ inputs.python_version }}" ]; then
97+
echo "version=${{ inputs.python_version }}" >> $GITHUB_OUTPUT
98+
else
99+
VERSION=$(grep -oP 'ARG PYTHON_VERSION=\K[0-9.]+' Dockerfile.python-base)
100+
echo "version=${VERSION}" >> $GITHUB_OUTPUT
101+
fi
102+
103+
- name: Create multi-arch manifest
104+
run: |
105+
LOWER_IMAGE_NAME=$(echo "${IMAGE_NAME}" | tr '[:upper:]' '[:lower:]')
106+
PYTHON_VERSION="${{ steps.python.outputs.version }}"
107+
108+
docker manifest create $REGISTRY/${LOWER_IMAGE_NAME}:${PYTHON_VERSION}-trixie \
109+
$REGISTRY/${LOWER_IMAGE_NAME}:${PYTHON_VERSION}-trixie-amd64 \
110+
$REGISTRY/${LOWER_IMAGE_NAME}:${PYTHON_VERSION}-trixie-arm64
111+
docker manifest push $REGISTRY/${LOWER_IMAGE_NAME}:${PYTHON_VERSION}-trixie
112+
113+
# Also tag as latest
114+
docker manifest create $REGISTRY/${LOWER_IMAGE_NAME}:latest \
115+
$REGISTRY/${LOWER_IMAGE_NAME}:${PYTHON_VERSION}-trixie-amd64 \
116+
$REGISTRY/${LOWER_IMAGE_NAME}:${PYTHON_VERSION}-trixie-arm64
117+
docker manifest push $REGISTRY/${LOWER_IMAGE_NAME}:latest

‎.github/workflows/release.yaml‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,17 @@ on:
1616
env:
1717
REGISTRY: ghcr.io
1818
IMAGE_NAME: ${{ github.repository }}
19+
PYTHON_BASE_IMAGE_NAME: ${{ github.repository }}/python-base
1920

2021
jobs:
2122
build:
2223
strategy:
2324
matrix:
24-
platform: [ "linux/amd64", "linux/arm64" ]
25+
include:
26+
- platform: linux/amd64
27+
arch: amd64
28+
- platform: linux/arm64
29+
arch: arm64
2530
# Use GitHub-hosted runner for amd64; arm64 uses partner runner (ensure ubuntu-24.04-arm is enabled for the repo/org)
2631
runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
2732
permissions:
@@ -42,15 +47,23 @@ jobs:
4247
username: ${{ github.actor }}
4348
password: ${{ secrets.GITHUB_TOKEN }}
4449

50+
- name: Get Python version and base image
51+
id: python
52+
run: |
53+
VERSION=$(grep -oP 'ARG PYTHON_VERSION=\K[0-9.]+' Dockerfile.python-base)
54+
echo "version=${VERSION}" >> $GITHUB_OUTPUT
55+
LOWER_BASE_IMAGE=$(echo "${{ env.PYTHON_BASE_IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]')
56+
echo "base_image=${LOWER_BASE_IMAGE}" >> $GITHUB_OUTPUT
57+
4558
# Use docker/metadata-action to generate tags with an architecture suffix
4659
- name: Extract Docker metadata
4760
id: meta
4861
uses: docker/metadata-action@v5
4962
with:
5063
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
5164
tags: |
52-
type=raw,value=latest-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
53-
type=raw,value=${{ inputs.version }}-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
65+
type=raw,value=latest-${{ matrix.arch }}
66+
type=raw,value=${{ inputs.version }}-${{ matrix.arch }}
5467
5568
- name: Build and push Docker image for ${{ matrix.platform }}
5669
uses: docker/build-push-action@v6
@@ -61,6 +74,8 @@ jobs:
6174
platforms: ${{ matrix.platform }}
6275
tags: ${{ steps.meta.outputs.tags }}
6376
labels: ${{ steps.meta.outputs.labels }}
77+
build-args: |
78+
PYTHON_BASE_IMAGE=${{ env.REGISTRY }}/${{ steps.python.outputs.base_image }}:${{ steps.python.outputs.version }}-trixie-${{ matrix.arch }}
6479
6580
validate:
6681
needs: build

‎Dockerfile‎

Lines changed: 6 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,16 @@
11
# Default build produces the "runtime" stage (slim). Use --target gtsam for a dev image with build tools and shell.
2-
FROM debian:trixie-20260112 AS dependencies
3-
ARG PYTHON_VERSION=3.11.2
2+
# Pre-built Python base image (build once with Dockerfile.python-base, push to registry)
3+
# To build locally without registry: docker build -f Dockerfile.python-base -t python-optimized:3.11.2-trixie .
4+
ARG PYTHON_BASE_IMAGE=python-optimized:3.11.2-trixie
5+
FROM ${PYTHON_BASE_IMAGE} AS dependencies
46

57
# Disable GUI prompts
68
ENV DEBIAN_FRONTEND=noninteractive
79

8-
# Install required build dependencies (single update for better layer caching)
10+
# Install GTSAM build dependencies (Python already installed in base image)
911
RUN apt-get update && apt-get install -y --no-install-recommends \
10-
ca-certificates \
11-
build-essential \
12-
wget \
13-
libssl-dev \
14-
libbz2-dev \
15-
libreadline-dev \
16-
libsqlite3-dev \
17-
libffi-dev \
18-
zlib1g-dev \
19-
libncursesw5-dev \
20-
tk-dev \
21-
libgdbm-dev \
22-
liblzma-dev \
2312
apt-utils \
13+
build-essential \
2414
libboost-all-dev \
2515
cmake \
2616
libtbb-dev \
@@ -36,22 +26,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
3626
# Set working directory
3727
WORKDIR /usr/src
3828

39-
# Install Python with --enable-shared
40-
RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \
41-
tar xvf Python-${PYTHON_VERSION}.tgz && \
42-
cd Python-${PYTHON_VERSION} && \
43-
./configure --enable-optimizations --with-lto --enable-shared && \
44-
make -j$(nproc) && \
45-
make install && \
46-
cd .. && \
47-
rm -rf Python-${PYTHON_VERSION} Python-${PYTHON_VERSION}.tgz && \
48-
ldconfig
49-
50-
# Ensure /usr/local/bin is in the PATH
51-
ENV PATH="/usr/local/bin:${PATH}"
52-
53-
RUN python3 -m pip install --no-cache-dir --upgrade pip
54-
5529
# Use git to clone gtsam and specific GTSAM version
5630
FROM alpine/git:2.52.0 AS gtsam-clone
5731

‎Dockerfile.python-base‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Pre-built optimized Python base image for GTSAM builds.
2+
# Build and push once per Python version:
3+
# docker build -f Dockerfile.python-base -t ghcr.io/yourorg/python-optimized:3.11.2-trixie .
4+
# docker push ghcr.io/yourorg/python-optimized:3.11.2-trixie
5+
FROM debian:trixie-20260112
6+
ARG PYTHON_VERSION=3.11.2
7+
8+
ENV DEBIAN_FRONTEND=noninteractive
9+
10+
# Install Python build dependencies
11+
RUN apt-get update && apt-get install -y --no-install-recommends \
12+
ca-certificates \
13+
build-essential \
14+
wget \
15+
libssl-dev \
16+
libbz2-dev \
17+
libreadline-dev \
18+
libsqlite3-dev \
19+
libffi-dev \
20+
zlib1g-dev \
21+
libncursesw5-dev \
22+
tk-dev \
23+
libgdbm-dev \
24+
liblzma-dev \
25+
&& rm -rf /var/lib/apt/lists/*
26+
27+
WORKDIR /usr/src
28+
29+
# Build Python with PGO and LTO optimizations (takes ~15 min but only done once)
30+
RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \
31+
tar xvf Python-${PYTHON_VERSION}.tgz && \
32+
cd Python-${PYTHON_VERSION} && \
33+
./configure --enable-optimizations --with-lto --enable-shared && \
34+
make -j$(nproc) && \
35+
make install && \
36+
cd .. && \
37+
rm -rf Python-${PYTHON_VERSION} Python-${PYTHON_VERSION}.tgz && \
38+
ldconfig
39+
40+
ENV PATH="/usr/local/bin:${PATH}"
41+
42+
RUN python3 -m pip install --no-cache-dir --upgrade pip

0 commit comments

Comments
 (0)