From 076d6fb3596ca01a79d5d6617e0d38d6762429b1 Mon Sep 17 00:00:00 2001 From: eaydingol Date: Mon, 8 Dec 2025 10:29:17 +0300 Subject: [PATCH 1/4] Add so files from the specified version to the test image --- circleci/images/Makefile | 7 +- .../citusupgradetester/files/sbin/build-citus | 3 +- circleci/images/exttester/Dockerfile | 100 ++++++++++++++++++ 3 files changed, 107 insertions(+), 3 deletions(-) diff --git a/circleci/images/Makefile b/circleci/images/Makefile index d4a574d..06acefe 100644 --- a/circleci/images/Makefile +++ b/circleci/images/Makefile @@ -26,6 +26,9 @@ CITUS_UPGRADE_VERSIONS_17=v13.2.0 # Function to get Citus versions for a specific PG major version get_citus_versions = $(CITUS_UPGRADE_VERSIONS_$(1)) +# Function to get the last Citus version for a specific PG major version +get_last_citus_version = $(lastword $(CITUS_UPGRADE_VERSIONS_$(1))) + # code below creates targets for all postgres versions in PG_VERSIONS define make-image-targets # $1 = PG_VERSION @@ -52,12 +55,12 @@ push-all:: push-extbuilder-$1 push-extbuilder-all:: push-extbuilder-$1 build-exttester-$1: - docker build \ - exttester/ \ + docker build . \ -f exttester/Dockerfile \ --build-arg=PG_VERSION=$1 \ --build-arg=PG_MAJOR=$2 \ --build-arg=PG_VERSION_CLEAN=$3 \ + --build-arg=CITUS_VERSION="$(call get_last_citus_version,$2)" \ --tag=${DOCKER_REPO}/exttester:$3${TAG_SUFFIX} build-all:: build-exttester-$1 diff --git a/circleci/images/citusupgradetester/files/sbin/build-citus b/circleci/images/citusupgradetester/files/sbin/build-citus index 2ae9bac..15aa7ee 100755 --- a/circleci/images/citusupgradetester/files/sbin/build-citus +++ b/circleci/images/citusupgradetester/files/sbin/build-citus @@ -50,7 +50,8 @@ build_ext() { cd "${installdir}" && find . -type f -print > "${builddir}/files.lst" tar cvf "${basedir}/install-pg${pg_major}-citus${citus_version}.tar" `cat ${builddir}/files.lst` - + find usr/lib/postgresql/${pg_major}/lib -type f -name '*.so' -printf '%P\n' > "${builddir}/so-files.lst" + tar cvf "${basedir}/so-pg${pg_major}-citus${citus_version}.tar" -C usr/lib/postgresql/${pg_major}/lib -T "${builddir}/so-files.lst" cd "${builddir}" && rm -rf install files.lst && make clean } for citus_version in ${CITUS_VERSIONS} diff --git a/circleci/images/exttester/Dockerfile b/circleci/images/exttester/Dockerfile index 661cf3c..7bfbf36 100644 --- a/circleci/images/exttester/Dockerfile +++ b/circleci/images/exttester/Dockerfile @@ -84,6 +84,96 @@ COPY --from=dev-tools-builder /build/postgresql-${PG_VERSION_CLEAN}/build/src/te COPY --from=dev-tools-builder /build/postgresql-${PG_VERSION_CLEAN}/src/test/regress/ usr/lib/postgresql/${PG_MAJOR}/lib/regress/ RUN rm -rf usr/lib/postgresql/${PG_MAJOR}/lib/regress/*.c usr/lib/postgresql/${PG_MAJOR}/lib/regress/*.h +FROM buildpack-deps:bullseye AS build-citus + +# add unpriviliged user for tests +RUN useradd -ms /bin/bash circleci + +RUN <<'EOF' +# install dependencies +set -eux +apt-get update + +apt-get install -y --no-install-recommends \ + apt-transport-https \ + autoconf \ + build-essential \ + ca-certificates \ + curl \ + debian-archive-keyring \ + gcc \ + gnupg \ + gosu \ + libcurl4 \ + libcurl4-openssl-dev \ + libicu-dev \ + liblz4-1 \ + liblz4-dev \ + libreadline-dev \ + libselinux1-dev \ + libssl-dev \ + libxslt-dev \ + libzstd-dev \ + libzstd1 \ + locales \ + make \ + perl \ + +# clear apt cache +rm -rf /var/lib/apt/lists/* +EOF + +ARG PG_VERSION +ARG PG_MAJOR +ENV PG_VERSION=$PG_VERSION +ENV PG_MAJOR=$PG_MAJOR + +RUN <<'EOF' +# install postgres ecosystem for pg version: $PG_VERSION +set -eux + +# install key and repositories +curl -sf https://www.postgresql.org/media/keys/ACCC4CF8.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add - +echo "deb https://apt.postgresql.org/pub/repos/apt bullseye-pgdg main" >> /etc/apt/sources.list.d/postgresql.list +echo "deb https://apt-archive.postgresql.org/pub/repos/apt bullseye-pgdg-archive main" >> /etc/apt/sources.list.d/postgresql.list + +apt-get update + +# infer the pgdgversion of postgres based on the $PG_VERSION +pgdg_version=$(apt list -a postgresql-server-dev-${PG_MAJOR} 2>/dev/null | grep "${PG_VERSION}" | awk '{print $2}' | head -n1 ) + +apt-get install -y --no-install-recommends --allow-downgrades \ + libpq-dev=${pgdg_version} \ + libpq5=${pgdg_version} \ + postgresql-${PG_MAJOR}=${pgdg_version} \ + postgresql-client-${PG_MAJOR}=${pgdg_version} \ + postgresql-${PG_MAJOR}-dbgsym=${pgdg_version} \ + postgresql-server-dev-${PG_MAJOR}=${pgdg_version} + +# clear apt cache +rm -rf /var/lib/apt/lists/* +EOF + +# add postgress to the path +ENV PATH=/usr/lib/postgresql/$PG_MAJOR/bin/:$PATH + +# setup /var/run/postgresql for use with circleci +RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql + + +ARG CITUS_VERSION +ENV CITUS_VERSION=$CITUS_VERSION +ENV CITUS_VERSIONS=$CITUS_VERSION + +WORKDIR /build-citus/ +COPY citusupgradetester/files/sbin/build-citus . +RUN ./build-citus + + +USER circleci +WORKDIR /home/circleci + + FROM buildpack-deps:bullseye # add unpriviliged user for tests @@ -183,4 +273,14 @@ RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgres # copy the collected files from the collection container at once into the final container COPY --from=dev-tools-collection /collect/ / +ARG CITUS_VERSION +ENV CITUS_VERSION=$CITUS_VERSION + +# copy citus so files from the build-citus stage +WORKDIR /opt/citus-versions/${CITUS_VERSION} +COPY --from=build-citus /build-citus/so-pg${PG_MAJOR}-citus${CITUS_VERSION}.tar . +RUN tar xvf so-pg${PG_MAJOR}-citus${CITUS_VERSION}.tar +# cleanup build artifacts +RUN rm so-pg${PG_MAJOR}-citus${CITUS_VERSION}.tar + WORKDIR /home/circleci From ecde43c8a0784824467a3480c6e270b18bdece5e Mon Sep 17 00:00:00 2001 From: eaydingol Date: Thu, 18 Dec 2025 13:12:54 +0300 Subject: [PATCH 2/4] add build stage --- .../citusupgradetester/files/sbin/build-citus | 2 +- circleci/images/exttester/Dockerfile | 114 ++++-------------- 2 files changed, 22 insertions(+), 94 deletions(-) diff --git a/circleci/images/citusupgradetester/files/sbin/build-citus b/circleci/images/citusupgradetester/files/sbin/build-citus index 15aa7ee..6ec6fdc 100755 --- a/circleci/images/citusupgradetester/files/sbin/build-citus +++ b/circleci/images/citusupgradetester/files/sbin/build-citus @@ -52,7 +52,7 @@ build_ext() { tar cvf "${basedir}/install-pg${pg_major}-citus${citus_version}.tar" `cat ${builddir}/files.lst` find usr/lib/postgresql/${pg_major}/lib -type f -name '*.so' -printf '%P\n' > "${builddir}/so-files.lst" tar cvf "${basedir}/so-pg${pg_major}-citus${citus_version}.tar" -C usr/lib/postgresql/${pg_major}/lib -T "${builddir}/so-files.lst" - cd "${builddir}" && rm -rf install files.lst && make clean + cd "${builddir}" && rm -rf install files.lst so-files.lst && make clean } for citus_version in ${CITUS_VERSIONS} do diff --git a/circleci/images/exttester/Dockerfile b/circleci/images/exttester/Dockerfile index 7bfbf36..db4c1ba 100644 --- a/circleci/images/exttester/Dockerfile +++ b/circleci/images/exttester/Dockerfile @@ -84,97 +84,7 @@ COPY --from=dev-tools-builder /build/postgresql-${PG_VERSION_CLEAN}/build/src/te COPY --from=dev-tools-builder /build/postgresql-${PG_VERSION_CLEAN}/src/test/regress/ usr/lib/postgresql/${PG_MAJOR}/lib/regress/ RUN rm -rf usr/lib/postgresql/${PG_MAJOR}/lib/regress/*.c usr/lib/postgresql/${PG_MAJOR}/lib/regress/*.h -FROM buildpack-deps:bullseye AS build-citus - -# add unpriviliged user for tests -RUN useradd -ms /bin/bash circleci - -RUN <<'EOF' -# install dependencies -set -eux -apt-get update - -apt-get install -y --no-install-recommends \ - apt-transport-https \ - autoconf \ - build-essential \ - ca-certificates \ - curl \ - debian-archive-keyring \ - gcc \ - gnupg \ - gosu \ - libcurl4 \ - libcurl4-openssl-dev \ - libicu-dev \ - liblz4-1 \ - liblz4-dev \ - libreadline-dev \ - libselinux1-dev \ - libssl-dev \ - libxslt-dev \ - libzstd-dev \ - libzstd1 \ - locales \ - make \ - perl \ - -# clear apt cache -rm -rf /var/lib/apt/lists/* -EOF - -ARG PG_VERSION -ARG PG_MAJOR -ENV PG_VERSION=$PG_VERSION -ENV PG_MAJOR=$PG_MAJOR - -RUN <<'EOF' -# install postgres ecosystem for pg version: $PG_VERSION -set -eux - -# install key and repositories -curl -sf https://www.postgresql.org/media/keys/ACCC4CF8.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add - -echo "deb https://apt.postgresql.org/pub/repos/apt bullseye-pgdg main" >> /etc/apt/sources.list.d/postgresql.list -echo "deb https://apt-archive.postgresql.org/pub/repos/apt bullseye-pgdg-archive main" >> /etc/apt/sources.list.d/postgresql.list - -apt-get update - -# infer the pgdgversion of postgres based on the $PG_VERSION -pgdg_version=$(apt list -a postgresql-server-dev-${PG_MAJOR} 2>/dev/null | grep "${PG_VERSION}" | awk '{print $2}' | head -n1 ) - -apt-get install -y --no-install-recommends --allow-downgrades \ - libpq-dev=${pgdg_version} \ - libpq5=${pgdg_version} \ - postgresql-${PG_MAJOR}=${pgdg_version} \ - postgresql-client-${PG_MAJOR}=${pgdg_version} \ - postgresql-${PG_MAJOR}-dbgsym=${pgdg_version} \ - postgresql-server-dev-${PG_MAJOR}=${pgdg_version} - -# clear apt cache -rm -rf /var/lib/apt/lists/* -EOF - -# add postgress to the path -ENV PATH=/usr/lib/postgresql/$PG_MAJOR/bin/:$PATH - -# setup /var/run/postgresql for use with circleci -RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql - - -ARG CITUS_VERSION -ENV CITUS_VERSION=$CITUS_VERSION -ENV CITUS_VERSIONS=$CITUS_VERSION - -WORKDIR /build-citus/ -COPY citusupgradetester/files/sbin/build-citus . -RUN ./build-citus - - -USER circleci -WORKDIR /home/circleci - - -FROM buildpack-deps:bullseye +FROM buildpack-deps:bullseye AS setup-environment # add unpriviliged user for tests RUN useradd -ms /bin/bash circleci @@ -232,8 +142,10 @@ RUN locale-gen ARG PG_VERSION ARG PG_MAJOR +ARG CITUS_VERSION ENV PG_VERSION=$PG_VERSION ENV PG_MAJOR=$PG_MAJOR +ENV CITUS_VERSION=$CITUS_VERSION RUN <<'EOF' # install postgres ecosystem for pg version: $PG_VERSION @@ -270,11 +182,27 @@ ENV PATH=/usr/lib/postgresql/$PG_MAJOR/bin/:$PATH # setup /var/run/postgresql for use with circleci RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql -# copy the collected files from the collection container at once into the final container -COPY --from=dev-tools-collection /collect/ / + +FROM setup-environment AS build-citus + +ENV CITUS_VERSIONS=$CITUS_VERSION + +WORKDIR /build-citus/ +COPY citusupgradetester/files/sbin/build-citus . +RUN ./build-citus + + +FROM setup-environment ARG CITUS_VERSION +ARG PG_VERSION +ARG PG_MAJOR ENV CITUS_VERSION=$CITUS_VERSION +ENV PG_MAJOR=$PG_MAJOR +ENV PG_VERSION=$PG_VERSION + +# copy the collected files from the collection container at once into the final container +COPY --from=dev-tools-collection /collect/ / # copy citus so files from the build-citus stage WORKDIR /opt/citus-versions/${CITUS_VERSION} From cf83cdd23daad8f3b9d9358db15b14cb4fbc7039 Mon Sep 17 00:00:00 2001 From: eaydingol Date: Thu, 18 Dec 2025 13:50:52 +0300 Subject: [PATCH 3/4] fix copy statements --- circleci/images/exttester/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circleci/images/exttester/Dockerfile b/circleci/images/exttester/Dockerfile index db4c1ba..a59193e 100644 --- a/circleci/images/exttester/Dockerfile +++ b/circleci/images/exttester/Dockerfile @@ -50,7 +50,7 @@ RUN tar jxf "postgresql-${PG_VERSION_CLEAN}.tar.bz2" # apply optional patches that might be required for a successful testsuite WORKDIR /build/postgresql-${PG_VERSION_CLEAN}/ -COPY patches/ patches/ +COPY exttester/patches/ patches/ RUN <<'EOF' # apply postgres patches @@ -137,7 +137,7 @@ ENV PATH="/root/.cargo/bin:${PATH}" RUN cpanm install IPC::Run # make special locales available -COPY locale.gen /etc/locale.gen +COPY exttester/locale.gen /etc/locale.gen RUN locale-gen ARG PG_VERSION From 3ea2bbe476badd6b3ec0b86af3c7131b7eca1dfe Mon Sep 17 00:00:00 2001 From: eaydingol Date: Thu, 18 Dec 2025 15:15:53 +0300 Subject: [PATCH 4/4] skip build step if citus_version is empty --- circleci/images/exttester/Dockerfile | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/circleci/images/exttester/Dockerfile b/circleci/images/exttester/Dockerfile index a59193e..4afad9e 100644 --- a/circleci/images/exttester/Dockerfile +++ b/circleci/images/exttester/Dockerfile @@ -189,7 +189,7 @@ ENV CITUS_VERSIONS=$CITUS_VERSION WORKDIR /build-citus/ COPY citusupgradetester/files/sbin/build-citus . -RUN ./build-citus +RUN if [ -n "${CITUS_VERSION}" ]; then ./build-citus; else echo "Skipping citus build - CITUS_VERSION not set"; fi FROM setup-environment @@ -205,10 +205,17 @@ ENV PG_VERSION=$PG_VERSION COPY --from=dev-tools-collection /collect/ / # copy citus so files from the build-citus stage -WORKDIR /opt/citus-versions/${CITUS_VERSION} -COPY --from=build-citus /build-citus/so-pg${PG_MAJOR}-citus${CITUS_VERSION}.tar . -RUN tar xvf so-pg${PG_MAJOR}-citus${CITUS_VERSION}.tar -# cleanup build artifacts -RUN rm so-pg${PG_MAJOR}-citus${CITUS_VERSION}.tar +COPY --from=build-citus /build-citus/ /tmp/build-citus/ + +# Extract conditionally in shell +RUN <<'EOF' +set -eux +if [ -n "${CITUS_VERSION}" ] && [ -f "/tmp/build-citus/so-pg${PG_MAJOR}-citus${CITUS_VERSION}.tar" ]; then + mkdir -p /opt/citus-versions/${CITUS_VERSION} + cd /opt/citus-versions/${CITUS_VERSION} + tar xvf /tmp/build-citus/so-pg${PG_MAJOR}-citus${CITUS_VERSION}.tar +fi +EOF +RUN rm -rf /tmp/build-citus/ WORKDIR /home/circleci