From 65a9b48c8fd4d53c68bc4c81ff308efe10e7cc7a Mon Sep 17 00:00:00 2001 From: mforsyth Date: Tue, 13 May 2025 16:47:28 -0500 Subject: [PATCH 1/8] Add xmlsec build to Docker container In the Airflow update from 2.9 to 2.10, xmlsec was added as a dependency. xmlsec is a Python package that requires the xmlsec C library installed. This script installs the C library. --- Dockerfile | 6 +++ scripts/build_xmlsec_3_7.sh | 80 +++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 scripts/build_xmlsec_3_7.sh diff --git a/Dockerfile b/Dockerfile index 0607243..9e4335c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,12 @@ RUN set -ex && \ liblzma-dev \ default-libmysqlclient-dev +# Install xmlsec C library dependencies. This needs the build script to be copied into +# the container and run as root. THIS IS TEMPORARY AND CAN BE REMOVED ONCE THESE +# PACKAGES ARE UPDATED IN LINUX AND/OR THE JENKINS IMAGE. +COPY scripts/build_xmlsec_3_7.sh /tmp/build_xmlsec.sh +RUN chmod +x /tmp/build_xmlsec.sh && /tmp/build_xmlsec.sh + RUN chown -R jenkins ${JENKINS_HOME} ENV PYENV_ROOT $JENKINS_HOME/.pyenv diff --git a/scripts/build_xmlsec_3_7.sh b/scripts/build_xmlsec_3_7.sh new file mode 100644 index 0000000..cd072a8 --- /dev/null +++ b/scripts/build_xmlsec_3_7.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# This script installs xmlsec dependencies and should be run as root + +# The following explanation is for why we need to install pkg-config and other system +# packages. +# +# TL;DR: In the Airflow update from 2.9 to 2.10, xmlsec was added as a dependency. +# xmlsec is a Python package that requires the xmlsec C library installed. This script +# installs the C library. +# +# In the Jenkins container the `install-deps` target in the Makefile is called. The +# Makefile `install-deps` target is installing dependencies from the MWAA requirements +# file: docker/config/mwaa-base-providers-requirements.txt. In that file, it lists +# apache-airflow-providers-amazon[aiobotocore]==x.x.x. The trailing [aiobotocore] is +# not just pulling in aiobotocore itself, but also pulls in all of the Amazon-provider's +# extras, including the bits that power AWS SSO support. The AWS SSO hook in the Amazon +# provider depends on the Python xmlsec library under the hood (to handle SAML digital +# signatures). So we need xmlsec installed. +# +# Our pip is pointed to Dwolla's hosted PyPI artifact repository, and we don't have a +# wheel for xmlsec. Public PyPI (https://pypi.org/project/xmlsec/) has wheels for Mac +# and Windows, but not Linux. Because Linux distributions are so diverse, it's common +# for Linux wheels to be unavailable and require building from source. This is common +# for packages like xmlsec. Because of this we need to build xmlsec from source. +# +# In addition to this, the Python xmlsec package are binding to the C library, not the C +# library itself. So we need to build and install the C library too. This script +# installs the C library. The C library requires all of the dependencies listed below. +# +# Difference between and -dev: +# libxml2 is for running software that uses libxml2. +# libxml2-dev is for building or compiling software against libxml2 (which is the +# case for building Python wheels for packages like xmlsec). +# +# After installing, the `rm` cleans up the local repository of retrieved package files. +# This reduces the size of the Docker image by removing unnecessary cached files after +# installation. + +# Exit on error +set -e + +echo -e "\033[94mInstalling xmlsec C library dependencies\033[0m" + +# Install xmlsec dependencies +apt-get update +apt-get install -y pkg-config libxml2-dev libltdl-dev libxmlsec1-dev libxmlsec1-openssl + +# Clean up apt cache to reduce image size +rm -rf /var/lib/apt/lists/* + +# Explanation for this section: +# Because we need xmlsec from above there is a specific issue with getting xmlsec that +# we address here. We need to build and install xmlsec1 v1.3.x because when running +# apt-get for xmlsec1, version 1.3.7 is currently only available in the Debian +# experimental repository, not in the stable repository. The main Debian and Ubuntu +# distributions are still using 1.2.x versions. According to the Debian Package Tracker, +# version 1.3.7-1 was only accepted into experimental on March 25, 2025, with standard +# repositories still using 1.2.41-1. +# +# The public PyPI xmlsec wrapper (v1.3.x) expects newer C header files (including +# xmlSecKeyDataFormatEngine) introduced in the 1.3.x series. So we need to build and +# install the 1.3.x series. This whole script is temporary and can be removed once these +# packages are updated in Linux and/or the Jenkins image. +cd /tmp +curl -LO https://github.com/lsh123/xmlsec/releases/download/1.3.7/xmlsec1-1.3.7.tar.gz +tar xzf xmlsec1-1.3.7.tar.gz +cd xmlsec1-1.3.7 +# Prepare build with OpenSSL support +./configure --with-openssl +# Compile source code +make +# Install library to the system +make install +# Update dynamic linker cache so programs can find the library +ldconfig +cd / +# Clean up temporary files +rm -rf /tmp/xmlsec1-1.3.7* + +echo -e "\033[94mxmlsec C library dependencies installed successfully\033[0m" \ No newline at end of file From ead2c2a9e903081bda7d2b83ddf7bbf9cce96f1f Mon Sep 17 00:00:00 2001 From: mforsyth Date: Tue, 13 May 2025 17:01:39 -0500 Subject: [PATCH 2/8] Only run for Linux architectures --- Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9e4335c..cca51a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,8 +24,12 @@ RUN set -ex && \ # Install xmlsec C library dependencies. This needs the build script to be copied into # the container and run as root. THIS IS TEMPORARY AND CAN BE REMOVED ONCE THESE # PACKAGES ARE UPDATED IN LINUX AND/OR THE JENKINS IMAGE. -COPY scripts/build_xmlsec_3_7.sh /tmp/build_xmlsec.sh -RUN chmod +x /tmp/build_xmlsec.sh && /tmp/build_xmlsec.sh +# These only run for Linux architectures. +ARG TARGETPLATFORM +COPY --platform=linux/amd64,linux/arm64 scripts/build_xmlsec_3_7.sh /tmp/build_xmlsec.sh +RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] || [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ + chmod +x /tmp/build_xmlsec.sh && /tmp/build_xmlsec.sh; \ + fi RUN chown -R jenkins ${JENKINS_HOME} From f8f0beca0871e813fe353cd952bc0e9fb7185abf Mon Sep 17 00:00:00 2001 From: mforsyth Date: Wed, 14 May 2025 07:08:28 -0500 Subject: [PATCH 3/8] Remove platform flag, not supported --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index cca51a6..bf3ff5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,9 +24,9 @@ RUN set -ex && \ # Install xmlsec C library dependencies. This needs the build script to be copied into # the container and run as root. THIS IS TEMPORARY AND CAN BE REMOVED ONCE THESE # PACKAGES ARE UPDATED IN LINUX AND/OR THE JENKINS IMAGE. -# These only run for Linux architectures. ARG TARGETPLATFORM -COPY --platform=linux/amd64,linux/arm64 scripts/build_xmlsec_3_7.sh /tmp/build_xmlsec.sh +COPY scripts/build_xmlsec_3_7.sh /tmp/build_xmlsec.sh +# These only run for Linux architectures. RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] || [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ chmod +x /tmp/build_xmlsec.sh && /tmp/build_xmlsec.sh; \ fi From 91405d02f3abedd09d4460c0cd5c9475f2651e8f Mon Sep 17 00:00:00 2001 From: mforsyth Date: Wed, 14 May 2025 10:35:51 -0500 Subject: [PATCH 4/8] Remove 'apt-get update', in Dockerfile already --- scripts/build_xmlsec_3_7.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build_xmlsec_3_7.sh b/scripts/build_xmlsec_3_7.sh index cd072a8..8ac57d4 100644 --- a/scripts/build_xmlsec_3_7.sh +++ b/scripts/build_xmlsec_3_7.sh @@ -42,7 +42,6 @@ set -e echo -e "\033[94mInstalling xmlsec C library dependencies\033[0m" # Install xmlsec dependencies -apt-get update apt-get install -y pkg-config libxml2-dev libltdl-dev libxmlsec1-dev libxmlsec1-openssl # Clean up apt cache to reduce image size From 3e7920f9b9648aa10eccac6089a261624b2ce4f4 Mon Sep 17 00:00:00 2001 From: mforsyth Date: Wed, 14 May 2025 10:37:34 -0500 Subject: [PATCH 5/8] Update 'ldconfig' to continue script if it fails CI/CD environments like GitHub Actions typically run on x86_64/amd64 systems, and when they need to build for arm64, they rely on QEMU for emulation. QEMU emulation has known limitations with certain system calls, and ldconfig is one that commonly has issues in cross-architecture builds. For this situation, it is ok for ldconfig to fail with the arm build because the containers typically run on x86_64/amd64 systems in AWS. --- scripts/build_xmlsec_3_7.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/build_xmlsec_3_7.sh b/scripts/build_xmlsec_3_7.sh index 8ac57d4..e4582fb 100644 --- a/scripts/build_xmlsec_3_7.sh +++ b/scripts/build_xmlsec_3_7.sh @@ -70,8 +70,19 @@ cd xmlsec1-1.3.7 make # Install library to the system make install -# Update dynamic linker cache so programs can find the library -ldconfig + +# Update dynamic linker cache so programs can find the library with ldconfig. +# - Use '|| true' to prevent script from failing if ldconfig fails. CI/CD environments +# like GitHub Actions typically run on x86_64/amd64 systems, and when they need to +# build for arm64, they rely on QEMU for emulation. QEMU emulation has known +# limitations with certain system calls, and ldconfig is one that commonly has +# issues in cross-architecture builds. For this situation, it is ok for ldconfig to +# fail with the arm build because the containers typically run on x86_64/amd64 systems +# in AWS. +ldconfig || echo "Warning: ldconfig failed, continuing build. This is expected in " \ + "some emulated environments. See comment in scripts/build_xmlsec_3_7.sh for more" \ + "details." + cd / # Clean up temporary files rm -rf /tmp/xmlsec1-1.3.7* From 7ad8a355f0963b94c0b94a6aa281ac92cb8d3a33 Mon Sep 17 00:00:00 2001 From: mforsyth Date: Wed, 21 May 2025 15:31:46 -0500 Subject: [PATCH 6/8] Testing if ldconfig runs --- scripts/build_xmlsec_3_7.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/build_xmlsec_3_7.sh b/scripts/build_xmlsec_3_7.sh index e4582fb..0356d21 100644 --- a/scripts/build_xmlsec_3_7.sh +++ b/scripts/build_xmlsec_3_7.sh @@ -79,9 +79,10 @@ make install # issues in cross-architecture builds. For this situation, it is ok for ldconfig to # fail with the arm build because the containers typically run on x86_64/amd64 systems # in AWS. -ldconfig || echo "Warning: ldconfig failed, continuing build. This is expected in " \ - "some emulated environments. See comment in scripts/build_xmlsec_3_7.sh for more" \ - "details." +# ldconfig || echo "Warning: ldconfig failed, continuing build. This is expected in " \ +# "some emulated environments. See comment in scripts/build_xmlsec_3_7.sh for more" \ +# "details." +ldconfig cd / # Clean up temporary files From d141900c2c6dc18c88b23ba65abd915b71206d72 Mon Sep 17 00:00:00 2001 From: mforsyth Date: Wed, 21 May 2025 15:36:05 -0500 Subject: [PATCH 7/8] Adding test prints --- scripts/build_xmlsec_3_7.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build_xmlsec_3_7.sh b/scripts/build_xmlsec_3_7.sh index 0356d21..230265b 100644 --- a/scripts/build_xmlsec_3_7.sh +++ b/scripts/build_xmlsec_3_7.sh @@ -71,6 +71,7 @@ make # Install library to the system make install +echo -e "\033[94mRunning ldconfig\033[0m" # Update dynamic linker cache so programs can find the library with ldconfig. # - Use '|| true' to prevent script from failing if ldconfig fails. CI/CD environments # like GitHub Actions typically run on x86_64/amd64 systems, and when they need to @@ -83,6 +84,7 @@ make install # "some emulated environments. See comment in scripts/build_xmlsec_3_7.sh for more" \ # "details." ldconfig +echo -e "\033[94mldconfig completed\033[0m" cd / # Clean up temporary files From bbc732265d9ad5060b5f95e474ab1f1e6ea7ef06 Mon Sep 17 00:00:00 2001 From: mforsyth Date: Thu, 22 May 2025 10:06:30 -0500 Subject: [PATCH 8/8] Run ldconfig normally, it's not failing It seems that the original issue was a red herring and not the actual problem. Now updating to run ldconfig normally. --- scripts/build_xmlsec_3_7.sh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/scripts/build_xmlsec_3_7.sh b/scripts/build_xmlsec_3_7.sh index 230265b..01604dd 100644 --- a/scripts/build_xmlsec_3_7.sh +++ b/scripts/build_xmlsec_3_7.sh @@ -71,20 +71,7 @@ make # Install library to the system make install -echo -e "\033[94mRunning ldconfig\033[0m" -# Update dynamic linker cache so programs can find the library with ldconfig. -# - Use '|| true' to prevent script from failing if ldconfig fails. CI/CD environments -# like GitHub Actions typically run on x86_64/amd64 systems, and when they need to -# build for arm64, they rely on QEMU for emulation. QEMU emulation has known -# limitations with certain system calls, and ldconfig is one that commonly has -# issues in cross-architecture builds. For this situation, it is ok for ldconfig to -# fail with the arm build because the containers typically run on x86_64/amd64 systems -# in AWS. -# ldconfig || echo "Warning: ldconfig failed, continuing build. This is expected in " \ -# "some emulated environments. See comment in scripts/build_xmlsec_3_7.sh for more" \ -# "details." ldconfig -echo -e "\033[94mldconfig completed\033[0m" cd / # Clean up temporary files