From 044676cb941d65b8605bfa344c0bbf15befc3101 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Thu, 28 May 2026 15:06:15 +0200 Subject: [PATCH] ci: replace Vault CLI downloads with Vault HTTP API --- .gitlab/dockerhub-login.sh | 56 +++--------------------------- .gitlab/generate-appsec.php | 25 +++---------- .gitlab/generate-package.php | 2 +- .gitlab/upload-junit-to-datadog.sh | 42 +++------------------- 4 files changed, 14 insertions(+), 111 deletions(-) diff --git a/.gitlab/dockerhub-login.sh b/.gitlab/dockerhub-login.sh index 357f73d69b0..d52776e0453 100755 --- a/.gitlab/dockerhub-login.sh +++ b/.gitlab/dockerhub-login.sh @@ -2,18 +2,16 @@ set -e -export VAULT_VERSION="1.20.0" - echo "=== Setting up Docker Hub authentication ===" -# Determine architecture for binary downloads +# Determine architecture for jq binary download arch="$(uname -m)" case "${arch}" in x86_64) - vault_arch="amd64" + jq_arch="amd64" ;; aarch64|arm64) - vault_arch="arm64" + jq_arch="arm64" ;; *) echo "Warning: Unsupported architecture: ${arch}. Skipping Docker Hub authentication." >&2 @@ -27,7 +25,7 @@ if ! command -v jq > /dev/null 2>&1; then jq_path="/tmp/jq" - if ! curl -L --fail "https://github.com/jqlang/jq/releases/latest/download/jq-linux-${vault_arch}" \ + if ! curl -L --fail "https://github.com/jqlang/jq/releases/latest/download/jq-linux-${jq_arch}" \ --output "${jq_path}"; then echo "Warning: Failed to download jq. Skipping Docker Hub authentication." >&2 exit 0 @@ -37,53 +35,9 @@ if ! command -v jq > /dev/null 2>&1; then export PATH="/tmp:${PATH}" fi -# Install unzip if not already available -if ! command -v unzip > /dev/null 2>&1; then - echo "Installing unzip..." - if command -v apt-get > /dev/null 2>&1; then - apt-get update -qq && apt-get install -y -qq unzip > /dev/null 2>&1 || { - echo "Warning: Failed to install unzip. Skipping Docker Hub authentication." >&2 - exit 0 - } - elif command -v apk > /dev/null 2>&1; then - apk add --no-cache unzip > /dev/null 2>&1 || { - echo "Warning: Failed to install unzip. Skipping Docker Hub authentication." >&2 - exit 0 - } - else - echo "Warning: No package manager found to install unzip. Skipping Docker Hub authentication." >&2 - exit 0 - fi -fi - -# Install Vault if not already available -vault_cmd="vault" -if ! command -v vault > /dev/null 2>&1; then - echo "Installing Vault CLI..." - - vault_path="/tmp/vault" - vault_zip="${vault_path}.zip" - - if ! curl -L --fail "https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_${vault_arch}.zip" \ - --output "${vault_zip}"; then - echo "Warning: Failed to download Vault. Skipping Docker Hub authentication." >&2 - exit 0 - fi - - if ! unzip -q "${vault_zip}" -d /tmp; then - echo "Warning: Failed to extract Vault. Skipping Docker Hub authentication." >&2 - exit 0 - fi - - chmod +x "${vault_path}" - rm -f "${vault_zip}" - - vault_cmd="${vault_path}" -fi - # Fetch Docker Hub credentials from Vault echo "Fetching Docker Hub credentials from Vault..." -vaultoutput="$("${vault_cmd}" kv get --format=json kv/k8s/gitlab-runner/dd-trace-php/dockerhub)" || { +vaultoutput="$(curl -sf -H "X-Vault-Token:${VAULT_TOKEN}" "${VAULT_ADDR}/v1/kv/data/k8s/gitlab-runner/dd-trace-php/dockerhub")" || { echo "Warning: Failed to fetch Docker Hub credentials from Vault. Skipping Docker Hub authentication." >&2 exit 0 } diff --git a/.gitlab/generate-appsec.php b/.gitlab/generate-appsec.php index 3f3aff72e33..a978beb2f9c 100644 --- a/.gitlab/generate-appsec.php +++ b/.gitlab/generate-appsec.php @@ -260,12 +260,6 @@ rm codecov.SHA256SUM.sig codecov.SHA256SUM chmod +x codecov mv codecov /usr/local/bin/codecov - - | - echo "Installing vault for codecov token" - curl -o vault.zip https://releases.hashicorp.com/vault/1.20.0/vault_1.20.0_linux_amd64.zip - unzip vault.zip - mv vault /usr/local/bin/vault - rm vault.zip - | cd appsec/tests/integration CACHE_PATH=build/php-appsec-volume-caches-${ARCH}.tar.gz @@ -283,7 +277,7 @@ - | echo "Uploading helper-rust unit test coverage to codecov" cd "$CI_PROJECT_DIR" - if ! VAULT_OUTPUT=$(vault kv get --format=json kv/k8s/gitlab-runner/dd-trace-php/codecov); then + if ! VAULT_OUTPUT=$(curl -sf -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_ADDR/v1/kv/data/k8s/gitlab-runner/dd-trace-php/codecov"); then echo "ERROR: vault unreachable while fetching CODECOV_TOKEN; exiting 75 so GitLab auto-retries (see default retry.exit_codes in generate-common.php)" exit 75 fi @@ -335,12 +329,6 @@ rm codecov.SHA256SUM.sig codecov.SHA256SUM chmod +x codecov mv codecov /usr/local/bin/codecov - - | - echo "Installing vault for codecov token" - curl -o vault.zip https://releases.hashicorp.com/vault/1.20.0/vault_1.20.0_linux_amd64.zip - unzip vault.zip - mv vault /usr/local/bin/vault - rm vault.zip - | cd appsec/tests/integration CACHE_PATH=build/php-appsec-volume-caches-${ARCH}.tar.gz @@ -362,7 +350,7 @@ - | echo "Uploading helper-rust integration test coverage to codecov" cd "$CI_PROJECT_DIR" - if ! VAULT_OUTPUT=$(vault kv get --format=json kv/k8s/gitlab-runner/dd-trace-php/codecov); then + if ! VAULT_OUTPUT=$(curl -sf -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_ADDR/v1/kv/data/k8s/gitlab-runner/dd-trace-php/codecov"); then echo "ERROR: vault unreachable while fetching CODECOV_TOKEN; exiting 75 so GitLab auto-retries (see default retry.exit_codes in generate-common.php)" exit 75 fi @@ -402,17 +390,12 @@ script: - | echo "Installing dependencies" - cd /tmp - curl -o vault.zip https://releases.hashicorp.com/vault/1.20.0/vault_1.20.0_linux_amd64.zip - unzip vault.zip - sudo cp -v vault /usr/local/bin - cd - sudo sed -i 's|http://deb.debian.org/debian|http://archive.debian.org/debian|g; s|http://security.debian.org/debian-security|http://archive.debian.org/debian-security|g' /etc/apt/sources.list sudo apt-get update && sudo apt-get install -y jq gcovr llvm-17 clang-17 echo "Installing codecov" - CODECOV_TOKEN=$(vault kv get --format=json kv/k8s/gitlab-runner/dd-trace-php/codecov | jq -r .data.data.token) + CODECOV_TOKEN=$(curl -sf -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_ADDR/v1/kv/data/k8s/gitlab-runner/dd-trace-php/codecov" | jq -r .data.data.token) CODECOV_VERSION=0.6.1 CODECOV_ARCH=linux curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import @@ -462,7 +445,7 @@ echo "Uploading coverage to Datadog" cd "$CI_PROJECT_DIR" - DATADOG_API_KEY=$(vault kv get --format=json kv/k8s/gitlab-runner/dd-trace-php/datadoghq-api-key | jq -r .data.data.key) + DATADOG_API_KEY=$(curl -sf -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_ADDR/v1/kv/data/k8s/gitlab-runner/dd-trace-php/datadoghq-api-key" | jq -r .data.data.key) export DATADOG_API_KEY export DD_SITE="datadoghq.com" diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 8c68e9f2765..4936ee55d73 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1257,7 +1257,7 @@ # Install Python dependencies pip install -U pip virtualenv - - /tmp/vault kv get --format=json "kv/k8s/gitlab-runner/dd-trace-php/datadoghq-api-key" 2>/dev/null | python3 -c "import sys,json;print(json.load(sys.stdin)['data']['data']['key'])" > /tmp/.dd-api-key 2>/dev/null || true + - curl -sf -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_ADDR/v1/kv/data/k8s/gitlab-runner/dd-trace-php/datadoghq-api-key" 2>/dev/null | python3 -c "import sys,json;print(json.load(sys.stdin)['data']['data']['key'])" > /tmp/.dd-api-key 2>/dev/null || true - git clone https://github.com/DataDog/system-tests.git - mv packages/{datadog-setup.php,dd-library-php-*x86_64-linux-gnu.tar.gz} system-tests/binaries - cd system-tests diff --git a/.gitlab/upload-junit-to-datadog.sh b/.gitlab/upload-junit-to-datadog.sh index 00067683ad3..a256114df0d 100755 --- a/.gitlab/upload-junit-to-datadog.sh +++ b/.gitlab/upload-junit-to-datadog.sh @@ -6,7 +6,6 @@ export DATADOG_SITE="datadoghq.com" export DD_ENV="ci" export DD_SERVICE="${DD_SERVICE:-dd-trace-php-tests}" export VAULT_SECRET_PATH="kv/k8s/gitlab-runner/dd-trace-php/datadoghq-api-key" -export VAULT_VERSION="1.20.0" # Parse arguments for tags (e.g., component:tracer test.type:unit) TAGS="${*}" @@ -20,7 +19,6 @@ fi arch="$(uname -m)" case "${arch}" in x86_64) - vault_arch="amd64" datadog_ci_arch="x64" ;; aarch64|arm64) @@ -34,7 +32,7 @@ case "${arch}" in esac # Detect package manager and install dependencies -echo "Installing required dependencies (curl, jq, nodejs, npm, unzip)..." +echo "Installing required dependencies (curl, jq, nodejs, npm)..." is_alpine=false if command -v apk &> /dev/null; then @@ -63,7 +61,7 @@ elif command -v apt-get &> /dev/null; then echo "Installing packages individually..." # Install packages one by one, continue if some fail - for pkg in curl jq unzip nodejs npm; do + for pkg in curl jq nodejs npm; do if ! command -v $pkg &> /dev/null; then echo "Installing $pkg..." $use_sudo apt-get install -y $pkg || echo "Warning: Failed to install $pkg, continuing..." @@ -88,43 +86,11 @@ fi echo "Dependencies installed successfully" -# Install Vault if not already available -if ! command -v vault &> /dev/null; then - echo "Installing Vault CLI..." - - vault_path="/tmp/vault" - vault_zip="${vault_path}.zip" - - echo "Downloading Vault ${VAULT_VERSION} for ${vault_arch}..." - if ! curl -L --fail "https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_${vault_arch}.zip" \ - --output "${vault_zip}"; then - echo "Warning: Failed to download Vault. Skipping JUnit upload." >&2 - exit 0 - fi - - echo "Extracting Vault..." - if ! unzip -o -q "${vault_zip}" -d /tmp; then - echo "Warning: Failed to extract Vault. Skipping JUnit upload." >&2 - exit 0 - fi - - chmod +x "${vault_path}" - rm -f "${vault_zip}" - - echo "Vault installed successfully" -fi - # Fetch DATADOG_API_KEY from Vault if not already set if [[ -z "${DATADOG_API_KEY:-}" ]]; then echo "DATADOG_API_KEY not set, attempting to fetch from Vault..." - - # Use the downloaded vault binary if it exists, otherwise use system vault - vault_cmd="vault" - if [ -f "/tmp/vault" ]; then - vault_cmd="/tmp/vault" - fi - - DATADOG_API_KEY="$("${vault_cmd}" kv get --format=json "${VAULT_SECRET_PATH}" | jq -r '.data.data.key')" || { + vault_secret_api_path="${VAULT_SECRET_PATH#kv/}" + DATADOG_API_KEY="$(curl -sf -H "X-Vault-Token:${VAULT_TOKEN}" "${VAULT_ADDR}/v1/kv/data/${vault_secret_api_path}" | jq -r '.data.data.key')" || { echo "Warning: Failed to fetch DATADOG_API_KEY from Vault. Skipping JUnit upload." >&2 exit 0 }