diff --git a/.dockerignore b/.dockerignore index b527cb132d..d1fa750bc2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,6 +21,7 @@ /scripts/ci/* !/scripts/ci/third-party-licenses.sh !/scripts/ci/render-node-licenses.mjs +!/scripts/ci/connector-plugins.sh **/target /target !/target/debug/iggy diff --git a/.github/actions/utils/docker-buildx/action.yml b/.github/actions/utils/docker-buildx/action.yml index dc318b6867..f41e337a23 100644 --- a/.github/actions/utils/docker-buildx/action.yml +++ b/.github/actions/utils/docker-buildx/action.yml @@ -46,6 +46,10 @@ inputs: description: "Single platform to build (e.g., linux/amd64). If set, builds only this platform without QEMU. Leave empty for multi-arch build." required: false default: "" + target: + description: "Dockerfile stage to build. Empty builds the default (last) stage. Used to select an image flavor from a multi-flavor Dockerfile." + required: false + default: "" gha-cache: description: "Whether to use GitHub Actions cache for Docker layers (disable to save cache space)" required: false @@ -342,6 +346,7 @@ runs: with: context: ${{ steps.ctx.outputs.context }} file: ${{ steps.config.outputs.dockerfile }} + target: ${{ inputs.target }} platforms: ${{ steps.platforms.outputs.platforms }} labels: ${{ steps.meta.outputs.labels }} cache-from: ${{ env.CACHE_FROM }} @@ -357,6 +362,7 @@ runs: with: context: ${{ steps.ctx.outputs.context }} file: ${{ steps.config.outputs.dockerfile }} + target: ${{ inputs.target }} platforms: ${{ steps.platforms.outputs.platforms }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/actions/utils/validate-third-party-licenses/action.yml b/.github/actions/utils/validate-third-party-licenses/action.yml index 711bbf2fbf..63acea7e69 100644 --- a/.github/actions/utils/validate-third-party-licenses/action.yml +++ b/.github/actions/utils/validate-third-party-licenses/action.yml @@ -66,9 +66,17 @@ runs: shell: bash run: ./scripts/ci/third-party-licenses.sh --validate --manifest core/bench/dashboard/server/Cargo.toml - - name: Validate iggy-connect bundle + - name: Validate iggy-connect bundle (runtime + connector plugins) shell: bash - run: ./scripts/ci/third-party-licenses.sh --validate --manifest core/connectors/runtime/Cargo.toml + run: | + # The fat image bundles the runtime plus every connector plugin, so + # validate the plugins' dependency closures too. The manifest list is + # derived from cargo metadata so a newly added connector is license + # validated on the PR that adds it, with no change here. + MANIFEST_FLAGS="$(scripts/ci/connector-plugins.sh --manifest-flags)" + ./scripts/ci/third-party-licenses.sh --validate \ + --manifest core/connectors/runtime/Cargo.toml \ + $MANIFEST_FLAGS - name: Validate Python wheel bundle shell: bash diff --git a/.github/config/publish.yml b/.github/config/publish.yml index 36376d34c6..878007a899 100644 --- a/.github/config/publish.yml +++ b/.github/config/publish.yml @@ -96,9 +96,24 @@ components: platforms: ["linux/amd64", "linux/arm64"] version_file: "core/connectors/runtime/Cargo.toml" version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"' - # Image ships only the runtime binary; plugin .so files are not bundled. + # Two published flavors from one Dockerfile. The fat flavor (empty suffix, + # runtime-fat stage) bundles every connector plugin and is the default; + # -slim (runtime stage) ships the runtime binary only. Each flavor keeps + # its own per-arch digest set and manifest tags. Components without a + # flavors list build their single default stage. + flavors: + - suffix: "" + target: runtime-fat + - suffix: "-slim" + target: runtime + # The fat image bundles the runtime plus every connector plugin, so any + # change under core/connectors/ must refresh :edge. crates: keeps + # sensitivity to shared workspace deps in the runtime's closure (e.g. + # iggy_common); paths: covers all plugin crates without a hardcoded + # per-plugin list, so a newly added connector is gated automatically. gate: crates: [iggy-connectors] + paths: [core/connectors] web-ui: tag_pattern: "^web-ui-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" diff --git a/.github/workflows/_build_rust_artifacts.yml b/.github/workflows/_build_rust_artifacts.yml index 56232ff1b0..2064cfcc71 100644 --- a/.github/workflows/_build_rust_artifacts.yml +++ b/.github/workflows/_build_rust_artifacts.yml @@ -46,8 +46,8 @@ on: connector_plugins: type: string required: false - default: "iggy_connector_elasticsearch_sink,iggy_connector_elasticsearch_source,iggy_connector_iceberg_sink,iggy_connector_postgres_sink,iggy_connector_postgres_source,iggy_connector_quickwit_sink,iggy_connector_random_source,iggy_connector_s3_sink,iggy_connector_stdout_sink,iggy_connector_surrealdb_sink" - description: "Comma-separated list of connector plugin crates to build as shared libraries" + default: "" + description: "Comma-separated list of connector plugin crates to build as shared libraries. Empty (default) derives the full cdylib set from scripts/ci/connector-plugins.sh so new connectors are picked up automatically." outputs: artifact_name: description: "Name of the uploaded artifact containing all artifacts" @@ -216,12 +216,25 @@ jobs: - name: Add Rust target run: rustup target add ${{ matrix.target }} + - name: Resolve connector plugin list + env: + OVERRIDE: ${{ inputs.connector_plugins }} + run: | + if [[ -n "$OVERRIDE" ]]; then + plugins="$OVERRIDE" + echo "Using connector plugin list from workflow input (override)" + else + plugins="$(scripts/ci/connector-plugins.sh --comma-names)" + echo "Derived connector plugin list from cargo metadata" + fi + echo "Plugins: $plugins" + echo "CONNECTOR_PLUGINS=$plugins" >> "$GITHUB_ENV" + - name: Build connector plugins run: | - plugins="${{ inputs.connector_plugins }}" pkg_flags=() - IFS=',' read -ra pkgs <<< "$plugins" + IFS=',' read -ra pkgs <<< "$CONNECTOR_PLUGINS" for pkg in "${pkgs[@]}"; do name="$(echo "$pkg" | xargs)" [[ -z "$name" ]] && continue @@ -238,13 +251,16 @@ jobs: outdir="dist/${target}" mkdir -p "${outdir}" - plugins="${{ inputs.connector_plugins }}" - IFS=',' read -ra pkgs <<< "$plugins" + IFS=',' read -ra pkgs <<< "$CONNECTOR_PLUGINS" for pkg in "${pkgs[@]}"; do lib_name="$(echo "$pkg" | xargs)" [[ -z "$lib_name" ]] && continue so_file="target/${target}/release/lib${lib_name}.so" - [[ -f "$so_file" ]] && cp "$so_file" "${outdir}/" + if [[ ! -f "$so_file" ]]; then + echo "::error::expected connector plugin artifact missing: ${so_file}. The plugin crate built no cdylib, or its name does not match lib.so." >&2 + exit 1 + fi + cp "$so_file" "${outdir}/" done tarball="iggy-connectors-${target}-${version}.tar.gz" diff --git a/.github/workflows/edge-release.yml b/.github/workflows/edge-release.yml index f98f99c84d..92a9414593 100644 --- a/.github/workflows/edge-release.yml +++ b/.github/workflows/edge-release.yml @@ -59,6 +59,17 @@ jobs: server_version=$(scripts/extract-version.sh rust-server) echo "server_version=${server_version}" >> "$GITHUB_OUTPUT" + - name: Render bundled connector plugin list + id: plugins + run: | + # Derive the plugin list from cargo metadata so the release notes + # stay in sync with what is actually built (see connector-plugins.sh). + { + echo "list<> "$GITHUB_OUTPUT" + - name: Download all artifacts uses: actions/download-artifact@v8 with: @@ -101,16 +112,7 @@ jobs: - `iggy-connectors` - The connectors runtime ## Connector plugins included (.so) - - `iggy_connector_elasticsearch_sink` - - `iggy_connector_elasticsearch_source` - - `iggy_connector_iceberg_sink` - - `iggy_connector_postgres_sink` - - `iggy_connector_postgres_source` - - `iggy_connector_quickwit_sink` - - `iggy_connector_random_source` - - `iggy_connector_s3_sink` - - `iggy_connector_stdout_sink` - - `iggy_connector_surrealdb_sink` + ${{ steps.plugins.outputs.list }} ## Downloads diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4dc6c84a24..030cffc617 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -399,21 +399,38 @@ jobs: // Output non-Docker, non-Rust targets (SDKs only) core.setOutput('non_docker_targets', JSON.stringify(nonDockerTargets.length ? { include: nonDockerTargets } : { include: [{ key: 'noop', type: 'noop' }] })); - // Build Docker matrix: components × platforms for native runner builds + // Build Docker matrix: components × flavors × platforms for native + // runner builds. A component may declare multiple image flavors + // (e.g. fat + slim) built from different Dockerfile stages; each + // flavor gets its own suffix and target stage. Components without a + // flavors list build a single default flavor (empty suffix, empty + // target = default stage). flavor_suffix/flavor_target ride on every + // matrix entry so the build and manifest jobs keep flavors' digest + // sets and tags separate. const platforms = [ { platform: 'linux/amd64', arch: 'amd64', runner: 'ubuntu-latest' }, { platform: 'linux/arm64', arch: 'arm64', runner: 'ubuntu-24.04-arm' } ]; + const flavorsOf = (key) => { + const fl = cfg[key] && cfg[key].flavors; + return Array.isArray(fl) && fl.length ? fl : [{ suffix: '', target: '' }]; + }; + const dockerMatrix = []; + const dockerComponents = []; for (const t of dockerTargets) { - for (const p of platforms) { - dockerMatrix.push({ ...t, ...p }); + for (const f of flavorsOf(t.key)) { + const flavor = { flavor_suffix: f.suffix || '', flavor_target: f.target || '' }; + dockerComponents.push({ ...t, ...flavor }); + for (const p of platforms) { + dockerMatrix.push({ ...t, ...p, ...flavor }); + } } } core.setOutput('docker_matrix', JSON.stringify(dockerMatrix.length ? { include: dockerMatrix } : { include: [{ key: 'noop', type: 'noop' }] })); - core.setOutput('docker_components', JSON.stringify(dockerTargets.length ? { include: dockerTargets } : { include: [{ key: 'noop', type: 'noop' }] })); + core.setOutput('docker_components', JSON.stringify(dockerComponents.length ? { include: dockerComponents } : { include: [{ key: 'noop', type: 'noop' }] })); core.setOutput('has_docker', String(dockerTargets.length > 0)); core.setOutput('count', String(targets.length)); @@ -804,6 +821,7 @@ jobs: task: publish libc: ${{ steps.libc.outputs.libc }} component: ${{ matrix.key }} + target: ${{ matrix.flavor_target }} version: ${{ steps.ver.outputs.version }} platform: ${{ matrix.platform }} dry_run: ${{ inputs.dry_run }} @@ -827,7 +845,9 @@ jobs: if: ${{ !inputs.dry_run }} uses: actions/upload-artifact@v7 with: - name: docker-digest-${{ matrix.key }}-${{ matrix.arch }} + # flavor_suffix keeps each flavor's per-arch digests in a distinct + # artifact so the manifest job never merges flavors into one image. + name: docker-digest-${{ matrix.key }}${{ matrix.flavor_suffix }}-${{ matrix.arch }} path: ${{ runner.temp }}/digests/* if-no-files-found: error retention-days: 1 @@ -931,13 +951,13 @@ jobs: - name: Download amd64 digest uses: actions/download-artifact@v8 with: - name: docker-digest-${{ matrix.key }}-amd64 + name: docker-digest-${{ matrix.key }}${{ matrix.flavor_suffix }}-amd64 path: ${{ runner.temp }}/digests - name: Download arm64 digest uses: actions/download-artifact@v8 with: - name: docker-digest-${{ matrix.key }}-arm64 + name: docker-digest-${{ matrix.key }}${{ matrix.flavor_suffix }}-arm64 path: ${{ runner.temp }}/digests - name: Set up Docker Buildx @@ -956,16 +976,22 @@ jobs: run: | IMAGE="${{ steps.config.outputs.image }}" VERSION="${{ steps.ver.outputs.version }}" - - echo "Creating manifests for $IMAGE from digests:" + # Per-flavor tag suffix (e.g. -slim). Empty for the default flavor, + # so single-flavor components keep their existing tags unchanged. + SUFFIX="${{ matrix.flavor_suffix }}" + + # This job's digest dir holds only this flavor's per-arch digests + # (the upload/download names carry the same suffix), so globbing every + # digest here composes one manifest per flavor with no cross-merge. + echo "Creating manifests for ${IMAGE} (flavor '${SUFFIX:-default}') from digests:" ls -la if [ "${{ inputs.create_edge_docker_tag }}" = "true" ]; then # Auto-publish: :edge is the rolling tag and is ALWAYS refreshed. docker buildx imagetools create \ - -t "${IMAGE}:edge" \ + -t "${IMAGE}:edge${SUFFIX}" \ $(printf "${IMAGE}@sha256:%s " *) - echo "✅ Pushed manifest: ${IMAGE}:edge" + echo "✅ Pushed manifest: ${IMAGE}:edge${SUFFIX}" # The versioned :version manifest is part of the immutable release # and ships only alongside a new git tag. should_tag is false when @@ -974,26 +1000,26 @@ jobs: # refreshes :edge and nothing else. if [ "$SHOULD_TAG" = "true" ]; then docker buildx imagetools create \ - -t "${IMAGE}:${VERSION}" \ + -t "${IMAGE}:${VERSION}${SUFFIX}" \ $(printf "${IMAGE}@sha256:%s " *) - echo "✅ Pushed manifest: ${IMAGE}:${VERSION}" + echo "✅ Pushed manifest: ${IMAGE}:${VERSION}${SUFFIX}" else - echo "ℹ️ should_tag=false - :edge refreshed, skipping versioned :${VERSION} manifest" + echo "ℹ️ should_tag=false - :edge${SUFFIX} refreshed, skipping versioned :${VERSION}${SUFFIX} manifest" fi else # Manual publish: always push the versioned manifest, plus :latest # for stable (non edge/rc) releases. docker buildx imagetools create \ - -t "${IMAGE}:${VERSION}" \ + -t "${IMAGE}:${VERSION}${SUFFIX}" \ $(printf "${IMAGE}@sha256:%s " *) - echo "✅ Pushed manifest: ${IMAGE}:${VERSION}" + echo "✅ Pushed manifest: ${IMAGE}:${VERSION}${SUFFIX}" if [[ ! "$VERSION" =~ -(edge|rc) ]]; then - echo "Creating 'latest' manifest" + echo "Creating 'latest${SUFFIX}' manifest" docker buildx imagetools create \ - -t "${IMAGE}:latest" \ + -t "${IMAGE}:latest${SUFFIX}" \ $(printf "${IMAGE}@sha256:%s " *) - echo "✅ Pushed manifest: ${IMAGE}:latest" + echo "✅ Pushed manifest: ${IMAGE}:latest${SUFFIX}" fi fi @@ -1003,24 +1029,29 @@ jobs: run: | IMAGE="${{ steps.config.outputs.image }}" VERSION="${{ steps.ver.outputs.version }}" + SUFFIX="${{ matrix.flavor_suffix }}" # Auto-publish pushed :version only when should_tag was true; in # every other auto-publish case (:edge-only) inspect :edge instead. if [ "${{ inputs.create_edge_docker_tag }}" = "true" ] && [ "$SHOULD_TAG" != "true" ]; then - echo "Inspecting :edge manifest (versioned manifest was skipped: should_tag=false)" - docker buildx imagetools inspect "${IMAGE}:edge" + echo "Inspecting :edge${SUFFIX} manifest (versioned manifest was skipped: should_tag=false)" + docker buildx imagetools inspect "${IMAGE}:edge${SUFFIX}" else - docker buildx imagetools inspect "${IMAGE}:${VERSION}" + docker buildx imagetools inspect "${IMAGE}:${VERSION}${SUFFIX}" fi # Inline per-component tagging: tightly couple the git tag to the # multi-arch manifest that just shipped. should_tag was computed in the # version step above and already encodes the SNAPSHOT and auto-publish # stable-Docker skip rules. dry_run is gated at the job level. + # The git release tag is flavor-independent (one tag per component + # version), so only the default flavor creates it; other flavors ship + # their manifests but must not race to create the same tag. - name: Tag Docker release (${{ matrix.key }}) if: | success() && inputs.skip_tag_creation == false && + matrix.flavor_suffix == '' && steps.ver.outputs.should_tag == 'true' uses: ./.github/actions/utils/create-git-tag with: diff --git a/core/connectors/README.md b/core/connectors/README.md index 64bfc9a1f8..04aca6f311 100644 --- a/core/connectors/README.md +++ b/core/connectors/README.md @@ -2,7 +2,7 @@ The highly performant and modular runtime for statically typed, yet dynamically loaded connectors. Ingest the data from the external sources and push it further to the Iggy streams, or fetch the data from the Iggy streams and push it further to the external sources. Create your own Rust plugins by simply implementing either the `Source` or `Sink` trait and build custom pipelines for the data processing. -The [docker image](https://hub.docker.com/r/apache/iggy-connect) is available, and can be fetched via `docker pull apache/iggy-connect`. +The [docker image](https://hub.docker.com/r/apache/iggy-connect) is available via `docker pull apache/iggy-connect`. It ships in two flavors: a default image with every connector plugin bundled, and a `-slim` runtime-only image (see [Docker image](#docker-image)). ## Features @@ -18,6 +18,38 @@ The [docker image](https://hub.docker.com/r/apache/iggy-connect) is available, a - **Observability**: Prometheus metrics with per-stage latency histograms, plus an opt-in per-batch tracing benchmark target. - **Structured logging**: Selectable text or JSON log format via `[logging]` configuration. +## Docker image + +Published to Docker Hub as [`apache/iggy-connect`](https://hub.docker.com/r/apache/iggy-connect) in two flavors: + +| Tag | Contents | +| --- | --- | +| `edge`, `x.y.z`, `latest` | Runtime plus every bundled connector plugin (default) | +| `edge-slim`, `x.y.z-slim`, `latest-slim` | Runtime binary only | + +The default (fat) image bakes every connector plugin into `/usr/local/lib`, which is on the runtime's [plugin search path](#plugin-path-resolution). A connector config can therefore load a bundled plugin by library file name alone, with no directory and no extension: + +```toml +# a connector definition in your config_dir +type = "sink" +key = "stdout" +name = "Stdout sink" +path = "libiggy_connector_stdout_sink" +``` + +Run the runtime with a config mounted in, pointing `IGGY_CONNECTORS_CONFIG_PATH` at it: + +```bash +docker run --rm \ + -v "$PWD/my_config:/config" \ + -e IGGY_CONNECTORS_CONFIG_PATH=/config/config.toml \ + apache/iggy-connect +``` + +The mounted `config.toml` must set `[connectors].config_dir` to the connector definitions inside the container, and each definition's `path` must resolve there; bundled plugins resolve by file name as shown above. + +Choose a `-slim` tag when you supply your own plugins and want a smaller image with a narrower dependency surface, and mount the plugin `.so` files into one of the search-path directories (for example `/usr/local/lib`). + ## Quick Start 1. Build the project in release mode (or debug, and update the connectors paths in the config accordingly), and make sure that the plugins specified in `core/connectors/runtime/example_config/connectors/` directory under `path` are available. The configuration must be provided in `toml` format, with files following the `{connector_name}_{type}[_v{N}].toml` naming convention. diff --git a/core/connectors/runtime/Dockerfile b/core/connectors/runtime/Dockerfile index 45daa8bbf3..71121634c8 100644 --- a/core/connectors/runtime/Dockerfile +++ b/core/connectors/runtime/Dockerfile @@ -20,11 +20,18 @@ ARG ALPINE_VERSION=3.23 FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-${RUST_VERSION}-alpine${ALPINE_VERSION} AS chef WORKDIR /app -RUN apk add --no-cache musl-dev pkgconfig +# bash + jq back scripts/ci/connector-plugins.sh, which the planner runs to +# derive the connector plugin set (see below). +RUN apk add --no-cache musl-dev pkgconfig bash jq FROM --platform=$BUILDPLATFORM chef AS planner COPY . . -RUN cargo chef prepare --recipe-path recipe.json +# Prepare the chef recipe and, from the same source snapshot, derive the +# connector plugin set (crate names + manifest paths) so the cook, build and +# license steps below stay in sync with the workspace without a hardcoded list. +RUN cargo chef prepare --recipe-path recipe.json && \ + scripts/ci/connector-plugins.sh --names > connector-plugins.txt && \ + scripts/ci/connector-plugins.sh --manifests > connector-manifests.txt FROM --platform=$BUILDPLATFORM chef AS builder ARG PROFILE=release @@ -45,6 +52,8 @@ RUN apk add --no-cache bash curl tar zig make autoconf automake libtool pkgconfi ENV PKG_CONFIG_ALLOW_CROSS=1 COPY --from=planner /app/recipe.json recipe.json +COPY --from=planner /app/connector-plugins.txt connector-plugins.txt +COPY --from=planner /app/connector-manifests.txt connector-manifests.txt # # Cook dependencies @@ -59,12 +68,13 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARG "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \ *) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" && exit 1 ;; \ esac && \ + PLUGIN_FLAGS="$(sed 's/^/-p /' connector-plugins.txt | tr '\n' ' ')" && \ if [ "$PROFILE" = "debug" ]; then \ cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET} --zigbuild \ - -p iggy-connectors; \ + -p iggy-connectors $PLUGIN_FLAGS; \ else \ cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET} --zigbuild --release \ - -p iggy-connectors; \ + -p iggy-connectors $PLUGIN_FLAGS; \ fi COPY . . @@ -82,28 +92,40 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARG "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \ *) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" && exit 1 ;; \ esac && \ - if [ "$PROFILE" = "debug" ]; then \ - cargo zigbuild --locked --target ${RUST_TARGET} --bin iggy-connectors && \ - cp /app/target/${RUST_TARGET}/debug/iggy-connectors /app/iggy-connectors; \ - else \ - cargo zigbuild --locked --target ${RUST_TARGET} --bin iggy-connectors --release && \ - cp /app/target/${RUST_TARGET}/release/iggy-connectors /app/iggy-connectors; \ - fi + PLUGIN_FLAGS="$(sed 's/^/-p /' connector-plugins.txt | tr '\n' ' ')" && \ + if [ "$PROFILE" = "debug" ]; then PROFILE_DIR=debug; PROFILE_FLAG=""; else PROFILE_DIR=release; PROFILE_FLAG="--release"; fi && \ + cargo zigbuild --locked --target ${RUST_TARGET} -p iggy-connectors $PLUGIN_FLAGS $PROFILE_FLAG && \ + cp /app/target/${RUST_TARGET}/${PROFILE_DIR}/iggy-connectors /app/iggy-connectors && \ + mkdir -p /app/plugins && \ + while IFS= read -r name; do \ + [ -z "$name" ] && continue; \ + so_file="/app/target/${RUST_TARGET}/${PROFILE_DIR}/lib${name}.so"; \ + if [ ! -f "$so_file" ]; then echo "expected plugin artifact missing: $so_file" >&2 && exit 1; fi; \ + cp "$so_file" /app/plugins/; \ + done < connector-plugins.txt # -# Generate third-party license manifest. Required by ASF release policy: +# Generate third-party license manifests. Required by ASF release policy: # convenience binaries that statically link third-party code MUST include # the full license text of every bundled crate inside the artifact. # +# Two manifests: LICENSE-binary covers the runtime only (slim image); +# LICENSE-binary-fat additionally covers every bundled plugin closure (fat +# image). Each image ships the manifest that matches what it actually bundles. +# RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC} \ --mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC} \ TARGET="$(uname -m)-unknown-linux-musl" && \ curl -sSfL "https://github.com/EmbarkStudios/cargo-about/releases/download/0.9.0/cargo-about-0.9.0-${TARGET}.tar.gz" \ | tar -xz -C /usr/local/bin --strip-components=1 "cargo-about-0.9.0-${TARGET}/cargo-about" && \ - ./scripts/ci/third-party-licenses.sh --generate --manifest core/connectors/runtime/Cargo.toml --output /app/LICENSE-binary + ./scripts/ci/third-party-licenses.sh --generate --manifest core/connectors/runtime/Cargo.toml --output /app/LICENSE-binary && \ + MANIFEST_FLAGS="$(sed 's/^/--manifest /' connector-manifests.txt | tr '\n' ' ')" && \ + ./scripts/ci/third-party-licenses.sh --generate --manifest core/connectors/runtime/Cargo.toml $MANIFEST_FLAGS --output /app/LICENSE-binary-fat # -# Final runtime - Debian trixie Slim +# Slim runtime - Debian trixie Slim. Runtime binary only, no plugins. +# Published as the `-slim` image flavor. Built explicitly with +# `--target runtime`. # FROM debian:trixie-slim AS runtime WORKDIR /app @@ -113,3 +135,13 @@ COPY --from=builder /app/LICENSE-binary /usr/share/doc/iggy-connect/LICENSE-bina COPY LICENSE NOTICE /usr/share/doc/iggy-connect/ ENTRYPOINT ["iggy-connectors"] + +# +# Fat runtime - slim plus every connector plugin baked into /usr/local/lib, +# which is already on the runtime plugin search path, so a config can load a +# plugin by name with no path. This is the default build target (last stage) +# and the default published image flavor. +# +FROM runtime AS runtime-fat +COPY --from=builder /app/plugins/ /usr/local/lib/ +COPY --from=builder /app/LICENSE-binary-fat /usr/share/doc/iggy-connect/LICENSE-binary diff --git a/core/connectors/runtime/README.md b/core/connectors/runtime/README.md index 1c1339f49c..a67417f17a 100644 --- a/core/connectors/runtime/README.md +++ b/core/connectors/runtime/README.md @@ -12,7 +12,7 @@ By default, runtime will look for the configuration file, to decide which connec To start the connector runtime, simply run `cargo run --bin iggy-connectors`. -The [docker image](https://hub.docker.com/r/apache/iggy-connect) is available, and can be fetched via `docker pull apache/iggy-connect`. +The [docker image](https://hub.docker.com/r/apache/iggy-connect) is available via `docker pull apache/iggy-connect`. It ships in two flavors: a default image with every connector plugin bundled, and a `-slim` runtime-only image. The minimal viable configuration requires at least the Iggy credentials to create 2 separate instances of producer & consumer connections, the state directory path where source connectors can store their optional state, and the connectors configuration provider settings. diff --git a/scripts/ci/connector-plugins.sh b/scripts/ci/connector-plugins.sh new file mode 100755 index 0000000000..5d7a6c6da4 --- /dev/null +++ b/scripts/ci/connector-plugins.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +# Single source of truth for the connector plugin set: every workspace crate +# under core/connectors/ that builds a cdylib (a .so the iggy-connectors +# runtime loads via dlopen). Derived from `cargo metadata` so a newly added +# connector is picked up automatically by every consumer: +# +# - .github/workflows/_build_rust_artifacts.yml (edge tarball plugin list) +# - core/connectors/runtime/Dockerfile (fat image build + bundle) +# - .github/actions/utils/validate-third-party-licenses (license gate) +# +# Output modes (one plugin per line unless noted): +# --names crate names (iggy_connector_postgres_sink) +# --comma-names crate names, one CSV line +# --package-flags cargo -p flags, one line (-p iggy_connector_... ...) +# --manifests repo-relative Cargo.toml paths +# --manifest-flags --manifest flags, one line (for third-party-licenses.sh) + +MODE="--names" +if [[ $# -gt 0 ]]; then + MODE="$1" +fi + +METADATA="$(cargo metadata --format-version 1 --no-deps)" +WORKSPACE_ROOT="$(jq -r '.workspace_root' <<<"$METADATA")" + +# cdylib packages whose manifest lives under core/connectors/. The path guard +# keeps a future non-connector cdylib elsewhere in the workspace out of the set. +NAMES="$(jq -r --arg root "$WORKSPACE_ROOT" ' + .packages[] + | select(.manifest_path | startswith($root + "/core/connectors/")) + | select(.targets[] | .kind[] == "cdylib") + | .name +' <<<"$METADATA" | sort -u)" + +if [[ -z "$NAMES" ]]; then + echo "connector-plugins: no cdylib plugin crates found under core/connectors/" >&2 + exit 1 +fi + +# Map each crate name back to its repo-relative manifest path. +manifest_for() { + jq -r --arg root "$WORKSPACE_ROOT" --arg name "$1" ' + .packages[] + | select(.name == $name) + | .manifest_path + | ltrimstr($root + "/") + ' <<<"$METADATA" +} + +case "$MODE" in + --names) + echo "$NAMES" + ;; + --comma-names) + paste -sd, - <<<"$NAMES" + ;; + --package-flags) + mapfile -t names_arr <<<"$NAMES" + printf '%s\n' "${names_arr[@]/#/-p }" | paste -sd' ' - + ;; + --manifests) + while IFS= read -r name; do manifest_for "$name"; done <<<"$NAMES" + ;; + --manifest-flags) + while IFS= read -r name; do echo "--manifest $(manifest_for "$name")"; done <<<"$NAMES" | paste -sd' ' - + ;; + *) + echo "connector-plugins: unknown mode '$MODE'" >&2 + echo "usage: $0 [--names|--comma-names|--package-flags|--manifests|--manifest-flags]" >&2 + exit 2 + ;; +esac