From 3aa5844c0cdacd477fba220ea47a0e65b7944095 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 17 Jun 2026 13:59:27 -0700 Subject: [PATCH] ci(java): parallelize macOS build, split publish; scope release lock updates The Java publish workflow built the macOS native lib and published the multi-platform JAR in a single job, so the macOS build was gated behind the Linux builds even though only the publish step depends on them. - Collapse the two near-identical Debian 10 Linux builds into a matrix. - Make the macOS job build-only and run it concurrently with the Linux builds. - Add a separate `publish` job on ubuntu-latest that gathers all three native libs and packages/deploys with `-Dskip.build.jni=true`. This no longer holds the macOS runner during publish. Also: - Drop the no-op `-P shade-jar` profile from the deploy command. Shading is an always-on plugin bound to the `package` phase; the flag only logged a "could not be activated" warning. - Use `cargo update --workspace` in the release scripts. Bare `cargo update` after the version bump refreshed all transitive dependencies, sweeping incidental bumps into release commits (e.g. brotli, hyper, jiff, zerocopy in https://github.com/lance-format/lance/actions/runs/26624032632/job/78456375994#step:5:2496). `--workspace` re-pins only the local crates whose versions changed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/java-publish.yml | 139 ++++++++++------------------- ci/create_release_branch.sh | 12 +-- ci/publish_beta.sh | 12 +-- ci/release_common.sh | 6 +- 4 files changed, 62 insertions(+), 107 deletions(-) diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml index a51cf969a87..2b22b60dc92 100644 --- a/.github/workflows/java-publish.yml +++ b/.github/workflows/java-publish.yml @@ -28,10 +28,24 @@ permissions: contents: read jobs: - linux-arm64: - name: Build on Linux Arm64 - runs-on: ubuntu-24.04-arm64-8x + build-linux: + name: Build on Linux ${{ matrix.arch }} + runs-on: ${{ matrix.runner }} timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - arch: x86-64 + runner: ubuntu-24.04 + docker_platform: linux/amd64 + protoc_arch: x86_64 + artifact: liblance_jni_linux_x86_64.zip + - arch: arm64 + runner: ubuntu-24.04-arm64-8x + docker_platform: linux/arm64 + protoc_arch: aarch_64 + artifact: liblance_jni_linux_arm_64.zip steps: - name: Checkout repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 @@ -41,9 +55,9 @@ jobs: uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Check glibc version outside docker run: ldd --version - - name: Build and run in Debian 10 Arm64 container + - name: Build and run in Debian 10 container run: | - docker run --platform linux/arm64 -v ${{ github.workspace }}:/workspace -w /workspace debian:10 bash -c " + docker run --platform ${{ matrix.docker_platform }} -v ${{ github.workspace }}:/workspace -w /workspace debian:10 bash -c " set -ex # Update sources.list to use archive repositories for Debian 10 (EOL) @@ -81,7 +95,7 @@ jobs: unzip # https://github.com/databendlabs/databend/issues/8035 - PROTOC_ZIP=protoc-3.15.0-linux-aarch_64.zip + PROTOC_ZIP=protoc-3.15.0-linux-${{ matrix.protoc_arch }}.zip curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.15.0/\$PROTOC_ZIP unzip -o \$PROTOC_ZIP -d /usr/local rm -f \$PROTOC_ZIP @@ -102,101 +116,44 @@ jobs: " - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: liblance_jni_linux_arm_64.zip + name: ${{ matrix.artifact }} path: java/lance-jni/target/release/liblance_jni.so retention-days: 1 if-no-files-found: error - linux-x86: - name: Build on Linux x86-64 - runs-on: ubuntu-24.04 + build-macos: + name: Build on MacOS Arm64 + runs-on: warp-macos-14-arm64-6x timeout-minutes: 60 steps: - name: Checkout repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: ref: ${{ inputs.ref || github.ref }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - - name: Check glibc version outside docker - run: ldd --version - - name: Build and run in Debian 10 X86-64 container - run: | - docker run --platform linux/amd64 -v ${{ github.workspace }}:/workspace -w /workspace debian:10 bash -c " - - set -ex - # Update sources.list to use archive repositories for Debian 10 (EOL) - echo 'deb http://archive.debian.org/debian/ buster main' > /etc/apt/sources.list - echo 'deb http://archive.debian.org/debian-security buster/updates main' >> /etc/apt/sources.list - echo 'deb http://archive.debian.org/debian/ buster-updates main' >> /etc/apt/sources.list - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --assume-yes \ - apt-transport-https \ - ca-certificates \ - curl \ - gpg \ - bash \ - less \ - openssl \ - libssl-dev \ - pkg-config \ - libsqlite3-dev \ - libsqlite3-0 \ - libreadline-dev \ - git \ - cmake \ - dh-autoreconf \ - clang \ - g++ \ - libc++-dev \ - libc++abi-dev \ - libprotobuf-dev \ - libncurses5-dev \ - libncursesw5-dev \ - libudev-dev \ - libhidapi-dev \ - zip \ - unzip - - # https://github.com/databendlabs/databend/issues/8035 - PROTOC_ZIP=protoc-3.15.0-linux-x86_64.zip - curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.15.0/\$PROTOC_ZIP - unzip -o \$PROTOC_ZIP -d /usr/local - rm -f \$PROTOC_ZIP - protoc --version - - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable - source \$HOME/.cargo/env - cargo --version - - cd java/lance-jni - - # https://github.com/rustls/rustls/issues/1967 - export CC=clang - export CXX=clang++ - ldd --version - - cargo build --release - " + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + - uses: Homebrew/actions/setup-homebrew@50b8c2ab4a835c38897ed2c56c293b07167c0b59 # master 2026-03-07 + - name: Install dependencies + run: brew install protobuf + - name: Build native lib + working-directory: java/lance-jni + run: cargo build --release - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: liblance_jni_linux_x86_64.zip - path: java/lance-jni/target/release/liblance_jni.so + name: liblance_jni_darwin_aarch64.zip + path: java/lance-jni/target/release/liblance_jni.dylib retention-days: 1 if-no-files-found: error - macos-arm64: - name: Build on MacOS Arm64 and release - runs-on: warp-macos-14-arm64-6x - timeout-minutes: 60 + publish: + name: Publish Java packages + runs-on: ubuntu-latest + timeout-minutes: 30 needs: - - linux-arm64 - - linux-x86 + - build-linux + - build-macos steps: - name: Checkout repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: ref: ${{ inputs.ref || github.ref }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - name: Set up Java 11 uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 with: @@ -208,18 +165,16 @@ jobs: server-password: SONATYPE_TOKEN gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }} - - uses: Homebrew/actions/setup-homebrew@50b8c2ab4a835c38897ed2c56c293b07167c0b59 # master 2026-03-07 - - name: Install dependencies - run: | - brew install protobuf - brew install gpg - - name: Download artifact + - name: Download artifacts uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 - name: Copy native libs run: | - mkdir -p ./java/target/classes/nativelib/linux-x86-64 ./java/target/classes/nativelib/linux-aarch64 + mkdir -p ./java/target/classes/nativelib/linux-x86-64 \ + ./java/target/classes/nativelib/linux-aarch64 \ + ./java/target/classes/nativelib/darwin-aarch64 cp ./liblance_jni_linux_x86_64.zip/liblance_jni.so ./java/target/classes/nativelib/linux-x86-64/liblance_jni.so cp ./liblance_jni_linux_arm_64.zip/liblance_jni.so ./java/target/classes/nativelib/linux-aarch64/liblance_jni.so + cp ./liblance_jni_darwin_aarch64.zip/liblance_jni.dylib ./java/target/classes/nativelib/darwin-aarch64/liblance_jni.dylib - name: Set github run: | git config --global user.email "Lance Github Runner" @@ -230,7 +185,7 @@ jobs: inputs.mode == 'dry_run' working-directory: java run: | - mvn --batch-mode -DskipTests -Drust.release.build=true package + mvn --batch-mode -DskipTests -Dskip.build.jni=true package - name: Publish with Java 11 if: | github.event_name == 'release' || @@ -240,14 +195,14 @@ jobs: echo "use-agent" >> ~/.gnupg/gpg.conf echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf export GPG_TTY=$(tty) - mvn --batch-mode -DskipTests -Drust.release.build=true -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -P deploy-to-ossrh -P shade-jar + mvn --batch-mode -DskipTests -Dskip.build.jni=true -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -P deploy-to-ossrh env: SONATYPE_USER: ${{ secrets.SONATYPE_USER }} SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }} report-failure: name: Report Workflow Failure runs-on: ubuntu-latest - needs: [linux-arm64, linux-x86, macos-arm64] + needs: [build-linux, build-macos, publish] if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch') permissions: contents: read diff --git a/ci/create_release_branch.sh b/ci/create_release_branch.sh index 9c7d9d3e58a..db88f5b6b24 100755 --- a/ci/create_release_branch.sh +++ b/ci/create_release_branch.sh @@ -229,9 +229,9 @@ else bump-my-version bump -vv --new-version "${RC_VERSION}" --no-tag patch # Update Cargo.lock files after version bump - cargo update - (cd python && cargo update) - (cd java/lance-jni && cargo update) + cargo update --workspace + (cd python && cargo update --workspace) + (cd java/lance-jni && cargo update --workspace) # Commit the RC version git add -A @@ -259,9 +259,9 @@ else bump-my-version bump -vv --new-version "${NEXT_VERSION}" --no-tag patch # Update Cargo.lock files after version bump - cargo update - (cd python && cargo update) - (cd java/lance-jni && cargo update) + cargo update --workspace + (cd python && cargo update --workspace) + (cd java/lance-jni && cargo update --workspace) git add -A git commit -m "chore: bump main to ${NEXT_VERSION} diff --git a/ci/publish_beta.sh b/ci/publish_beta.sh index f50798a52e0..06fa5c16a91 100644 --- a/ci/publish_beta.sh +++ b/ci/publish_beta.sh @@ -93,9 +93,9 @@ if [[ "${BRANCH}" == "main" ]] && [[ "${CURRENT_VERSION}" =~ -beta\.[0-9]+$ ]]; bump-my-version bump -vv --new-version "${NEXT_VERSION}" --no-tag patch # Update Cargo.lock files after version bump - cargo update - (cd python && cargo update) - (cd java/lance-jni && cargo update) + cargo update --workspace + (cd python && cargo update --workspace) + (cd java/lance-jni && cargo update --workspace) git add -A git commit -m "chore: bump to ${NEXT_VERSION} based on breaking change detection" @@ -133,9 +133,9 @@ echo "Bumping beta version" bump-my-version bump -vv prerelease_num # Update Cargo.lock files after version bump -cargo update -(cd python && cargo update) -(cd java/lance-jni && cargo update) +cargo update --workspace +(cd python && cargo update --workspace) +(cd java/lance-jni && cargo update --workspace) # Get new version NEW_VERSION=$(grep '^version = ' Cargo.toml | head -n1 | cut -d'"' -f2) diff --git a/ci/release_common.sh b/ci/release_common.sh index cd653212aae..573202d1689 100644 --- a/ci/release_common.sh +++ b/ci/release_common.sh @@ -29,9 +29,9 @@ bump_and_commit_version() { bump-my-version bump -vv --new-version "${NEW_VERSION}" --no-tag patch # Update Cargo.lock files after version bump - cargo update - (cd python && cargo update) - (cd java/lance-jni && cargo update) + cargo update --workspace + (cd python && cargo update --workspace) + (cd java/lance-jni && cargo update --workspace) git add -A git commit -m "${COMMIT_MESSAGE}"