diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index 536e3056e0..0fb8b28a31 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -20,12 +20,16 @@ description: Rust pre-merge testing and linting github iggy actions inputs: task: - description: "Task to run (check, check-msrv, fmt, clippy, sort, machete, doctest, verify-publish, test-1, test-2, compat, miri)" + description: "Task to run (check, check-msrv, fmt, clippy, sort, machete, doctest, verify-publish, test-build, test-run-N, compat, miri)" required: true component: description: "Component name (for context)" required: false default: "" + test-partition-count: + description: "Total number of nextest hash partitions" + required: false + default: "2" runs: using: "composite" @@ -33,6 +37,12 @@ runs: - name: Setup Rust with cache uses: ./.github/actions/utils/setup-rust-with-cache with: + # Only test jobs execute nextest. Avoid downloading it in lint, metadata, + # and cross-build jobs. + install-nextest: ${{ inputs.task == 'test-build' || startsWith(inputs.task, 'test-run-') }} + # These tasks neither compile nor execute workspace binaries. Test-run + # jobs still need dynamic libraries such as libhwloc at process load. + install-system-dependencies: ${{ inputs.task != 'fmt' && inputs.task != 'sort' && inputs.task != 'machete' }} # Miri builds against a nightly toolchain with a separate `target/miri` # subtree; isolate its cache from the stable `dev` namespace so the # two don't evict each other. @@ -42,10 +52,10 @@ runs: # the warm `~/.cargo` restore is a free win, and its 1.95 target # artifacts rebuild regardless since the rustc hash differs. shared-key: ${{ inputs.task == 'miri' && 'miri' || 'dev' }} - # fmt/sort/machete never build into target/, so the multi-GB cache - # restore is pure overhead. Compiling legs (check/clippy/doctest/test-*) - # keep it; a cold cache there recompiles the whole dep tree. - read-cache: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'machete') && 'false' || 'true' }} + # fmt/sort/machete and archived test-run jobs never build into target/, + # so the multi-GB cache restore is pure overhead. Compiling legs, + # including test-build, keep it to avoid rebuilding the dependency tree. + read-cache: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'machete' || startsWith(inputs.task, 'test-run-')) && 'false' || 'true' }} # Light legs fit in the runner's ~89 GiB default headroom; skip the # ~20-45s reclaim. Disk-heavy legs (coverage build + testcontainers # images on test-*, cross-builds, miri, verify-publish) keep it. @@ -79,18 +89,18 @@ runs: # Safety: cargo check/clippy run on the full workspace separately, catching all # compilation errors. This only scopes test BUILD and EXECUTION. - name: Fetch base branch for DAG analysis - if: startsWith(inputs.task, 'test-') + if: inputs.task == 'test-build' run: git fetch origin master --depth=1 2>/dev/null || true shell: bash - name: Install cargo-rail - if: startsWith(inputs.task, 'test-') + if: inputs.task == 'test-build' uses: taiki-e/install-action@v2 with: tool: cargo-rail - name: Compute affected crates (cargo-rail) - if: startsWith(inputs.task, 'test-') + if: inputs.task == 'test-build' run: | METADATA_JSON=$(cargo metadata --format-version 1 --no-deps 2>/dev/null || echo "{}") TOTAL_CRATES=$(echo "$METADATA_JSON" | jq '.workspace_members | length' 2>/dev/null || echo "?") @@ -110,8 +120,6 @@ runs: CRATE_COUNT=$(echo "$CRATES" | wc -l) # Build nextest filter expression for cargo nextest run (affected crates only) echo "$CRATES" | sed 's/^/package(/; s/$/)/' | paste -sd '|' | sed 's/|/ | /g' > /tmp/nextest-filter.txt - # Save affected-only -p flags for cargo test fallback (no nextest filter) - echo "$CRATES" | sed 's/^/-p /' | tr '\n' ' ' > /tmp/test-packages.txt # Build -p flags: affected crates + packages with bin/cdylib targets. # Binary packages: nextest sets CARGO_BIN_EXE_ from compiled # artifacts; cdylib packages: connector plugins loaded via dlopen. @@ -187,34 +195,23 @@ runs: shell: bash - name: Install dependencies for Rust tests - if: startsWith(inputs.task, 'test-') && runner.os == 'Linux' + if: startsWith(inputs.task, 'test-run-') && runner.os == 'Linux' run: | - sudo apt-get install --yes musl-tools gnome-keyring keyutils dbus-x11 libsecret-tools + sudo apt-get install --yes gnome-keyring keyutils dbus-x11 libsecret-tools rm -f $HOME/.local/share/keyrings/* shell: bash - name: Install cargo-llvm-cov - if: startsWith(inputs.task, 'test-') + if: inputs.task == 'test-build' || startsWith(inputs.task, 'test-run-') uses: taiki-e/install-action@v2 with: - tool: cargo-llvm-cov + tool: cargo-llvm-cov@0.8.7 - - name: Build and test with coverage - if: startsWith(inputs.task, 'test-') + - name: Build and archive tests with coverage + if: inputs.task == 'test-build' run: | - # Parse partition index from task name (test-1 -> hash:1/2, test-2 -> hash:2/2) - TASK="${{ inputs.task }}" - PARTITION_FLAG="" - if [[ "$TASK" =~ ^test-([0-9]+)$ ]]; then - PARTITION_INDEX="${BASH_REMATCH[1]}" - PARTITION_FLAG="--partition hash:${PARTITION_INDEX}/2" - echo "::notice::Running test partition ${PARTITION_INDEX}/2" - fi - - # Read DAG-based affected crate filter (computed in earlier step) NEXTEST_FILTER="" PACKAGE_FLAGS="" - TEST_PACKAGE_FLAGS="" TOTAL_CRATES="?" if [[ -f /tmp/nextest-filter.txt ]]; then NEXTEST_FILTER=$(cat /tmp/nextest-filter.txt) @@ -222,9 +219,6 @@ runs: if [[ -f /tmp/packages.txt ]]; then PACKAGE_FLAGS=$(cat /tmp/packages.txt) fi - if [[ -f /tmp/test-packages.txt ]]; then - TEST_PACKAGE_FLAGS=$(cat /tmp/test-packages.txt) - fi if [[ -f /tmp/total-crates.txt ]]; then TOTAL_CRATES=$(cat /tmp/total-crates.txt) fi @@ -237,17 +231,8 @@ runs: echo "::notice::Full workspace build (no DAG filter available)" fi - source <(cargo llvm-cov show-env --export-prefix) - - # Doris 4.0.3's start_be.sh hard-`exit 1`s unless vm.max_map_count >= 2000000. - # This kernel param can only be set on the host (no container can raise it), - # so we raise it here. Everything else about booting Doris — image, heap/mem - # caps, port mappings, BE-alive wait — lives in the testcontainers fixture - # at core/integration/tests/connectors/fixtures/doris/container.rs, so - # `cargo test` and CI follow exactly the same path. - if [[ "$RUNNER_OS" == "Linux" ]]; then - sudo sysctl -w vm.max_map_count=2000000 || true - fi + cargo llvm-cov show-env --sh > "$RUNNER_TEMP/llvm-cov-env.sh" + source "$RUNNER_TEMP/llvm-cov-env.sh" bins_start=$(date +%s) if [[ -n "$PACKAGE_FLAGS" ]]; then @@ -259,52 +244,84 @@ runs: bins_duration=$((bins_end - bins_start)) echo "::notice::Binaries and libraries built in ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))" + ARTIFACT_DIR="${RUNNER_TEMP}/rust-nextest-archive" + mkdir -p "$ARTIFACT_DIR" + compile_start=$(date +%s) if [[ -n "$PACKAGE_FLAGS" ]]; then - cargo test --locked --no-run $PACKAGE_FLAGS + cargo nextest archive --locked $PACKAGE_FLAGS \ + --archive-file "$ARTIFACT_DIR/rust-tests.tar.zst" else - cargo test --locked --no-run + cargo nextest archive --locked \ + --archive-file "$ARTIFACT_DIR/rust-tests.tar.zst" fi compile_end=$(date +%s) compile_duration=$((compile_end - compile_start)) - echo "::notice::Tests compiled in ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" + echo "::notice::Tests compiled and archived in ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" - # Start D-Bus and unlock keyring right before test execution to avoid - # gnome-keyring auto-locking the collection during the build phase. - # Previously this ran before `cargo build`, leaving a 7+ minute idle - # window that triggered org.freedesktop.Secret.Error.IsLocked ~10% of runs. - if [[ "$RUNNER_OS" == "Linux" ]]; then - eval $(dbus-launch --sh-syntax) - export DBUS_SESSION_BUS_ADDRESS - eval $(echo -n "test" | gnome-keyring-daemon --unlock --components=secrets) - echo -n "warmup" | secret-tool store --label="ci-warmup" ci-test warmup + if [[ -n "$NEXTEST_FILTER" ]]; then + printf '%s\n' "$NEXTEST_FILTER" > "$ARTIFACT_DIR/nextest-filter.txt" + else + printf '%s\n' 'all()' > "$ARTIFACT_DIR/nextest-filter.txt" fi - test_start=$(date +%s) - if command -v cargo-nextest &> /dev/null; then - if [[ -n "$NEXTEST_FILTER" ]]; then - cargo nextest run --locked --no-fail-fast --profile ci $PARTITION_FLAG $PACKAGE_FLAGS -E "$NEXTEST_FILTER" - else - cargo nextest run --locked --no-fail-fast --profile ci $PARTITION_FLAG - fi - else - if [[ -n "$PARTITION_FLAG" ]]; then - echo "::error::cargo-nextest not found, falling back to cargo test without partitioning (all tests will run on every partition)" - fi - # Use TEST_PACKAGE_FLAGS (affected crates only), not PACKAGE_FLAGS - # (which includes binary packages whose tests should NOT run). - if [[ -n "$TEST_PACKAGE_FLAGS" ]]; then - cargo test --locked --no-fail-fast $TEST_PACKAGE_FLAGS - else - cargo test --locked --no-fail-fast - fi + # Re-read metadata without fallbacks so an incomplete DAG package list + # cannot produce a partial plugin archive. + EXPECTED_PLUGIN_COUNT=$( + cargo metadata --format-version 1 --no-deps | + jq '[.packages[].targets[] | select( + (.kind | index("cdylib")) and + (.name | startswith("iggy_connector_")) + )] | length' + ) + if [[ ! "$EXPECTED_PLUGIN_COUNT" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::Invalid expected connector plugin count: ${EXPECTED_PLUGIN_COUNT}" + exit 1 fi - test_end=$(date +%s) - test_duration=$((test_end - test_start)) - echo "::notice::Tests executed in ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))" + + PLUGIN_FILES=() + while IFS= read -r -d '' plugin_file; do + PLUGIN_FILES+=("${plugin_file##*/}") + done < <( + find target/debug -maxdepth 1 -type f \ + -name 'libiggy_connector_*.so' -print0 + ) + ACTUAL_PLUGIN_COUNT=${#PLUGIN_FILES[@]} + if (( ACTUAL_PLUGIN_COUNT != EXPECTED_PLUGIN_COUNT )); then + echo "::error::Expected ${EXPECTED_PLUGIN_COUNT} connector plugin libraries in target/debug, found ${ACTUAL_PLUGIN_COUNT}" + exit 1 + fi + tar -C target/debug -I 'zstd -T0' \ + -cf "$ARTIFACT_DIR/connector-plugins.tar.zst" \ + "${PLUGIN_FILES[@]}" + + # The integration package launches binaries owned by other workspace + # packages through assert_cmd::cargo_bin. Nextest only exposes + # CARGO_BIN_EXE_* for binaries Cargo associates with the current + # integration-test package. Preserve these cross-package executables + # explicitly so test-run jobs can provide the expected paths. + RUNTIME_BINARIES=( + iggy + iggy-server + iggy-server-ng + iggy-connectors + iggy-mcp + iggy-bench + ) + for binary in "${RUNTIME_BINARIES[@]}"; do + if [[ ! -x "target/debug/${binary}" ]]; then + echo "::error::Required test runtime binary not found: target/debug/${binary}" + exit 1 + fi + done + tar -C target/debug -czf "$ARTIFACT_DIR/runtime-binaries.tar.gz" \ + "${RUNTIME_BINARIES[@]}" + + cargo llvm-cov report --codecov --output-path codecov.json + echo "Build coverage report generated: codecov.json" + ls -la codecov.json build_duration=$((bins_duration + compile_duration)) - total_duration=$((build_duration + test_duration)) echo "" echo "=========================================" if [[ -n "$PACKAGE_FLAGS" ]]; then @@ -316,12 +333,106 @@ runs: echo "DAG scope: full workspace (${TOTAL_CRATES} crates)" fi echo "All targets build: ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))" - echo "Tests compile: ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" - echo "Tests execute: ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))" + echo "Tests compile and archive: ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" echo "-----------------------------------------" echo "Total build: ${build_duration}s ($(date -ud @${build_duration} +'%M:%S'))" - echo "Total time: ${total_duration}s ($(date -ud @${total_duration} +'%M:%S'))" echo "=========================================" + du -h "$ARTIFACT_DIR"/* + shell: bash + + - name: Upload archived Rust tests + if: inputs.task == 'test-build' + uses: actions/upload-artifact@v7 + with: + name: rust-nextest-archive-${{ inputs.component }}-${{ github.run_id }} + path: ${{ runner.temp }}/rust-nextest-archive + compression-level: 0 + if-no-files-found: error + retention-days: 30 + overwrite: true + + - name: Download archived Rust tests + if: startsWith(inputs.task, 'test-run-') + uses: actions/download-artifact@v8 + with: + name: rust-nextest-archive-${{ inputs.component }}-${{ github.run_id }} + path: ${{ runner.temp }}/rust-nextest-archive + + - name: Run archived tests with coverage + if: startsWith(inputs.task, 'test-run-') + run: | + TASK="${{ inputs.task }}" + PARTITION_COUNT="${{ inputs.test-partition-count }}" + if [[ ! "$TASK" =~ ^test-run-([1-9][0-9]*)$ ]]; then + echo "::error::Invalid archived test task: $TASK" + exit 1 + fi + PARTITION_INDEX="${BASH_REMATCH[1]}" + if [[ ! "$PARTITION_COUNT" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::Invalid test partition count: $PARTITION_COUNT" + exit 1 + fi + if (( PARTITION_INDEX > PARTITION_COUNT )); then + echo "::error::Test partition index ${PARTITION_INDEX} exceeds partition count ${PARTITION_COUNT}" + exit 1 + fi + ARTIFACT_DIR="${RUNNER_TEMP}/rust-nextest-archive" + NEXTEST_FILTER=$(cat "$ARTIFACT_DIR/nextest-filter.txt") + + mkdir -p target/debug + tar -C target/debug -I 'zstd -T0' \ + -xf "$ARTIFACT_DIR/connector-plugins.tar.zst" + tar -C target/debug -xzf "$ARTIFACT_DIR/runtime-binaries.tar.gz" + + RUNTIME_BINARIES=( + iggy + iggy-server + iggy-server-ng + iggy-connectors + iggy-mcp + iggy-bench + ) + CARGO_BIN_ENV=() + for binary in "${RUNTIME_BINARIES[@]}"; do + binary_path="${GITHUB_WORKSPACE}/target/debug/${binary}" + if [[ ! -x "$binary_path" ]]; then + echo "::error::Required test runtime binary not found after extraction: ${binary_path}" + exit 1 + fi + CARGO_BIN_ENV+=("CARGO_BIN_EXE_${binary}=${binary_path}") + done + + cargo llvm-cov show-env --sh > "$RUNNER_TEMP/llvm-cov-env.sh" + source "$RUNNER_TEMP/llvm-cov-env.sh" + + if [[ "$RUNNER_OS" == "Linux" ]]; then + # Doris 4.0.3's start_be.sh exits unless vm.max_map_count is at least + # 2000000. A container cannot raise this host kernel limit, so set it + # before the testcontainers fixture starts Doris. The image, resource + # limits, port mappings, and readiness wait remain in: + # core/integration/tests/connectors/fixtures/doris/container.rs + sudo sysctl -w vm.max_map_count=2000000 || true + DBUS_ENV=$(dbus-launch --sh-syntax) + eval "$DBUS_ENV" + export DBUS_SESSION_BUS_ADDRESS + KEYRING_ENV=$(echo -n "test" | gnome-keyring-daemon --unlock --components=secrets) + eval "$KEYRING_ENV" + echo -n "warmup" | secret-tool store --label="ci-warmup" ci-test warmup + fi + + test_start=$(date +%s) + env "${CARGO_BIN_ENV[@]}" cargo nextest run \ + --archive-file "$ARTIFACT_DIR/rust-tests.tar.zst" \ + --extract-to "$GITHUB_WORKSPACE" \ + --extract-overwrite \ + --workspace-remap "$GITHUB_WORKSPACE" \ + --no-fail-fast \ + --profile ci \ + --partition "hash:${PARTITION_INDEX}/${PARTITION_COUNT}" \ + -E "$NEXTEST_FILTER" + test_end=$(date +%s) + test_duration=$((test_end - test_start)) + echo "::notice::Partition ${PARTITION_INDEX}/${PARTITION_COUNT} executed in ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))" cargo llvm-cov report --codecov --output-path codecov.json echo "Coverage report generated: codecov.json" @@ -398,3 +509,27 @@ runs: if: inputs.task == 'build-windows-sdk' run: cargo build --locked -p iggy -p iggy-cli shell: bash + + - name: Validate Rust task + if: always() + run: | + TASK="${{ inputs.task }}" + case "$TASK" in + check|check-msrv|fmt|clippy|sort|machete|doctest|verify-publish) + exit 0 + ;; + test-build|compat|miri|build-aarch64-gnu|build-aarch64-musl) + exit 0 + ;; + build-macos-aarch64|build-windows-sdk) + exit 0 + ;; + esac + + if [[ "$TASK" =~ ^test-run-[1-9][0-9]*$ ]]; then + exit 0 + fi + + echo "::error::Unknown Rust task: $TASK" + exit 1 + shell: bash diff --git a/.github/actions/utils/setup-rust-with-cache/action.yml b/.github/actions/utils/setup-rust-with-cache/action.yml index e01ce81f8d..72979b5d26 100644 --- a/.github/actions/utils/setup-rust-with-cache/action.yml +++ b/.github/actions/utils/setup-rust-with-cache/action.yml @@ -19,6 +19,14 @@ name: setup-rust-with-cache description: Setup Rust toolchain with Swatinem/rust-cache inputs: + install-nextest: + description: "Whether to install cargo-nextest" + required: false + default: "false" + install-system-dependencies: + description: "Whether to install system packages required to build or run Rust binaries" + required: false + default: "true" read-cache: description: "Whether to read from cache" required: false @@ -54,15 +62,15 @@ runs: with: aggressive: ${{ inputs.free-disk-space-aggressive }} - - name: Install system dependencies (Linux) - if: runner.os == 'Linux' + - name: Install Rust build and runtime dependencies (Linux) + if: runner.os == 'Linux' && inputs.install-system-dependencies == 'true' run: | sudo apt-get update sudo apt-get install -y libhwloc-dev pkg-config libudev-dev shell: bash - - name: Install system dependencies (macOS) - if: runner.os == 'macOS' + - name: Install Rust build and runtime dependencies (macOS) + if: runner.os == 'macOS' && inputs.install-system-dependencies == 'true' run: | # Pin version of hwloc to 2.12.2_1 # brew extract doesn't have this version, so we fetch the formula directly @@ -117,10 +125,11 @@ runs: shell: bash - name: Install cargo-nextest - if: runner.os == 'Linux' + if: runner.os == 'Linux' && inputs.install-nextest == 'true' run: | - if command -v cargo-nextest &> /dev/null; then - echo "cargo-nextest already installed" + NEXTEST_VERSION="0.9.137" + if cargo nextest --version 2>/dev/null | grep -q "^cargo-nextest ${NEXTEST_VERSION}"; then + echo "cargo-nextest ${NEXTEST_VERSION} already installed" cargo nextest --version exit 0 fi @@ -130,18 +139,17 @@ runs: x86_64) NEXTEST_PLATFORM="linux" ;; aarch64) NEXTEST_PLATFORM="linux-arm" ;; *) - echo "Unsupported architecture: $ARCH, skipping nextest" - exit 0 + echo "::error::Unsupported architecture for cargo-nextest: $ARCH" + exit 1 ;; esac - curl -LsSf "https://get.nexte.st/latest/${NEXTEST_PLATFORM}" | tar xzf - -C ${CARGO_HOME:-~/.cargo}/bin + curl -LsSf "https://get.nexte.st/${NEXTEST_VERSION}/${NEXTEST_PLATFORM}" | + tar xzf - -C ${CARGO_HOME:-~/.cargo}/bin cargo nextest --version shell: bash - continue-on-error: true - name: Configure Cargo for CI - if: inputs.read-cache == 'true' run: | echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV echo "CARGO_PROFILE_DEV_DEBUG=0" >> $GITHUB_ENV diff --git a/.github/config/components.yml b/.github/config/components.yml index a2ec096d5e..d051910fe2 100644 --- a/.github/config/components.yml +++ b/.github/config/components.yml @@ -174,8 +174,7 @@ components: - "sort" - "doctest" - "machete" - - "test-1" - - "test-2" + - "test" - "compat" - "build-aarch64-gnu" - "build-aarch64-musl" @@ -497,7 +496,6 @@ components: paths: - "core/bench/dashboard/**" - "scripts/dashboard/**" - tasks: ["build"] # Core tools component rust-tools: diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 80edc184bc..256ccc9090 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -62,18 +62,6 @@ jobs: task: ${{ inputs.task }} component: ${{ inputs.component }} - - name: Upload coverage to Codecov - if: startsWith(inputs.component, 'rust') && startsWith(inputs.task, 'test-') - uses: codecov/codecov-action@v7.0.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: codecov.json - disable_search: true - flags: rust - fail_ci_if_error: false - verbose: true - override_pr: ${{ github.event.pull_request.number }} - # Python SDK - name: Set up Docker Buildx for Python if: inputs.component == 'sdk-python' && inputs.task == 'test' diff --git a/.github/workflows/_test_rust.yml b/.github/workflows/_test_rust.yml new file mode 100644 index 0000000000..22119adbc5 --- /dev/null +++ b/.github/workflows/_test_rust.yml @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: _test_rust +on: + workflow_call: + inputs: + component: + type: string + required: true + description: "Rust component to test" + task: + type: string + required: true + description: "Rust task to run" + secrets: + CODECOV_TOKEN: + required: false + +permissions: + contents: read + security-events: write + pull-requests: write + +jobs: + run: + if: inputs.task != 'test' + uses: ./.github/workflows/_test.yml + with: + component: ${{ inputs.component }} + task: ${{ inputs.task }} + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + build-rust-tests: + if: inputs.task == 'test' + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@v7.0.1 + + - name: Build and archive Rust tests + uses: ./.github/actions/rust/pre-merge + with: + task: test-build + component: ${{ inputs.component }} + + - name: Upload build coverage to Codecov + uses: codecov/codecov-action@v7.0.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: codecov.json + disable_search: true + flags: rust + fail_ci_if_error: false + verbose: true + override_pr: ${{ github.event.pull_request.number }} + + - name: Upload build reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: rust-test-build-reports-${{ github.run_id }}-${{ github.run_attempt }} + path: | + target/llvm-cov/** + codecov.json + if-no-files-found: ignore + retention-days: 7 + + run-rust-tests: + if: inputs.task == 'test' + needs: build-rust-tests + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + partition: [1, 2] + steps: + - name: Checkout code + uses: actions/checkout@v7.0.1 + + - name: Run Rust test partition + uses: ./.github/actions/rust/pre-merge + with: + task: test-run-${{ matrix.partition }} + component: ${{ inputs.component }} + test-partition-count: ${{ strategy.job-total }} + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v7.0.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: codecov.json + disable_search: true + flags: rust + fail_ci_if_error: false + verbose: true + override_pr: ${{ github.event.pull_request.number }} + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: rust-test-${{ matrix.partition }}-reports-${{ github.run_id }}-${{ github.run_attempt }} + path: | + target/llvm-cov/** + codecov.json + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml index 1f580db3d6..af0e5e13ba 100644 --- a/.github/workflows/coverage-baseline.yml +++ b/.github/workflows/coverage-baseline.yml @@ -52,6 +52,7 @@ jobs: - name: Setup Rust with cache uses: ./.github/actions/utils/setup-rust-with-cache with: + install-nextest: "true" # Also warms the GitHub Actions build cache for subsequent PR builds save-cache: "true" # llvm-cov instrumentation roughly doubles object sizes; the light diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 03b5254675..a7866ff79e 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -56,7 +56,7 @@ jobs: strategy: fail-fast: true matrix: ${{ fromJson(needs.detect.outputs.rust_matrix) }} - uses: ./.github/workflows/_test.yml + uses: ./.github/workflows/_test_rust.yml with: component: ${{ matrix.component }} task: ${{ matrix.task }} diff --git a/core/integration/src/harness/context.rs b/core/integration/src/harness/context.rs index 57e4090394..42f2a6e1be 100644 --- a/core/integration/src/harness/context.rs +++ b/core/integration/src/harness/context.rs @@ -51,7 +51,9 @@ fn is_cleanup_disabled_by_env() -> bool { } static TEST_LOGS_DIR: LazyLock = LazyLock::new(|| { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) + std::env::var_os("CARGO_MANIFEST_DIR") + .map(PathBuf::from) + .expect("CARGO_MANIFEST_DIR should be set when running integration tests") .parent() .and_then(|p| p.parent()) .expect("Failed to find workspace root") diff --git a/core/integration/tests/server/http_tls.rs b/core/integration/tests/server/http_tls.rs index 80bd58910c..bf471d60b2 100644 --- a/core/integration/tests/server/http_tls.rs +++ b/core/integration/tests/server/http_tls.rs @@ -40,11 +40,21 @@ const READY_RETRY_INTERVAL: Duration = Duration::from_millis(50); const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); /// Absolute path to a repo loopback cert asset. The spawned server's CWD is a -/// temp dir, so relative paths break; `CARGO_MANIFEST_DIR` is the integration -/// crate, whose sibling `../certs` holds the checked-in loopback material. +/// temp dir, so relative paths break; nextest remaps the runtime +/// `CARGO_MANIFEST_DIR` to the integration crate, whose sibling `../certs` +/// holds the checked-in loopback material. fn cert_asset(file: &str) -> PathBuf { - std::fs::canonicalize(format!("{}/../certs/{file}", env!("CARGO_MANIFEST_DIR"))) - .unwrap_or_else(|error| panic!("canonicalize repo cert asset {file}: {error}")) + let manifest_dir = std::env::var_os("CARGO_MANIFEST_DIR") + .map(PathBuf::from) + .expect("CARGO_MANIFEST_DIR should be set when running integration tests"); + let asset_path = manifest_dir.join("../certs").join(file); + + std::fs::canonicalize(&asset_path).unwrap_or_else(|error| { + panic!( + "canonicalize repo cert asset {}: {error}", + asset_path.display() + ) + }) } /// Boot iggy-server-ng with `[http.tls]` enabled against the repo loopback diff --git a/scripts/ci/python-sdk-version-sync.sh b/scripts/ci/python-sdk-version-sync.sh index 3788573bc4..591f68edb8 100755 --- a/scripts/ci/python-sdk-version-sync.sh +++ b/scripts/ci/python-sdk-version-sync.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/sync-python-interpreter-version.sh b/scripts/ci/sync-python-interpreter-version.sh index bc6a2de293..4c71c3ef2f 100755 --- a/scripts/ci/sync-python-interpreter-version.sh +++ b/scripts/ci/sync-python-interpreter-version.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/sync-rustc-version.sh b/scripts/ci/sync-rustc-version.sh index 1d25027f69..cc494dabd3 100755 --- a/scripts/ci/sync-rustc-version.sh +++ b/scripts/ci/sync-rustc-version.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/third-party-licenses.sh b/scripts/ci/third-party-licenses.sh index 8c86b253a8..e2304b26dd 100755 --- a/scripts/ci/third-party-licenses.sh +++ b/scripts/ci/third-party-licenses.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/uv-lock-check.sh b/scripts/ci/uv-lock-check.sh index 643ca3d858..de76ad7bfe 100755 --- a/scripts/ci/uv-lock-check.sh +++ b/scripts/ci/uv-lock-check.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information