diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c13aa272..42c666276 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,12 @@ jobs: with: channel: nightly-2026-03-04 cache-base: main + - name: Setup Noir + uses: noir-lang/noirup@v0.1.2 + with: + toolchain: v1.0.0-beta.19 + - name: Generate mobile benchmark Noir artifacts + run: bench-mobile/scripts/generate-fixtures.sh - name: Build run: cargo build --all-targets --all-features --verbose - name: Run tests diff --git a/.github/workflows/mobile-bench-pr-auto.yml b/.github/workflows/mobile-bench-pr-auto.yml new file mode 100644 index 000000000..7a9a7dee5 --- /dev/null +++ b/.github/workflows/mobile-bench-pr-auto.yml @@ -0,0 +1,107 @@ +name: Mobile Bench PR Auto + +on: + pull_request: + types: [labeled] + workflow_run: + workflows: ["Cargo Build & Test"] + types: [completed] + +permissions: + contents: write + actions: write + pull-requests: write + issues: write + checks: read + +jobs: + resolve: + name: Check compile gate and resolve context + runs-on: ubuntu-latest + if: >- + (github.event_name == 'pull_request' && github.event.action == 'labeled') || + (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') + outputs: + should_run: ${{ steps.pr.outputs.should_run }} + pr_number: ${{ steps.pr.outputs.pr_number }} + head_sha: ${{ steps.pr.outputs.head_sha }} + requested_by: ${{ steps.pr.outputs.requested_by }} + steps: + - name: Resolve PR context + id: pr + env: + GH_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ github.event_name }} + PR_NUMBER_EVENT: ${{ github.event.pull_request.number }} + HEAD_SHA_PR: ${{ github.event.pull_request.head.sha }} + BASE_REF_PR: ${{ github.event.pull_request.base.ref }} + HEAD_SHA_WR: ${{ github.event.workflow_run.head_sha }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + + if [ "$EVENT_NAME" = "pull_request" ]; then + PR_NUMBER="$PR_NUMBER_EVENT" + HEAD_SHA="$HEAD_SHA_PR" + REQUESTED_BY="auto:pull_request" + else + pr_json=$(gh api "repos/${REPO}/pulls?state=open&sort=updated&direction=desc&per_page=50" \ + --jq ".[] | select(.head.sha == \"${HEAD_SHA_WR}\") | {number, base_ref: .base.ref}" \ + | head -1) + if [ -z "$pr_json" ]; then + echo "::notice::No open PR found for SHA ${HEAD_SHA_WR}, skipping" + echo "should_run=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + PR_NUMBER=$(jq -r '.number' <<<"$pr_json") + HEAD_SHA="$HEAD_SHA_WR" + REQUESTED_BY="auto:workflow_run" + fi + + has_label=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/labels" \ + --jq '.[].name' | grep -qx 'bench' && echo "true" || echo "false") + if [ "$has_label" != "true" ]; then + echo "::notice::PR #${PR_NUMBER} does not have 'bench' label, skipping" + echo "should_run=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ "$EVENT_NAME" = "workflow_run" ]; then + gate_status="success" + else + gate_status=$(gh api "repos/${REPO}/commits/${HEAD_SHA}/check-runs" \ + --jq '.check_runs[] | select((.name == "Build & Test (all features)" or .name == "Build and test" or .name == "Cargo Build & Test") and .conclusion == "success") | .conclusion' \ + | head -1) + fi + if [ "$gate_status" != "success" ]; then + echo "::notice::Compile gate 'Cargo Build & Test' not yet passed for ${HEAD_SHA} (status: ${gate_status:-pending})" + echo "should_run=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "pr_number=${PR_NUMBER}" >> "$GITHUB_OUTPUT" + echo "head_sha=${HEAD_SHA}" >> "$GITHUB_OUTPUT" + echo "requested_by=${REQUESTED_BY}" >> "$GITHUB_OUTPUT" + echo "should_run=true" >> "$GITHUB_OUTPUT" + + browserstack: + name: Run BrowserStack benchmarks + needs: resolve + if: needs.resolve.outputs.should_run == 'true' + uses: ./.github/workflows/mobile-bench-reusable.yml + secrets: inherit + with: + crate_path: ./bench-mobile + functions: '["bench_mobile::bench_passport_complete_age_check_prove","bench_mobile::bench_oprf_prove","bench_mobile::bench_p256_bigcurve_prove"]' + functions_ios: '["bench_mobile::bench_passport_complete_age_check_prove","bench_mobile::bench_oprf_prove","bench_mobile::bench_p256_bigcurve_prove"]' + functions_android: '["bench_mobile::bench_oprf_prove","bench_mobile::bench_p256_bigcurve_prove","bench_mobile::bench_passport_complete_age_check_prove"]' + platform: both + device_profile: triad + iterations: "2" + warmup: "1" + mobench_version: "0.1.41" + mobench_ref: "codex/native-c-abi-backend" + pr_number: ${{ needs.resolve.outputs.pr_number }} + requested_by: ${{ needs.resolve.outputs.requested_by }} + head_sha: ${{ needs.resolve.outputs.head_sha }} diff --git a/.github/workflows/mobile-bench-pr-command.yml b/.github/workflows/mobile-bench-pr-command.yml new file mode 100644 index 000000000..15c76fc56 --- /dev/null +++ b/.github/workflows/mobile-bench-pr-command.yml @@ -0,0 +1,118 @@ +name: Mobile Bench PR Command + +on: + issue_comment: + types: [created] + +permissions: + contents: write + actions: write + pull-requests: write + issues: write + +jobs: + resolve: + name: Parse /mobench and resolve context + if: >- + github.event_name == 'issue_comment' && + github.event.action == 'created' && + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/mobench') + runs-on: ubuntu-latest + outputs: + trusted: ${{ steps.trust.outputs.trusted }} + platform: ${{ steps.parse.outputs.platform }} + device_profile: ${{ steps.parse.outputs.device_profile }} + iterations: ${{ steps.parse.outputs.iterations }} + warmup: ${{ steps.parse.outputs.warmup }} + head_sha: ${{ steps.pr.outputs.head_sha }} + pr_number: ${{ github.event.issue.number }} + requested_by: ${{ github.event.comment.user.login }} + steps: + - name: Check trust + id: trust + env: + AUTHOR_ASSOCIATION: ${{ github.event.comment.author_association }} + run: | + if echo "OWNER,MEMBER,COLLABORATOR" | tr ',' '\n' | grep -qx "$AUTHOR_ASSOCIATION"; then + echo "trusted=true" >> "$GITHUB_OUTPUT" + else + echo "::warning::Untrusted author association: $AUTHOR_ASSOCIATION" + echo "trusted=false" >> "$GITHUB_OUTPUT" + fi + + - name: Parse command + if: steps.trust.outputs.trusted == 'true' + id: parse + env: + COMMENT_BODY: ${{ github.event.comment.body }} + run: | + set -euo pipefail + line=$(echo "$COMMENT_BODY" | head -1) + + extract_val() { + echo "$line" | sed -n "s/.*${1}=\([^ ]*\).*/\1/p" + } + + platform=$(extract_val platform) + device_profile=$(extract_val device_profile) + iterations=$(extract_val iterations) + warmup=$(extract_val warmup) + + case "${platform:-both}" in + android|ios|both) platform="${platform:-both}" ;; + *) echo "::warning::Invalid platform '${platform}', defaulting to 'both'"; platform="both" ;; + esac + + case "${device_profile:-triad}" in + smoke|triad|worst) device_profile="${device_profile:-triad}" ;; + *) echo "::warning::Invalid device_profile '${device_profile}', defaulting to 'triad'"; device_profile="triad" ;; + esac + + if ! [[ "${iterations:-2}" =~ ^[0-9]+$ ]]; then + echo "::warning::Invalid iterations '${iterations}', defaulting to '2'" + iterations="2" + else + iterations="${iterations:-2}" + fi + + if ! [[ "${warmup:-1}" =~ ^[0-9]+$ ]]; then + echo "::warning::Invalid warmup '${warmup}', defaulting to '1'" + warmup="1" + else + warmup="${warmup:-1}" + fi + + echo "platform=${platform}" >> "$GITHUB_OUTPUT" + echo "device_profile=${device_profile}" >> "$GITHUB_OUTPUT" + echo "iterations=${iterations}" >> "$GITHUB_OUTPUT" + echo "warmup=${warmup}" >> "$GITHUB_OUTPUT" + + - name: Resolve PR refs + if: steps.trust.outputs.trusted == 'true' + id: pr + env: + GH_TOKEN: ${{ github.token }} + PR_URL: ${{ github.event.issue.pull_request.url }} + run: | + head_sha=$(gh api "$PR_URL" --jq '.head.sha') + echo "head_sha=${head_sha}" >> "$GITHUB_OUTPUT" + + browserstack: + name: Run BrowserStack benchmarks + needs: resolve + if: needs.resolve.outputs.trusted == 'true' + uses: ./.github/workflows/mobile-bench-reusable.yml + secrets: inherit + with: + crate_path: ./bench-mobile + functions: '["bench_mobile::bench_passport_complete_age_check_prove","bench_mobile::bench_oprf_prove","bench_mobile::bench_p256_bigcurve_prove"]' + functions_ios: '["bench_mobile::bench_passport_complete_age_check_prove","bench_mobile::bench_oprf_prove","bench_mobile::bench_p256_bigcurve_prove"]' + functions_android: '["bench_mobile::bench_oprf_prove","bench_mobile::bench_p256_bigcurve_prove","bench_mobile::bench_passport_complete_age_check_prove"]' + platform: ${{ needs.resolve.outputs.platform }} + device_profile: ${{ needs.resolve.outputs.device_profile }} + iterations: ${{ needs.resolve.outputs.iterations }} + warmup: ${{ needs.resolve.outputs.warmup }} + pr_number: ${{ needs.resolve.outputs.pr_number }} + requested_by: ${{ needs.resolve.outputs.requested_by }} + head_sha: ${{ needs.resolve.outputs.head_sha }} diff --git a/.github/workflows/mobile-bench-reusable.yml b/.github/workflows/mobile-bench-reusable.yml new file mode 100644 index 000000000..daa2e316b --- /dev/null +++ b/.github/workflows/mobile-bench-reusable.yml @@ -0,0 +1,954 @@ +name: Reusable Mobile Benchmark (BrowserStack) + +on: + workflow_call: + inputs: + crate_path: + description: "Path to the benchmark crate in the caller repo" + required: true + type: string + functions: + description: "Comma-separated or JSON array list of benchmark function names to run" + required: true + type: string + functions_ios: + description: "Optional iOS-specific benchmark function list" + required: false + type: string + default: "" + functions_android: + description: "Optional Android-specific benchmark function list" + required: false + type: string + default: "" + iterations: + description: "Number of benchmark iterations" + required: false + type: string + default: "2" + warmup: + description: "Number of warmup iterations" + required: false + type: string + default: "1" + platform: + description: "Target platform: android, ios, or both" + required: false + type: string + default: "both" + device_profile: + description: "Device profile to run (smoke, triad, or worst)" + required: false + type: string + default: "triad" + rust_targets_ios: + description: "Comma-separated iOS Rust targets" + required: false + type: string + default: "aarch64-apple-ios,aarch64-apple-ios-sim,x86_64-apple-ios" + rust_targets_android: + description: "Comma-separated Android Rust targets" + required: false + type: string + default: "aarch64-linux-android" + build_release: + description: "Build in release mode" + required: false + type: boolean + default: true + mobench_version: + description: "Mobench version to install" + required: false + type: string + default: "0.1.41" + mobench_ref: + description: "Optional Git ref for mobile-bench-rs to override the released mobench install" + required: false + type: string + default: "codex/native-c-abi-backend" + pr_number: + description: "PR number for reporting" + required: false + type: string + report_repository: + description: "owner/repo to receive the sticky benchmark comment; defaults to the workflow repository" + required: false + type: string + default: "" + requested_by: + description: "Who triggered the run" + required: false + type: string + head_sha: + description: "Exact commit SHA to checkout in the caller repo" + required: false + type: string + secrets: + BROWSERSTACK_USERNAME: + required: false + BROWSERSTACK_ACCESS_KEY: + required: false + +permissions: + actions: read + contents: write + pull-requests: write + issues: write + +env: + CARGO_TERM_COLOR: always + RUST_TOOLCHAIN: nightly-2026-03-04 + +jobs: + ios: + name: iOS BrowserStack benchmark + if: inputs.platform == 'ios' || inputs.platform == 'both' + runs-on: macos-15 + environment: Browserstack + concurrency: + group: mobench-browserstack-device-cloud + cancel-in-progress: false + env: + BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }} + BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} + IOS_DEPLOYMENT_TARGET: "10.0" + IPHONEOS_DEPLOYMENT_TARGET: "10.0" + CFLAGS_aarch64_apple_ios: "-miphoneos-version-min=10.0" + CFLAGS_aarch64_apple_ios_sim: "-mios-simulator-version-min=10.0" + CFLAGS_x86_64_apple_ios: "-mios-simulator-version-min=10.0" + CARGO_TARGET_AARCH64_APPLE_IOS_RUSTFLAGS: "-C link-arg=-miphoneos-version-min=10.0" + CARGO_TARGET_AARCH64_APPLE_IOS_SIM_RUSTFLAGS: "-C link-arg=-mios-simulator-version-min=10.0" + CARGO_TARGET_X86_64_APPLE_IOS_RUSTFLAGS: "-C link-arg=-mios-simulator-version-min=10.0" + MOBENCH_ALLOW_UNSUPPORTED_IOS_DEPLOYMENT_TARGET: "1" + steps: + - name: Checkout caller repo + uses: actions/checkout@v4 + with: + path: caller + ref: ${{ inputs.head_sha || github.sha }} + + - name: Resolve iOS device profile + shell: bash + env: + DEVICE_PROFILE: ${{ inputs.device_profile }} + run: | + set -euo pipefail + case "${DEVICE_PROFILE}" in + smoke) + device_specs="iPhone 7" + fallback_device_specs="iPhone SE 2020-16" + fetch_timeout_secs="7200" + ;; + worst) + device_specs="iPhone 7" + fallback_device_specs="iPhone SE 2020-16" + fetch_timeout_secs="7200" + ;; + triad) + device_specs="iPhone SE 2020-16,iPhone 15-17,iPhone 16 Pro-18" + fallback_device_specs="" + fetch_timeout_secs="7200" + ;; + *) + echo "::error::Unsupported device_profile '${DEVICE_PROFILE}'. Supported values: smoke, triad, worst." + exit 1 + ;; + esac + + { + echo "MOBENCH_DEVICE_PROFILE=${DEVICE_PROFILE}" + echo "IOS_DEVICE_SPECS=${device_specs}" + echo "IOS_FALLBACK_DEVICE_SPECS=${fallback_device_specs}" + echo "MOBENCH_FETCH_TIMEOUT_SECS=${fetch_timeout_secs}" + } >> "$GITHUB_ENV" + + echo "Resolved iOS device profile '${DEVICE_PROFILE}' to ${device_specs}" + echo "Resolved iOS fallback devices to ${fallback_device_specs}" + echo "Resolved iOS fetch timeout to ${fetch_timeout_secs}s" + + - name: Setup Rust + shell: bash + env: + RUST_TARGETS: ${{ inputs.rust_targets_ios }} + run: | + set -euo pipefail + rustup toolchain install "${RUST_TOOLCHAIN}" --profile minimal + rustup default "${RUST_TOOLCHAIN}" + + IFS=',' read -r -a rust_targets <<<"${RUST_TARGETS}" + for target in "${rust_targets[@]}"; do + target="$(echo "$target" | xargs)" + if [[ -n "$target" ]]; then + rustup target add "$target" --toolchain "${RUST_TOOLCHAIN}" + fi + done + + rustc -Vv + cargo -V + + - name: Install mobench + shell: bash + env: + MOBENCH_VERSION: ${{ inputs.mobench_version }} + MOBENCH_REF: ${{ inputs.mobench_ref }} + run: | + set -euo pipefail + if [[ -n "${MOBENCH_REF}" ]]; then + echo "Installing mobench from ${MOBENCH_REF}" + git clone https://github.com/worldcoin/mobile-bench-rs mobench-src + git -C mobench-src checkout "${MOBENCH_REF}" + cargo install --path mobench-src/crates/mobench --locked --force + else + echo "Installing mobench ${MOBENCH_VERSION} from crates.io" + cargo install mobench --version "${MOBENCH_VERSION}" --locked --force + fi + cargo-mobench --version + + - name: Setup Noir + uses: noir-lang/noirup@v0.1.2 + with: + toolchain: v1.0.0-beta.19 + + - name: Install iOS tooling + run: brew install xcodegen swiftformat + + - name: Generate mobile benchmark Noir artifacts + working-directory: caller + run: bench-mobile/scripts/generate-fixtures.sh + + - name: Build iOS artifacts + working-directory: caller + shell: bash + env: + RELEASE_FLAG: ${{ inputs.build_release && '--release' || '' }} + CRATE_PATH: ${{ inputs.crate_path }} + run: | + set -euo pipefail + echo "Building iOS artifacts for profile ${MOBENCH_DEVICE_PROFILE}" + cargo-mobench build \ + --target ios \ + $RELEASE_FLAG \ + --crate-path "$CRATE_PATH" \ + --ios-deployment-target "${IOS_DEPLOYMENT_TARGET}" \ + --ios-runner uikit-legacy + + cargo-mobench package-ipa --method adhoc --crate-path "$CRATE_PATH" + cargo-mobench package-xcuitest --crate-path "$CRATE_PATH" + test -f target/mobench/ios/BenchRunner.ipa + test -f target/mobench/ios/BenchRunnerUITests.zip + + - name: Run iOS benchmarks + id: run_ios_benchmarks + timeout-minutes: 180 + working-directory: caller + shell: bash + env: + FUNCTIONS: ${{ inputs.functions_ios != '' && inputs.functions_ios || inputs.functions }} + ITERATIONS: ${{ inputs.iterations }} + WARMUP: ${{ inputs.warmup }} + RELEASE_FLAG: ${{ inputs.build_release && '--release' || '' }} + CRATE_PATH: ${{ inputs.crate_path }} + run: | + set -euo pipefail + + benchmark_functions=() + if [[ "${FUNCTIONS}" == \[* ]]; then + while IFS= read -r function_name; do + if [[ -n "${function_name}" ]]; then + benchmark_functions+=("${function_name}") + fi + done < <(jq -r '.[]' <<<"${FUNCTIONS}") + else + IFS=',' read -r -a raw_functions <<<"${FUNCTIONS}" + for function_name in "${raw_functions[@]}"; do + function_name="$(echo "$function_name" | xargs)" + if [[ -n "${function_name}" ]]; then + benchmark_functions+=("${function_name}") + fi + done + fi + + if [ "${#benchmark_functions[@]}" -eq 0 ]; then + echo "::error::No iOS benchmark functions resolved from '${FUNCTIONS}'" + exit 1 + fi + + echo "Running iOS benchmarks with profile ${MOBENCH_DEVICE_PROFILE}" + echo "iOS devices: ${IOS_DEVICE_SPECS}" + echo "iOS fallback devices: ${IOS_FALLBACK_DEVICE_SPECS}" + echo "iOS fetch timeout: ${MOBENCH_FETCH_TIMEOUT_SECS}s" + max_attempts=2 + retry_sleep_secs=60 + log_dir="target/mobench/retry-logs/ios" + mkdir -p "$log_dir" + rm -rf target/mobench/ci/ios target/browserstack/ios + + is_transient_fetch_failure() { + local attempt_log="$1" + local json_path + + if grep -Eiq 'BrowserStack API .*status 5[0-9]{2}|This website is under heavy load|fetch did not recover any benchmark payloads|Timeout waiting for build .* to complete' "$attempt_log"; then + return 0 + fi + + while IFS= read -r -d '' json_path; do + if jq -e ' + if (has("status") and ((.status | ascii_downcase) == "running")) then + true + elif (.testcases?.status?.running // 0) > 0 then + true + else + false + end + ' "$json_path" >/dev/null 2>&1; then + return 0 + fi + done < <(find target/browserstack/ios -type f \( -name build.json -o -name session.json \) -print0 2>/dev/null) + + return 1 + } + + is_ios_min_os_schedule_failure() { + local attempt_log="$1" + grep -Eq 'os version lower than the minimum required os version required for app, test_suite|BROWSERSTACK_NO_DEVICE_FOUND_WITH_REQUESTED_CRITERIA' "$attempt_log" + } + + make_device_args() { + local specs="$1" + device_args=() + + IFS=',' read -r -a device_specs <<<"${specs}" + for device in "${device_specs[@]}"; do + device="$(echo "$device" | xargs)" + if [[ -n "$device" ]]; then + device_args+=(--devices "$device") + fi + done + } + + effective_device_specs="${IOS_DEVICE_SPECS}" + for benchmark_function in "${benchmark_functions[@]}"; do + function_iterations="${ITERATIONS}" + function_warmup="${WARMUP}" + if [[ "${benchmark_function}" == "bench_mobile::bench_passport_complete_age_check_prove" ]]; then + function_iterations="1" + function_warmup="0" + fi + + function_slug="$(tr -c '[:alnum:]' '_' <<<"${benchmark_function}" | sed 's/_*$//')" + attempt=1 + while true; do + attempt_log="${log_dir}/${function_slug}-attempt-${attempt}.log" + fetch_output_dir="target/browserstack/ios/${function_slug}" + result_output_dir="target/mobench/ci/ios/${function_slug}" + rm -rf "${fetch_output_dir}" + mkdir -p "${fetch_output_dir}" "${result_output_dir}" + make_device_args "${effective_device_specs}" + + echo "mobench ios ${benchmark_function} attempt ${attempt}/${max_attempts} on ${effective_device_specs}" + set +e + cargo-mobench ci run \ + --target ios \ + --function "${benchmark_function}" \ + --iterations "${function_iterations}" \ + --warmup "${function_warmup}" \ + "${device_args[@]}" \ + --crate-path "$CRATE_PATH" \ + $RELEASE_FLAG \ + --ios-deployment-target "${IOS_DEPLOYMENT_TARGET}" \ + --ios-runner uikit-legacy \ + --fetch \ + --fetch-timeout-secs "${MOBENCH_FETCH_TIMEOUT_SECS}" \ + --fetch-output-dir "${fetch_output_dir}" \ + --output-dir "${result_output_dir}" \ + 2>&1 | tee "$attempt_log" + status=${PIPESTATUS[0]} + set -e + + if [ "$status" -eq 0 ]; then + break + fi + + if is_ios_min_os_schedule_failure "$attempt_log" && [[ -n "${IOS_FALLBACK_DEVICE_SPECS}" ]] && [[ "${effective_device_specs}" != "${IOS_FALLBACK_DEVICE_SPECS}" ]]; then + echo "::warning::iOS devices ${effective_device_specs} require a newer app/test-suite target in BrowserStack; retrying ${benchmark_function} on ${IOS_FALLBACK_DEVICE_SPECS}" + effective_device_specs="${IOS_FALLBACK_DEVICE_SPECS}" + attempt=1 + continue + fi + + if [ "$attempt" -ge "$max_attempts" ] || ! is_transient_fetch_failure "$attempt_log"; then + exit "$status" + fi + + build_id="$(grep -Eo 'Build ID: [a-f0-9]+' "$attempt_log" | awk '{print $3}' | tail -1 || true)" + if [[ -n "$build_id" ]]; then + echo "::warning::Transient BrowserStack fetch failure for iOS build ${build_id}; retrying ${benchmark_function}" + else + echo "::warning::Transient BrowserStack fetch failure for iOS; retrying ${benchmark_function}" + fi + + attempt=$((attempt + 1)) + sleep "$retry_sleep_secs" + done + done + + - name: Upload iOS results + if: always() + uses: actions/upload-artifact@v4 + with: + name: mobench-results-ios + path: | + caller/target/mobench/ci/ios/** + caller/target/browserstack/ios/** + if-no-files-found: warn + + android: + name: Android BrowserStack benchmark + if: inputs.platform == 'android' || inputs.platform == 'both' + runs-on: macos-14 + environment: Browserstack + concurrency: + group: mobench-browserstack-device-cloud + cancel-in-progress: false + env: + BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }} + BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} + steps: + - name: Checkout caller repo + uses: actions/checkout@v4 + with: + path: caller + ref: ${{ inputs.head_sha || github.sha }} + + - name: Resolve Android device profile + shell: bash + env: + DEVICE_PROFILE: ${{ inputs.device_profile }} + run: | + set -euo pipefail + case "${DEVICE_PROFILE}" in + smoke) + device_specs="Vivo Y21-11.0" + fetch_timeout_secs="7200" + ;; + worst) + device_specs="Vivo Y21-11.0" + fetch_timeout_secs="7200" + ;; + triad) + device_specs="Vivo Y21-11.0,Google Pixel 7-13.0,Samsung Galaxy S24-14.0" + fetch_timeout_secs="7200" + ;; + *) + echo "::error::Unsupported device_profile '${DEVICE_PROFILE}'. Supported values: smoke, triad, worst." + exit 1 + ;; + esac + + { + echo "MOBENCH_DEVICE_PROFILE=${DEVICE_PROFILE}" + echo "ANDROID_DEVICE_SPECS=${device_specs}" + echo "MOBENCH_FETCH_TIMEOUT_SECS=${fetch_timeout_secs}" + } >> "$GITHUB_ENV" + + echo "Resolved Android device profile '${DEVICE_PROFILE}' to ${device_specs}" + echo "Resolved Android fetch timeout to ${fetch_timeout_secs}s" + + - name: Setup Rust + shell: bash + env: + RUST_TARGETS: ${{ inputs.rust_targets_android }} + run: | + set -euo pipefail + rustup toolchain install "${RUST_TOOLCHAIN}" --profile minimal + rustup default "${RUST_TOOLCHAIN}" + + IFS=',' read -r -a rust_targets <<<"${RUST_TARGETS}" + for target in "${rust_targets[@]}"; do + target="$(echo "$target" | xargs)" + if [[ -n "$target" ]]; then + rustup target add "$target" --toolchain "${RUST_TOOLCHAIN}" + fi + done + + rustc -Vv + cargo -V + + - name: Setup Android SDK/NDK + uses: android-actions/setup-android@v3 + + - name: Install SDK packages and resolve NDK + shell: bash + run: | + SDKMGR="${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" + if [ ! -x "$SDKMGR" ]; then + SDKMGR=$(command -v sdkmanager 2>/dev/null || echo "sdkmanager") + fi + + $SDKMGR --install "platform-tools" "platforms;android-34" "build-tools;34.0.0" "ndk;26.1.10909125" 2>&1 || true + + if [ -d "${ANDROID_HOME}/ndk/26.1.10909125" ]; then + NDK_DIR="${ANDROID_HOME}/ndk/26.1.10909125" + else + NDK_VER=$(ls "${ANDROID_HOME}/ndk/" 2>/dev/null | sort -V | tail -1) + if [ -z "$NDK_VER" ]; then + echo "::error::No Android NDK found" + exit 1 + fi + NDK_DIR="${ANDROID_HOME}/ndk/${NDK_VER}" + fi + + echo "ANDROID_NDK_HOME=${NDK_DIR}" >> "$GITHUB_ENV" + echo "ANDROID_NDK_ROOT=${NDK_DIR}" >> "$GITHUB_ENV" + + - name: Install cargo-ndk + run: cargo install cargo-ndk --locked + + - name: Install mobench + shell: bash + env: + MOBENCH_VERSION: ${{ inputs.mobench_version }} + MOBENCH_REF: ${{ inputs.mobench_ref }} + run: | + set -euo pipefail + if [[ -n "${MOBENCH_REF}" ]]; then + echo "Installing mobench from ${MOBENCH_REF}" + git clone https://github.com/worldcoin/mobile-bench-rs mobench-src + git -C mobench-src checkout "${MOBENCH_REF}" + cargo install --path mobench-src/crates/mobench --locked --force + else + echo "Installing mobench ${MOBENCH_VERSION} from crates.io" + cargo install mobench --version "${MOBENCH_VERSION}" --locked --force + fi + cargo-mobench --version + + - name: Setup Noir + uses: noir-lang/noirup@v0.1.2 + with: + toolchain: v1.0.0-beta.19 + + - name: Generate mobile benchmark Noir artifacts + working-directory: caller + run: bench-mobile/scripts/generate-fixtures.sh + + - name: Build Android artifacts + working-directory: caller + shell: bash + env: + RELEASE_FLAG: ${{ inputs.build_release && '--release' || '' }} + CRATE_PATH: ${{ inputs.crate_path }} + run: | + set -euo pipefail + cargo-mobench build --target android $RELEASE_FLAG --crate-path "$CRATE_PATH" + + - name: Run Android benchmarks + id: run_android_benchmarks + timeout-minutes: 180 + working-directory: caller + shell: bash + env: + FUNCTIONS: ${{ inputs.functions_android != '' && inputs.functions_android || inputs.functions }} + ITERATIONS: ${{ inputs.iterations }} + WARMUP: ${{ inputs.warmup }} + RELEASE_FLAG: ${{ inputs.build_release && '--release' || '' }} + CRATE_PATH: ${{ inputs.crate_path }} + run: | + set -euo pipefail + + benchmark_functions=() + if [[ "${FUNCTIONS}" == \[* ]]; then + while IFS= read -r function_name; do + if [[ -n "${function_name}" ]]; then + benchmark_functions+=("${function_name}") + fi + done < <(jq -r '.[]' <<<"${FUNCTIONS}") + else + IFS=',' read -r -a raw_functions <<<"${FUNCTIONS}" + for function_name in "${raw_functions[@]}"; do + function_name="$(echo "$function_name" | xargs)" + if [[ -n "${function_name}" ]]; then + benchmark_functions+=("${function_name}") + fi + done + fi + + if [ "${#benchmark_functions[@]}" -eq 0 ]; then + echo "::error::No Android benchmark functions resolved from '${FUNCTIONS}'" + exit 1 + fi + + echo "Running Android benchmarks with profile ${MOBENCH_DEVICE_PROFILE}" + echo "Android devices: ${ANDROID_DEVICE_SPECS}" + echo "Android fetch timeout: ${MOBENCH_FETCH_TIMEOUT_SECS}s" + max_attempts=2 + retry_sleep_secs=60 + log_dir="target/mobench/retry-logs/android" + mkdir -p "$log_dir" + rm -rf target/mobench/ci/android target/browserstack/android + + is_transient_fetch_failure() { + local attempt_log="$1" + local json_path + + if grep -Eiq 'BrowserStack API .*status 5[0-9]{2}|This website is under heavy load|fetch did not recover any benchmark payloads|No benchmark results found|Timeout waiting for build .* to complete' "$attempt_log"; then + return 0 + fi + + while IFS= read -r -d '' json_path; do + if jq -e ' + if (has("status") and ((.status | ascii_downcase) == "running")) then + true + elif (.testcases?.status?.running // 0) > 0 then + true + else + false + end + ' "$json_path" >/dev/null 2>&1; then + return 0 + fi + done < <(find target/browserstack/android -type f \( -name build.json -o -name session.json \) -print0 2>/dev/null) + + return 1 + } + + make_device_args() { + device_args=() + + IFS=',' read -r -a device_specs <<<"${ANDROID_DEVICE_SPECS}" + for device in "${device_specs[@]}"; do + device="$(echo "$device" | xargs)" + if [[ -n "$device" ]]; then + device_args+=(--devices "$device") + fi + done + } + + for benchmark_function in "${benchmark_functions[@]}"; do + function_iterations="${ITERATIONS}" + function_warmup="${WARMUP}" + function_fetch_timeout_secs="${MOBENCH_FETCH_TIMEOUT_SECS}" + function_max_attempts="${max_attempts}" + if [[ "${benchmark_function}" == "bench_mobile::bench_passport_complete_age_check_prove" ]]; then + function_iterations="1" + function_warmup="0" + function_max_attempts="1" + fi + + function_slug="$(tr -c '[:alnum:]' '_' <<<"${benchmark_function}" | sed 's/_*$//')" + attempt=1 + while true; do + attempt_log="${log_dir}/${function_slug}-attempt-${attempt}.log" + fetch_output_dir="target/browserstack/android/${function_slug}" + result_output_dir="target/mobench/ci/android/${function_slug}" + rm -rf "${fetch_output_dir}" + mkdir -p "${fetch_output_dir}" "${result_output_dir}" + make_device_args + + echo "mobench android ${benchmark_function} attempt ${attempt}/${function_max_attempts}" + set +e + cargo-mobench ci run \ + --target android \ + --function "${benchmark_function}" \ + --iterations "${function_iterations}" \ + --warmup "${function_warmup}" \ + "${device_args[@]}" \ + --crate-path "$CRATE_PATH" \ + $RELEASE_FLAG \ + --android-benchmark-timeout-secs 7200 \ + --android-heartbeat-interval-secs 10 \ + --fetch \ + --fetch-timeout-secs "${function_fetch_timeout_secs}" \ + --fetch-output-dir "${fetch_output_dir}" \ + --output-dir "${result_output_dir}" \ + 2>&1 | tee "$attempt_log" + status=${PIPESTATUS[0]} + set -e + + if [ "$status" -eq 0 ]; then + break + fi + + if [ "$attempt" -ge "$function_max_attempts" ] && grep -Eiq 'No benchmark results found|Timeout waiting for build' "$attempt_log"; then + echo "::warning::Android ${benchmark_function} did not return benchmark results on ${ANDROID_DEVICE_SPECS}; preserving partial fixture results" + { + echo "### Android fixture incomplete" + echo "" + echo "- Function: \`${benchmark_function}\`" + echo "- Device: \`${ANDROID_DEVICE_SPECS}\`" + echo "- Iterations/Warmup: \`${function_iterations} / ${function_warmup}\`" + echo "- Reason: BrowserStack did not return benchmark results after ${function_max_attempts} attempt(s)." + } > "${result_output_dir}/failure.md" + jq -n \ + --arg platform "android" \ + --arg function "${benchmark_function}" \ + --arg devices "${ANDROID_DEVICE_SPECS}" \ + --arg reason "BrowserStack did not return benchmark results after ${function_max_attempts} attempt(s)." \ + '{platform: $platform, function: $function, devices: $devices, reason: $reason}' \ + > "${result_output_dir}/failure.json" + break + fi + + if [ "$attempt" -ge "$function_max_attempts" ] || ! is_transient_fetch_failure "$attempt_log"; then + exit "$status" + fi + + build_id="$(grep -Eo 'Build ID: [a-f0-9]+' "$attempt_log" | awk '{print $3}' | tail -1 || true)" + if [[ -n "$build_id" ]]; then + echo "::warning::Transient BrowserStack fetch failure for Android build ${build_id}; retrying ${benchmark_function}" + else + echo "::warning::Transient BrowserStack fetch failure for Android; retrying ${benchmark_function}" + fi + + attempt=$((attempt + 1)) + sleep "$retry_sleep_secs" + done + done + + - name: Upload Android results + if: always() + uses: actions/upload-artifact@v4 + with: + name: mobench-results-android + path: | + caller/target/mobench/ci/android/** + caller/target/browserstack/android/** + if-no-files-found: warn + + summarize: + name: Summarize benchmark results + needs: [ios, android] + if: always() + runs-on: ubuntu-latest + steps: + - name: Checkout caller repo + uses: actions/checkout@v4 + with: + path: caller + ref: ${{ inputs.head_sha || github.sha }} + + - name: Setup Rust + shell: bash + run: | + set -euo pipefail + rustup toolchain install "${RUST_TOOLCHAIN}" --profile minimal + rustup default "${RUST_TOOLCHAIN}" + rustc -Vv + cargo -V + + - name: Install mobench + shell: bash + env: + MOBENCH_VERSION: ${{ inputs.mobench_version }} + MOBENCH_REF: ${{ inputs.mobench_ref }} + run: | + set -euo pipefail + if [[ -n "${MOBENCH_REF}" ]]; then + echo "Installing mobench from ${MOBENCH_REF}" + git clone https://github.com/worldcoin/mobile-bench-rs mobench-src + git -C mobench-src checkout "${MOBENCH_REF}" + cargo install --path mobench-src/crates/mobench --locked --force + else + echo "Installing mobench ${MOBENCH_VERSION} from crates.io" + cargo install mobench --version "${MOBENCH_VERSION}" --locked --force + fi + cargo-mobench --version + + - name: Download iOS results + if: always() + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: mobench-results-ios + path: results/ios + + - name: Download Android results + if: always() + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: mobench-results-android + path: results/android + + - name: Setup Python for plot rendering + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install plot rendering dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install matplotlib + + - name: Render plot-capable platform summaries + id: render_summaries + shell: bash + run: | + set -euo pipefail + mkdir -p rendered + rendered_count=0 + + render_platform_summary() { + local platform="$1" + local results_dir="results/${platform}" + if [ ! -d "${results_dir}" ]; then + return 0 + fi + + local summary_count + summary_count=$(find "${results_dir}" -type f -name summary.json | wc -l | tr -d ' ') + if [ "${summary_count}" -eq 0 ]; then + echo "::warning::No ${platform} summary.json found under ${results_dir}" + return 0 + fi + + local csv_count + csv_count=$(find "${results_dir}" -type f -name results.csv | wc -l | tr -d ' ') + if [ "${csv_count}" -eq 0 ]; then + echo "::warning::No ${platform} results.csv found under ${results_dir}" + return 0 + fi + + mkdir -p "rendered/${platform}" + cargo-mobench ci summarize \ + --results-dir "${results_dir}" \ + --output-format markdown \ + --output-file "rendered/${platform}/summary.md" + + while IFS= read -r failure_md; do + { + echo "" + cat "${failure_md}" + } >> "rendered/${platform}/summary.md" + done < <(find "${results_dir}" -type f -name failure.md | sort) + } + + for platform in ios android; do + if render_platform_summary "${platform}" && [ -f "rendered/${platform}/summary.md" ]; then + rendered_count=$((rendered_count + 1)) + fi + done + + if [ "${rendered_count}" -eq 0 ]; then + echo "::warning::No benchmark summaries were rendered." + fi + + echo "rendered_count=${rendered_count}" >> "$GITHUB_OUTPUT" + + - name: Publish plot assets + id: publish_plots + shell: bash + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + ASSET_BRANCH: mobench-plots + run: | + set -euo pipefail + + if ! find rendered -type f -path "*/plots/*.svg" | grep -q .; then + echo "base_url=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + remote="https://x-access-token:${GH_TOKEN}@github.com/${REPO}.git" + asset_path="runs/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + publish_root="$(mktemp -d)" + + if git clone --quiet --branch "${ASSET_BRANCH}" "${remote}" "${publish_root}" 2>/dev/null; then + : + else + git clone --quiet "${remote}" "${publish_root}" + git -C "${publish_root}" checkout --orphan "${ASSET_BRANCH}" + git -C "${publish_root}" rm -rf . >/dev/null 2>&1 || true + fi + + git -C "${publish_root}" config user.name "github-actions[bot]" + git -C "${publish_root}" config user.email "41898282+github-actions[bot]@users.noreply.github.com" + mkdir -p "${publish_root}/${asset_path}" + + for platform in ios android; do + if [ -d "rendered/${platform}/plots" ]; then + mkdir -p "${publish_root}/${asset_path}/${platform}" + rm -rf "${publish_root}/${asset_path}/${platform}/plots" + cp -R "rendered/${platform}/plots" "${publish_root}/${asset_path}/${platform}/plots" + fi + done + + git -C "${publish_root}" add "${asset_path}" + if ! git -C "${publish_root}" diff --cached --quiet; then + git -C "${publish_root}" commit -m "mobench plots for run ${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >/dev/null + git -C "${publish_root}" push origin "${ASSET_BRANCH}" >/dev/null + fi + + echo "base_url=https://raw.githubusercontent.com/${REPO}/${ASSET_BRANCH}/${asset_path}" >> "$GITHUB_OUTPUT" + + - name: Rewrite platform summaries for GitHub markdown + shell: bash + env: + PLOT_BASE_URL: ${{ steps.publish_plots.outputs.base_url }} + run: | + set -euo pipefail + + rewrite_platform_summary() { + local platform="$1" + local input="rendered/${platform}/summary.md" + local output="rendered/${platform}/github-summary.md" + if [ ! -f "${input}" ]; then + return 0 + fi + + cp "${input}" "${output}" + if [ -n "${PLOT_BASE_URL:-}" ] && [ -d "rendered/${platform}/plots" ]; then + sed -i "s#](plots/#](${PLOT_BASE_URL}/${platform}/plots/#g" "${output}" + fi + } + + rewrite_platform_summary ios + rewrite_platform_summary android + + - name: Post sticky PR comment + if: inputs.pr_number != '' && steps.render_summaries.outputs.rendered_count != '0' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ inputs.pr_number }} + REPO: ${{ inputs.report_repository != '' && inputs.report_repository || github.repository }} + run: | + set -euo pipefail + MARKER="" + BODY="${MARKER} + ## Mobench Benchmark Results + + " + + for platform in ios android; do + PLATFORM_MD_FILE="rendered/${platform}/github-summary.md" + if [ -f "${PLATFORM_MD_FILE}" ]; then + PLATFORM_MD=$(cat "${PLATFORM_MD_FILE}") + BODY="${BODY}${PLATFORM_MD} + + " + fi + done + + BODY="${BODY} + --- + *Posted by [mobench](https://github.com/worldcoin/mobile-bench-rs) at $(date -u '+%Y-%m-%d %H:%M UTC')*" + + comments_json="$(mktemp)" + if gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" > "${comments_json}"; then + EXISTING_COMMENT_ID=$(jq -r --arg marker "${MARKER}" '.[] | select(.body | contains($marker)) | .id' "${comments_json}" | head -1) + else + echo "::warning::Unable to list comments for ${REPO}#${PR_NUMBER}; skipping sticky benchmark comment." + exit 0 + fi + + if [ -n "$EXISTING_COMMENT_ID" ]; then + gh api "repos/${REPO}/issues/comments/${EXISTING_COMMENT_ID}" \ + -X PATCH \ + -f body="${BODY}" \ + --silent || echo "::warning::Unable to update sticky benchmark comment ${EXISTING_COMMENT_ID} in ${REPO}." + else + gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ + -f body="${BODY}" \ + --silent || echo "::warning::Unable to create sticky benchmark comment in ${REPO}#${PR_NUMBER}." + fi diff --git a/.github/workflows/mobile-bench.yml b/.github/workflows/mobile-bench.yml new file mode 100644 index 000000000..60f044650 --- /dev/null +++ b/.github/workflows/mobile-bench.yml @@ -0,0 +1,136 @@ +name: Mobile Benchmarks + +on: + workflow_dispatch: + inputs: + crate_path: + description: "Path to the benchmark crate" + required: false + type: string + default: "./bench-mobile" + functions: + description: "JSON array of benchmark functions" + required: false + type: string + default: '["bench_mobile::bench_passport_complete_age_check_prove","bench_mobile::bench_oprf_prove","bench_mobile::bench_p256_bigcurve_prove"]' + functions_ios: + description: "Optional iOS-specific benchmark functions" + required: false + type: string + default: '["bench_mobile::bench_passport_complete_age_check_prove","bench_mobile::bench_oprf_prove","bench_mobile::bench_p256_bigcurve_prove"]' + functions_android: + description: "Optional Android-specific benchmark functions" + required: false + type: string + default: '["bench_mobile::bench_oprf_prove","bench_mobile::bench_p256_bigcurve_prove","bench_mobile::bench_passport_complete_age_check_prove"]' + platform: + description: "android | ios | both" + required: false + type: choice + default: both + options: + - android + - ios + - both + device_profile: + description: "Device profile to run" + required: false + type: choice + default: "triad" + options: + - smoke + - triad + - worst + iterations: + description: "Number of benchmark iterations" + required: false + type: string + default: "2" + warmup: + description: "Number of warmup iterations" + required: false + type: string + default: "1" + mobench_version: + description: "Mobench release version to install when mobench_ref is empty" + required: false + type: string + default: "0.1.41" + mobench_ref: + description: "Optional mobile-bench-rs Git ref to install instead of a released version" + required: false + type: string + default: "codex/native-c-abi-backend" + pr_number: + description: "PR number for reporting" + required: false + type: string + default: "" + report_repository: + description: "owner/repo to receive the sticky benchmark comment; defaults to this repository" + required: false + type: string + default: "" + head_sha: + description: "Exact commit SHA to benchmark" + required: false + type: string + default: "" + requested_by: + description: "Who triggered the run" + required: false + type: string + default: "" + +permissions: + contents: write + actions: read + pull-requests: write + issues: write + +concurrency: + group: mobench-${{ inputs.pr_number != '' && inputs.pr_number || github.run_id }} + cancel-in-progress: false + +jobs: + browserstack-preflight: + name: BrowserStack preflight + runs-on: ubuntu-latest + environment: Browserstack + outputs: + available: ${{ steps.check.outputs.available }} + env: + BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }} + BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} + steps: + - name: Check BrowserStack secrets + id: check + shell: bash + run: | + if [ -n "$BROWSERSTACK_USERNAME" ] && [ -n "$BROWSERSTACK_ACCESS_KEY" ]; then + echo "available=true" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + fi + + browserstack: + name: BrowserStack benchmarks + needs: browserstack-preflight + if: ${{ needs.browserstack-preflight.outputs.available == 'true' }} + uses: ./.github/workflows/mobile-bench-reusable.yml + secrets: inherit + with: + crate_path: ${{ inputs.crate_path }} + functions: ${{ inputs.functions }} + functions_ios: ${{ inputs.functions_ios }} + functions_android: ${{ inputs.functions_android }} + platform: ${{ inputs.platform }} + device_profile: ${{ inputs.device_profile }} + iterations: ${{ inputs.iterations }} + warmup: ${{ inputs.warmup }} + mobench_version: ${{ inputs.mobench_version }} + mobench_ref: ${{ inputs.mobench_ref }} + pr_number: ${{ inputs.pr_number }} + report_repository: ${{ inputs.report_repository }} + head_sha: ${{ inputs.head_sha }} + requested_by: ${{ inputs.requested_by }} diff --git a/Cargo.lock b/Cargo.lock index 92ae8c401..05ee0fa84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -697,6 +697,19 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bench-mobile" +version = "0.1.0" +dependencies = [ + "anyhow", + "inventory", + "mobench-sdk", + "provekit-ffi", + "serde", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "binary-merge" version = "0.1.2" @@ -2725,6 +2738,15 @@ dependencies = [ "smallvec", ] +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -3265,6 +3287,29 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mobench-macros" +version = "0.1.41" +source = "git+https://github.com/worldcoin/mobile-bench-rs?branch=codex%2Fnative-c-abi-backend#17f47930e41f0ad94cdcf7c8c6b071b1e87e43f0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "mobench-sdk" +version = "0.1.41" +source = "git+https://github.com/worldcoin/mobile-bench-rs?branch=codex%2Fnative-c-abi-backend#17f47930e41f0ad94cdcf7c8c6b071b1e87e43f0" +dependencies = [ + "inventory", + "libc", + "mobench-macros", + "serde", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "nargo" version = "1.0.0-beta.19" @@ -4662,12 +4707,14 @@ dependencies = [ "nargo_cli", "nargo_toml", "noirc_abi", + "noirc_artifacts", "noirc_driver", "parking_lot", "provekit-common", "provekit-prover", "provekit-r1cs-compiler", "provekit-verifier", + "serde_json", "tempfile", ] diff --git a/Cargo.toml b/Cargo.toml index 73d5ac541..7d94cadf4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] resolver = "2" members = [ + "bench-mobile", "skyscraper/fp-rounding", "skyscraper/hla", "skyscraper/bn254-multiplier", @@ -119,6 +120,8 @@ chrono = "0.4.41" divan = "0.1.21" hex = "0.4.3" itertools = "0.14.0" +inventory = "0.3" +mobench-sdk = { git = "https://github.com/worldcoin/mobile-bench-rs", branch = "codex/native-c-abi-backend", default-features = false, features = ["registry"] } num-bigint = "0.4" paste = "1.0.15" postcard = { version = "1.1.1", features = ["use-std"] } @@ -150,6 +153,7 @@ tracing-subscriber = { version = "0.3.18", features = ["env-filter", "ansi"] } tracing-tracy = "=0.11.4" tracy-client = "=0.18.0" tracy-client-sys = "=0.24.3" +uniffi = "0.28" parking_lot = "0.12" # Version-anchored: acvm_blackbox_solver (noir beta.19) requires keccak = "0.2.0-rc.0" # and calls keccak::f1600(), which was removed in keccak 0.2.0 stable. Pinning to diff --git a/bench-mobile/Cargo.toml b/bench-mobile/Cargo.toml new file mode 100644 index 000000000..1f5cc8cf5 --- /dev/null +++ b/bench-mobile/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "bench-mobile" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +publish = false +description = "Mobile benchmarks for ProveKit Noir passport proving" + +[package.metadata.cargo-machete] +ignored = ["inventory"] + +[lib] +crate-type = ["lib", "cdylib", "staticlib"] + +[dependencies] +anyhow.workspace = true +inventory.workspace = true +mobench-sdk.workspace = true +provekit-ffi.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror = "1.0" + +[lints] +workspace = true diff --git a/bench-mobile/README.md b/bench-mobile/README.md new file mode 100644 index 000000000..ba80b830f --- /dev/null +++ b/bench-mobile/README.md @@ -0,0 +1,259 @@ +# bench-mobile + +`bench-mobile` is ProveKit's mobile benchmark crate. It packages selected +ProveKit proving workloads behind the interface expected by +[mobench](https://github.com/worldcoin/mobile-bench-rs) so the same Rust code +can be built into Android and iOS runners, executed on real devices, and +reported through the CI workflow. + +The current scope covers three Noir examples: + +- source circuits: + `noir-examples/noir-passport-monolithic/complete_age_check` + `noir-examples/oprf` + `noir-examples/p256_bigcurve` +- benchmark inputs from the source examples: + `noir-examples/noir-passport-monolithic/complete_age_check/Prover.toml` + `noir-examples/oprf/Prover.toml` + `noir-examples/p256_bigcurve/Prover.toml` + +## What ProveKit uses mobench for + +ProveKit uses `mobench` to answer one question: how expensive are our proving +steps on real mobile hardware? + +This crate exposes prepare, prove, verify, and end-to-end benchmark functions +for each embedded fixture: + +- `bench_mobile::bench_passport_complete_age_check_prepare` +- `bench_mobile::bench_passport_complete_age_check_prove` +- `bench_mobile::bench_passport_complete_age_check_verify` +- `bench_mobile::bench_passport_complete_age_check_e2e` +- `bench_mobile::bench_oprf_prepare` +- `bench_mobile::bench_oprf_prove` +- `bench_mobile::bench_oprf_verify` +- `bench_mobile::bench_oprf_e2e` +- `bench_mobile::bench_p256_bigcurve_prepare` +- `bench_mobile::bench_p256_bigcurve_prove` +- `bench_mobile::bench_p256_bigcurve_verify` +- `bench_mobile::bench_p256_bigcurve_e2e` + +They let us measure different slices of the passport proving pipeline: + +- `prepare`: deserialize the Noir artifact, build the proof scheme, and produce + prover/verifier state +- `prove`: generate the proof from prepared prover state and parsed inputs +- `verify`: verify a prepared proof against a prepared verifier +- `e2e`: run prepare, prove, and verify in one measured benchmark + +That split matters because proving is not the whole story. On mobile devices we +care about setup cost, proof cost, verifier cost, and the full end-to-end path. + +## How mobench works with this crate + +At a high level, the flow is: + +1. `cargo-mobench build` cross-compiles this crate and generates a mobile test + runner app +2. the generated Android/iOS app receives a benchmark spec containing: + - function name + - measured iteration count + - warmup iteration count +3. the app calls the native JSON C ABI exported by `mobench-sdk` +4. `run_benchmark` forwards to `mobench_sdk::run_benchmark(...)` +5. `mobench-sdk` discovers the selected `#[benchmark]` function, performs + warmups, measures iterations, and returns a structured report +6. the mobile runner logs that report, and `mobench` turns the fetched device + output into CI artifacts such as: + - `summary.json` + - `summary.md` + - `results.csv` + +Inside this crate: + +- benchmark registration comes from `#[benchmark]` +- phase-level timing comes from `profile_phase(...)` +- the mobile boundary is the `mobench_run_benchmark_json` C symbol exported by + `mobench_sdk::export_native_c_abi!()` + +The exported report preserves the fields the generated mobile runners care +about: + +- wall-clock sample durations +- sample CPU time +- sample peak memory +- semantic phases +- harness timeline spans + +## How the benchmark code is structured + +```text +bench-mobile/ +├── Cargo.toml +├── README.md +├── build.rs +├── src/ +│ ├── examples.rs +│ ├── lib.rs +│ └── passport.rs +├── scripts/ +│ └── generate-fixtures.sh +└── tests/ + └── passport_smoke.rs +``` + +### `Cargo.toml` + +Declares `bench-mobile` as a library crate that can be built as: + +- `lib` +- `cdylib` +- `staticlib` + +Those crate types are what `mobench` needs to package the Rust code into mobile +artifacts. + +### `build.rs` + +Copies Noir artifacts generated under the source examples' `target/` +directories into Cargo's `OUT_DIR` so the mobile runner can embed them without +checking compiled JSON into git. + +### `src/lib.rs` + +This is the integration surface between ProveKit and `mobench`. + +It does three jobs: + +1. exports the native mobench JSON C ABI +2. keeps a host-side `run_benchmark(spec)` wrapper for tests and diagnostics +3. registers the benchmark functions themselves + +It also contains the benchmark-specific execution policy: + +- `prepare` measures raw fixture preparation +- `prove` reuses a thread-local prepared fixture so the measured region is proof + generation, not setup +- `verify` reuses a thread-local verified fixture so the measured region is + verification, not proof generation +- `e2e` measures the full path in one run + +### `src/examples.rs` + +Contains shared fixture loading, proving, and verification code for the +embedded Noir examples used by mobile benchmarks. + +### `src/passport.rs` + +Contains the ProveKit-specific benchmark fixture logic: + +- load the embedded Noir program artifact +- parse the source example `Prover.toml` +- prepare, prove, and verify through `provekit-ffi`'s in-process helper API + +This file is where the mobile benchmark stays tied to real ProveKit proving +code through `provekit-ffi` instead of synthetic stand-ins. + +### Generated Noir artifacts + +Compiled Noir JSON artifacts are generated by +`bench-mobile/scripts/generate-fixtures.sh` before CI or BrowserStack builds. +The generated files stay under each source example's ignored `target/` +directory and are copied into the mobile crate at build time. + +### `tests/passport_smoke.rs` + +Host-side smoke tests for the embedded fixture: + +- fixture preparation produces non-empty proving artifacts +- the embedded passport example can prove and verify successfully + +These are not mobile performance tests. They are correctness checks that keep +the benchmark fixture from silently drifting out of shape. + +### `tests/examples_smoke.rs` + +Host-side smoke tests for the shared fixture loader and the OPRF/p256 fixtures. +They verify that the embedded examples prepare, prove, and verify successfully. + +## Benchmark behavior and measurement boundaries + +The crate tries to keep the measured region tight: + +- benchmark setup and fixture parsing are excluded from `prove` and `verify` + measurements via cached thread-local fixtures +- `prepare` exists separately so setup cost is still measured explicitly +- `e2e` is available when we do want the full pipeline cost +- `black_box(...)` is used so benchmark outputs are not optimized away + +This matters because mobile benchmarking gets misleading very quickly if +artifact loading, serialization, and unrelated setup leak into every measured +iteration. + +## Refreshing fixtures + +Install the Noir toolchain expected by the repo: + +```bash +noirup --version v1.0.0-beta.19 +``` + +Generate the Noir artifacts consumed by the benchmark build: + +```bash +bench-mobile/scripts/generate-fixtures.sh +``` + +If a circuit or ABI changes, regenerate the artifacts before running +`bench-mobile` tests or mobile packaging. The generated JSON remains ignored. + +## Local mobench usage + +Build the mobile artifacts: + +```bash +cargo-mobench build --target ios --release --crate-path bench-mobile +cargo-mobench build --target android --release --crate-path bench-mobile +``` + +Repo-level `mobench` defaults live in `mobench.toml` at the workspace root. In +this repository that file pins Android packaging to `arm64-v8a`, which matches +the real-device CI path and avoids unsupported `armeabi-v7a` builds in +`skyscraper/fp-rounding`. It also sets `ffi_backend = "native-c-abi"`, so the +generated Android and iOS runners call the C ABI directly and skip UniFFI +binding generation. + +Run a local or CI-managed benchmark by selecting one of the exported benchmark +function names. The important knobs are: + +- `--function`: which benchmark to run +- `--iterations`: measured iterations +- `--warmup`: warmup iterations +- `--target`: `android` or `ios` + +For CI and BrowserStack runs, the repo workflows wrap these commands and fetch +the resulting reports back into `target/mobench/ci/...`. + +## BrowserStack device profiles used in this repo + +PR benchmarks run the triad profile by default: + +- Android: + - `Vivo Y21-11.0` + - `Google Pixel 7-13.0` + - `Samsung Galaxy S24-14.0` +- iOS: + - `iPhone SE 2020-16` + - `iPhone 15-17` + - `iPhone 16 Pro-18` + +Manual workflow dispatches and `/mobench` comments can select `smoke`, +`worst`, or `triad`; when omitted, PR commands also default to `triad`. + +The low-spec pair used for worst-case checks is: + +- Android: `Vivo Y21-11.0` +- iOS: `iPhone 7-10` + +The sticky PR comment is updated in place using the `` +marker so each rerun replaces the previous report. diff --git a/bench-mobile/build.rs b/bench-mobile/build.rs new file mode 100644 index 000000000..b57d51708 --- /dev/null +++ b/bench-mobile/build.rs @@ -0,0 +1,75 @@ +use std::{ + env, fs, io, + path::{Path, PathBuf}, +}; + +struct FixtureArtifact { + output_file: &'static str, + source_target_rel: &'static str, +} + +const FIXTURE_ARTIFACTS: &[FixtureArtifact] = &[ + FixtureArtifact { + output_file: "complete_age_check.json", + source_target_rel: "noir-examples/noir-passport-monolithic/complete_age_check/target/\ + complete_age_check.json", + }, + FixtureArtifact { + output_file: "oprf.json", + source_target_rel: "noir-examples/oprf/target/oprf.json", + }, + FixtureArtifact { + output_file: "p256.json", + source_target_rel: "noir-examples/p256_bigcurve/target/p256.json", + }, +]; + +fn copy_if_present(from: &Path, to: &Path) -> io::Result { + if from.exists() { + fs::copy(from, to)?; + Ok(true) + } else { + Ok(false) + } +} + +fn main() { + let manifest_dir = + PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR")); + let workspace_dir = manifest_dir + .parent() + .expect("bench-mobile crate should live at workspace root") + .to_path_buf(); + let out_dir = + PathBuf::from(env::var_os("OUT_DIR").expect("OUT_DIR")).join("bench_mobile_fixtures"); + let artifact_dir = env::var_os("PROVEKIT_MOBILE_BENCH_ARTIFACT_DIR").map(PathBuf::from); + + fs::create_dir_all(&out_dir).expect("create generated fixture output dir"); + + for artifact in FIXTURE_ARTIFACTS { + let out_path = out_dir.join(artifact.output_file); + let mut copied = false; + + if let Some(dir) = artifact_dir.as_ref() { + copied = copy_if_present(&dir.join(artifact.output_file), &out_path) + .expect("copy mobile benchmark artifact from override dir"); + println!("cargo:rerun-if-env-changed=PROVEKIT_MOBILE_BENCH_ARTIFACT_DIR"); + } + + if !copied { + let source_path = workspace_dir.join(artifact.source_target_rel); + copied = copy_if_present(&source_path, &out_path) + .expect("copy mobile benchmark artifact from Noir target dir"); + println!("cargo:rerun-if-changed={}", source_path.display()); + } + + if !copied { + println!( + "cargo:warning=missing generated Noir artifact {}; run the mobile fixture \ + generation workflow step before executing bench-mobile tests", + artifact.output_file + ); + fs::write(&out_path, "{}\n").expect("write placeholder mobile benchmark artifact"); + } + } +} diff --git a/bench-mobile/scripts/generate-fixtures.sh b/bench-mobile/scripts/generate-fixtures.sh new file mode 100755 index 000000000..6945bf30e --- /dev/null +++ b/bench-mobile/scripts/generate-fixtures.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" + +compile_fixture() { + local circuit_dir="$1" + echo "Generating Noir artifact in ${circuit_dir}" + ( + cd "${repo_root}/${circuit_dir}" + nargo compile --skip-brillig-constraints-check --force + ) +} + +compile_fixture "noir-examples/noir-passport-monolithic/complete_age_check" +compile_fixture "noir-examples/oprf" +compile_fixture "noir-examples/p256_bigcurve" diff --git a/bench-mobile/src/examples.rs b/bench-mobile/src/examples.rs new file mode 100644 index 000000000..3c8ba1119 --- /dev/null +++ b/bench-mobile/src/examples.rs @@ -0,0 +1,79 @@ +use { + anyhow::{Context, Result}, + provekit_ffi::in_process::{ + prepare_noir_program_from_json, PreparedNoirProgram, VerifiedNoirProgram, + }, +}; + +const COMPLETE_AGE_CHECK_PROGRAM: &str = include_str!(concat!( + env!("OUT_DIR"), + "/bench_mobile_fixtures/complete_age_check.json" +)); +const COMPLETE_AGE_CHECK_TOML: &str = + include_str!("../../noir-examples/noir-passport-monolithic/complete_age_check/Prover.toml"); +const OPRF_PROGRAM: &str = + include_str!(concat!(env!("OUT_DIR"), "/bench_mobile_fixtures/oprf.json")); +const OPRF_TOML: &str = include_str!("../../noir-examples/oprf/Prover.toml"); +const P256_BIGCURVE_PROGRAM: &str = + include_str!(concat!(env!("OUT_DIR"), "/bench_mobile_fixtures/p256.json")); +const P256_BIGCURVE_TOML: &str = include_str!("../../noir-examples/p256_bigcurve/Prover.toml"); + +#[derive(Clone, Copy)] +pub enum MobileBenchFixture { + CompleteAgeCheck, + Oprf, + P256Bigcurve, +} + +impl MobileBenchFixture { + fn name(self) -> &'static str { + match self { + Self::CompleteAgeCheck => "complete_age_check", + Self::Oprf => "oprf", + Self::P256Bigcurve => "p256_bigcurve", + } + } + + fn program_json(self) -> &'static str { + match self { + Self::CompleteAgeCheck => COMPLETE_AGE_CHECK_PROGRAM, + Self::Oprf => OPRF_PROGRAM, + Self::P256Bigcurve => P256_BIGCURVE_PROGRAM, + } + } + + fn prover_toml(self) -> &'static str { + match self { + Self::CompleteAgeCheck => COMPLETE_AGE_CHECK_TOML, + Self::Oprf => OPRF_TOML, + Self::P256Bigcurve => P256_BIGCURVE_TOML, + } + } +} + +pub type PreparedCircuitFixture = PreparedNoirProgram; +pub type VerifiedCircuitFixture = VerifiedNoirProgram; + +pub fn prepare_fixture(fixture: MobileBenchFixture) -> Result { + prepare_noir_program_from_json( + fixture.name(), + fixture.program_json(), + fixture.prover_toml(), + ) + .with_context(|| format!("while preparing {} benchmark fixture", fixture.name())) +} + +pub fn prove_fixture(prepared: PreparedCircuitFixture) -> Result { + prepared.prove() +} + +pub fn verify_fixture(verified: VerifiedCircuitFixture) -> Result { + verified.verify() +} + +pub fn fixture_end_to_end_smoke(fixture: MobileBenchFixture) -> Result<()> { + let prepared = prepare_fixture(fixture)?; + let verified = prove_fixture(prepared)?; + let _verified = verify_fixture(verified)?; + Ok(()) +} diff --git a/bench-mobile/src/lib.rs b/bench-mobile/src/lib.rs new file mode 100644 index 000000000..00b331675 --- /dev/null +++ b/bench-mobile/src/lib.rs @@ -0,0 +1,546 @@ +//! Mobile benchmarks for ProveKit's monolithic passport circuit. + +use { + crate::passport::{ + prove_complete_age_check_fixture, verify_complete_age_check_fixture, + PreparedCompleteAgeCheckFixture, VerifiedCompleteAgeCheckFixture, + }, + examples::{MobileBenchFixture, PreparedCircuitFixture, VerifiedCircuitFixture}, + mobench_sdk::{benchmark, profile_phase}, + serde_json::json, + std::{cell::RefCell, hint::black_box}, +}; + +pub mod examples; +pub mod passport; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct BenchSpec { + pub name: String, + pub iterations: u32, + pub warmup: u32, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct BenchSample { + pub duration_ns: u64, + pub cpu_time_ms: Option, + pub peak_memory_kb: Option, + pub process_peak_memory_kb: Option, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct SemanticPhase { + pub name: String, + pub duration_ns: u64, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct HarnessTimelineSpan { + pub phase: String, + pub start_offset_ns: u64, + pub end_offset_ns: u64, + pub iteration: Option, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct BenchReport { + pub spec: BenchSpec, + pub samples: Vec, + pub phases: Vec, + pub timeline: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum BenchError { + #[error("iterations must be greater than zero")] + InvalidIterations, + + #[error("unknown benchmark function: {name}")] + UnknownFunction { name: String }, + + #[error("benchmark execution failed: {reason}")] + ExecutionFailed { reason: String }, +} + +impl From for BenchSpec { + fn from(spec: mobench_sdk::BenchSpec) -> Self { + Self { + name: spec.name, + iterations: spec.iterations, + warmup: spec.warmup, + } + } +} + +impl From for mobench_sdk::BenchSpec { + fn from(spec: BenchSpec) -> Self { + Self { + name: spec.name, + iterations: spec.iterations, + warmup: spec.warmup, + } + } +} + +impl From for BenchSample { + fn from(sample: mobench_sdk::BenchSample) -> Self { + Self { + duration_ns: sample.duration_ns, + cpu_time_ms: sample.cpu_time_ms, + peak_memory_kb: sample.peak_memory_kb, + process_peak_memory_kb: sample.process_peak_memory_kb, + } + } +} + +impl From for SemanticPhase { + fn from(phase: mobench_sdk::SemanticPhase) -> Self { + Self { + name: phase.name, + duration_ns: phase.duration_ns, + } + } +} + +impl From for HarnessTimelineSpan { + fn from(span: mobench_sdk::HarnessTimelineSpan) -> Self { + Self { + phase: span.phase, + start_offset_ns: span.start_offset_ns, + end_offset_ns: span.end_offset_ns, + iteration: span.iteration, + } + } +} + +impl From for BenchReport { + fn from(report: mobench_sdk::RunnerReport) -> Self { + Self { + spec: report.spec.into(), + samples: report.samples.into_iter().map(Into::into).collect(), + phases: report.phases.into_iter().map(Into::into).collect(), + timeline: report.timeline.into_iter().map(Into::into).collect(), + } + } +} + +impl From for BenchError { + fn from(err: mobench_sdk::BenchError) -> Self { + match err { + mobench_sdk::BenchError::Runner(runner_err) => Self::ExecutionFailed { + reason: runner_err.to_string(), + }, + mobench_sdk::BenchError::UnknownFunction(name, _available) => { + Self::UnknownFunction { name } + } + _ => Self::ExecutionFailed { + reason: err.to_string(), + }, + } + } +} + +fn log_benchmark_lifecycle( + event: &str, + function: &str, + iterations: u32, + warmup: u32, + extra: serde_json::Value, +) { + let payload = json!({ + "tag": "MOBENCH_LIFECYCLE", + "event": event, + "function": function, + "iterations": iterations, + "warmup": warmup, + "extra": extra, + }); + + if event == "error" { + eprintln!("{payload}"); + } else { + println!("{payload}"); + } +} + +pub fn run_benchmark(spec: BenchSpec) -> Result { + let function = spec.name.clone(); + let iterations = spec.iterations; + let warmup = spec.warmup; + log_benchmark_lifecycle( + "start", + &function, + iterations, + warmup, + json!({ + "resolved_function": function, + }), + ); + + let sdk_spec: mobench_sdk::BenchSpec = spec.into(); + match mobench_sdk::run_benchmark(sdk_spec) { + Ok(report) => { + log_benchmark_lifecycle( + "success", + &report.spec.name, + report.spec.iterations, + report.spec.warmup, + json!({ + "sample_count": report.samples.len(), + "phase_count": report.phases.len(), + "timeline_span_count": report.timeline.len(), + "sample_resource_count": report + .samples + .iter() + .filter(|sample| { + sample.cpu_time_ms.is_some() + || sample.peak_memory_kb.is_some() + || sample.process_peak_memory_kb.is_some() + }) + .count(), + }), + ); + Ok(report.into()) + } + Err(err) => { + log_benchmark_lifecycle( + "error", + &function, + iterations, + warmup, + json!({ + "resolved_function": function, + "error": err.to_string(), + }), + ); + Err(err.into()) + } + } +} + +mobench_sdk::export_native_c_abi!(); + +thread_local! { + static PREPARED_COMPLETE_AGE_CHECK: RefCell> = + const { RefCell::new(None) }; + static VERIFIED_COMPLETE_AGE_CHECK: RefCell> = + const { RefCell::new(None) }; + static PREPARED_OPRF: RefCell> = + const { RefCell::new(None) }; + static VERIFIED_OPRF: RefCell> = + const { RefCell::new(None) }; + static PREPARED_P256_BIGCURVE: RefCell> = + const { RefCell::new(None) }; + static VERIFIED_P256_BIGCURVE: RefCell> = + const { RefCell::new(None) }; +} + +fn with_prepared_complete_age_check(f: impl FnOnce(&PreparedCompleteAgeCheckFixture) -> T) -> T { + PREPARED_COMPLETE_AGE_CHECK.with(|cache| { + if cache.borrow().is_none() { + *cache.borrow_mut() = Some( + passport::prepare_complete_age_check_fixture() + .expect("prepare complete_age_check fixture"), + ); + } + + let cache_ref = cache.borrow(); + let prepared = cache_ref + .as_ref() + .expect("prepared complete_age_check fixture"); + f(prepared) + }) +} + +fn with_verified_complete_age_check(f: impl FnOnce(&VerifiedCompleteAgeCheckFixture) -> T) -> T { + VERIFIED_COMPLETE_AGE_CHECK.with(|cache| { + if cache.borrow().is_none() { + let prepared = passport::prepare_complete_age_check_fixture().expect("prepare fixture"); + let verified = prove_complete_age_check_fixture(prepared).expect("prove fixture"); + *cache.borrow_mut() = Some(verified); + } + + let cache_ref = cache.borrow(); + let verified = cache_ref + .as_ref() + .expect("verified complete_age_check fixture"); + f(verified) + }) +} + +fn with_prepared_oprf(f: impl FnOnce(&PreparedCircuitFixture) -> T) -> T { + PREPARED_OPRF.with(|cache| { + if cache.borrow().is_none() { + *cache.borrow_mut() = Some( + examples::prepare_fixture(MobileBenchFixture::Oprf).expect("prepare oprf fixture"), + ); + } + + let cache_ref = cache.borrow(); + let prepared = cache_ref.as_ref().expect("prepared oprf fixture"); + f(prepared) + }) +} + +fn with_verified_oprf(f: impl FnOnce(&VerifiedCircuitFixture) -> T) -> T { + VERIFIED_OPRF.with(|cache| { + if cache.borrow().is_none() { + let prepared = + examples::prepare_fixture(MobileBenchFixture::Oprf).expect("prepare oprf fixture"); + let verified = examples::prove_fixture(prepared).expect("prove oprf fixture"); + *cache.borrow_mut() = Some(verified); + } + + let cache_ref = cache.borrow(); + let verified = cache_ref.as_ref().expect("verified oprf fixture"); + f(verified) + }) +} + +fn with_prepared_p256_bigcurve(f: impl FnOnce(&PreparedCircuitFixture) -> T) -> T { + PREPARED_P256_BIGCURVE.with(|cache| { + if cache.borrow().is_none() { + *cache.borrow_mut() = Some( + examples::prepare_fixture(MobileBenchFixture::P256Bigcurve) + .expect("prepare p256_bigcurve fixture"), + ); + } + + let cache_ref = cache.borrow(); + let prepared = cache_ref.as_ref().expect("prepared p256_bigcurve fixture"); + f(prepared) + }) +} + +fn with_verified_p256_bigcurve(f: impl FnOnce(&VerifiedCircuitFixture) -> T) -> T { + VERIFIED_P256_BIGCURVE.with(|cache| { + if cache.borrow().is_none() { + let prepared = examples::prepare_fixture(MobileBenchFixture::P256Bigcurve) + .expect("prepare p256_bigcurve fixture"); + let verified = examples::prove_fixture(prepared).expect("prove p256_bigcurve fixture"); + *cache.borrow_mut() = Some(verified); + } + + let cache_ref = cache.borrow(); + let verified = cache_ref.as_ref().expect("verified p256_bigcurve fixture"); + f(verified) + }) +} + +#[benchmark] +pub fn bench_passport_complete_age_check_prepare() { + let prepared = profile_phase("prepare", || { + passport::prepare_complete_age_check_fixture().expect("prepare complete_age_check fixture") + }); + + black_box(( + prepared.prover_size(), + prepared.constraint_count(), + prepared.input_count(), + )); +} + +#[benchmark] +pub fn bench_passport_complete_age_check_prove() { + with_prepared_complete_age_check(|prepared| { + let verified = profile_phase("prove", || { + prove_complete_age_check_fixture(prepared.clone()) + .expect("prove complete_age_check fixture") + }); + + black_box(verified); + }); +} + +#[benchmark] +pub fn bench_passport_complete_age_check_verify() { + with_verified_complete_age_check(|verified| { + let verified = profile_phase("verify", || { + verify_complete_age_check_fixture(verified.clone()) + .expect("verify complete_age_check fixture") + }); + + black_box(verified); + }); +} + +#[benchmark] +pub fn bench_passport_complete_age_check_e2e() { + let prepared = profile_phase("prepare", || { + passport::prepare_complete_age_check_fixture().expect("prepare complete_age_check fixture") + }); + let verified = profile_phase("prove", || { + prove_complete_age_check_fixture(prepared).expect("prove complete_age_check fixture") + }); + let verified = profile_phase("verify", || { + verify_complete_age_check_fixture(verified).expect("verify complete_age_check fixture") + }); + + black_box(verified); +} + +#[benchmark] +pub fn bench_oprf_prepare() { + let prepared = profile_phase("prepare", || { + examples::prepare_fixture(MobileBenchFixture::Oprf).expect("prepare oprf fixture") + }); + + black_box(( + prepared.prover_size(), + prepared.constraint_count(), + prepared.input_count(), + )); +} + +#[benchmark] +pub fn bench_oprf_prove() { + with_prepared_oprf(|prepared| { + let verified = profile_phase("prove", || { + examples::prove_fixture(prepared.clone()).expect("prove oprf fixture") + }); + + black_box(verified); + }); +} + +#[benchmark] +pub fn bench_oprf_verify() { + with_verified_oprf(|verified| { + let verified = profile_phase("verify", || { + examples::verify_fixture(verified.clone()).expect("verify oprf fixture") + }); + + black_box(verified); + }); +} + +#[benchmark] +pub fn bench_oprf_e2e() { + let prepared = profile_phase("prepare", || { + examples::prepare_fixture(MobileBenchFixture::Oprf).expect("prepare oprf fixture") + }); + let verified = profile_phase("prove", || { + examples::prove_fixture(prepared).expect("prove oprf fixture") + }); + let verified = profile_phase("verify", || { + examples::verify_fixture(verified).expect("verify oprf fixture") + }); + + black_box(verified); +} + +#[benchmark] +pub fn bench_p256_bigcurve_prepare() { + let prepared = profile_phase("prepare", || { + examples::prepare_fixture(MobileBenchFixture::P256Bigcurve) + .expect("prepare p256_bigcurve fixture") + }); + + black_box(( + prepared.prover_size(), + prepared.constraint_count(), + prepared.input_count(), + )); +} + +#[benchmark] +pub fn bench_p256_bigcurve_prove() { + with_prepared_p256_bigcurve(|prepared| { + let verified = profile_phase("prove", || { + examples::prove_fixture(prepared.clone()).expect("prove p256_bigcurve fixture") + }); + + black_box(verified); + }); +} + +#[benchmark] +pub fn bench_p256_bigcurve_verify() { + with_verified_p256_bigcurve(|verified| { + let verified = profile_phase("verify", || { + examples::verify_fixture(verified.clone()).expect("verify p256_bigcurve fixture") + }); + + black_box(verified); + }); +} + +#[benchmark] +pub fn bench_p256_bigcurve_e2e() { + let prepared = profile_phase("prepare", || { + examples::prepare_fixture(MobileBenchFixture::P256Bigcurve) + .expect("prepare p256_bigcurve fixture") + }); + let verified = profile_phase("prove", || { + examples::prove_fixture(prepared).expect("prove p256_bigcurve fixture") + }); + let verified = profile_phase("verify", || { + examples::verify_fixture(verified).expect("verify p256_bigcurve fixture") + }); + + black_box(verified); +} + +#[cfg(test)] +mod tests { + use super::BenchReport; + + #[test] + fn report_conversion_preserves_sample_resource_metrics() { + let report = mobench_sdk::RunnerReport { + spec: mobench_sdk::BenchSpec { + name: "bench_mobile::bench_passport_complete_age_check_prove".to_string(), + iterations: 1, + warmup: 0, + }, + samples: vec![mobench_sdk::BenchSample { + duration_ns: 123, + cpu_time_ms: Some(7), + peak_memory_kb: Some(48), + process_peak_memory_kb: Some(1024), + }], + phases: vec![], + timeline: vec![], + }; + + let value = + serde_json::to_value(BenchReport::from(report)).expect("serialize bench report"); + + assert_eq!(value["samples"][0]["cpu_time_ms"], 7); + assert_eq!(value["samples"][0]["peak_memory_kb"], 48); + assert_eq!(value["samples"][0]["process_peak_memory_kb"], 1024); + } + + #[test] + fn report_conversion_preserves_timeline_spans() { + let report = mobench_sdk::RunnerReport { + spec: mobench_sdk::BenchSpec { + name: "bench_mobile::bench_passport_complete_age_check_verify".to_string(), + iterations: 1, + warmup: 0, + }, + samples: vec![mobench_sdk::BenchSample { + duration_ns: 321, + cpu_time_ms: None, + peak_memory_kb: None, + process_peak_memory_kb: None, + }], + phases: vec![], + timeline: vec![mobench_sdk::HarnessTimelineSpan { + phase: "measured".to_string(), + start_offset_ns: 10, + end_offset_ns: 20, + iteration: Some(0), + }], + }; + + let value = + serde_json::to_value(BenchReport::from(report)).expect("serialize bench report"); + + assert_eq!(value["timeline"][0]["phase"], "measured"); + assert_eq!(value["timeline"][0]["start_offset_ns"], 10); + assert_eq!(value["timeline"][0]["end_offset_ns"], 20); + assert_eq!(value["timeline"][0]["iteration"], 0); + } +} diff --git a/bench-mobile/src/passport.rs b/bench-mobile/src/passport.rs new file mode 100644 index 000000000..a63b1cdb1 --- /dev/null +++ b/bench-mobile/src/passport.rs @@ -0,0 +1,44 @@ +use { + anyhow::{Context, Result}, + provekit_ffi::in_process::{ + prepare_noir_program_from_json, PreparedNoirProgram, VerifiedNoirProgram, + }, +}; + +const COMPLETE_AGE_CHECK_PROGRAM: &str = include_str!(concat!( + env!("OUT_DIR"), + "/bench_mobile_fixtures/complete_age_check.json" +)); +const COMPLETE_AGE_CHECK_TOML: &str = + include_str!("../../noir-examples/noir-passport-monolithic/complete_age_check/Prover.toml"); + +pub type PreparedCompleteAgeCheckFixture = PreparedNoirProgram; +pub type VerifiedCompleteAgeCheckFixture = VerifiedNoirProgram; + +pub fn prepare_complete_age_check_fixture() -> Result { + prepare_noir_program_from_json( + "complete_age_check", + COMPLETE_AGE_CHECK_PROGRAM, + COMPLETE_AGE_CHECK_TOML, + ) + .context("while preparing complete_age_check benchmark fixture") +} + +pub fn prove_complete_age_check_fixture( + prepared: PreparedCompleteAgeCheckFixture, +) -> Result { + prepared.prove() +} + +pub fn verify_complete_age_check_fixture( + verified: VerifiedCompleteAgeCheckFixture, +) -> Result { + verified.verify() +} + +pub fn passport_complete_age_check_end_to_end_smoke() -> Result<()> { + let prepared = prepare_complete_age_check_fixture()?; + let verified = prove_complete_age_check_fixture(prepared)?; + let _verified = verify_complete_age_check_fixture(verified)?; + Ok(()) +} diff --git a/bench-mobile/tests/examples_smoke.rs b/bench-mobile/tests/examples_smoke.rs new file mode 100644 index 000000000..ecfc17c65 --- /dev/null +++ b/bench-mobile/tests/examples_smoke.rs @@ -0,0 +1,27 @@ +use bench_mobile::examples::{fixture_end_to_end_smoke, prepare_fixture, MobileBenchFixture}; + +#[test] +fn embedded_example_fixtures_prepare_non_empty_artifacts() { + for fixture in [ + MobileBenchFixture::CompleteAgeCheck, + MobileBenchFixture::Oprf, + MobileBenchFixture::P256Bigcurve, + ] { + let prepared = prepare_fixture(fixture).expect("prepare fixture"); + let (constraints, witnesses) = prepared.prover_size(); + + assert!(constraints > 0, "expected non-empty constraint set"); + assert!(witnesses > 0, "expected non-empty witness set"); + } +} + +#[test] +fn embedded_oprf_fixture_proves_and_verifies() { + fixture_end_to_end_smoke(MobileBenchFixture::Oprf).expect("oprf smoke benchmark"); +} + +#[test] +fn embedded_p256_bigcurve_fixture_proves_and_verifies() { + fixture_end_to_end_smoke(MobileBenchFixture::P256Bigcurve) + .expect("p256_bigcurve smoke benchmark"); +} diff --git a/bench-mobile/tests/passport_smoke.rs b/bench-mobile/tests/passport_smoke.rs new file mode 100644 index 000000000..ec2596a5b --- /dev/null +++ b/bench-mobile/tests/passport_smoke.rs @@ -0,0 +1,17 @@ +use bench_mobile::passport::{ + passport_complete_age_check_end_to_end_smoke, prepare_complete_age_check_fixture, +}; + +#[test] +fn embedded_passport_fixture_prepares_non_empty_artifacts() { + let prepared = prepare_complete_age_check_fixture().expect("prepare fixture"); + let (constraints, witnesses) = prepared.prover_size(); + + assert!(constraints > 0, "expected non-empty constraint set"); + assert!(witnesses > 0, "expected non-empty witness set"); +} + +#[test] +fn embedded_passport_fixture_proves_and_verifies() { + passport_complete_age_check_end_to_end_smoke().expect("passport smoke benchmark"); +} diff --git a/mobench.toml b/mobench.toml new file mode 100644 index 000000000..b654550c4 --- /dev/null +++ b/mobench.toml @@ -0,0 +1,20 @@ +[project] +crate = "bench-mobile" +library_name = "bench_mobile" +ffi_backend = "native-c-abi" + +[android] +package = "dev.world.benchmobile" +min_sdk = 24 +target_sdk = 34 +abis = ["arm64-v8a"] + +[ios] +bundle_id = "dev.world.benchmobile" +deployment_target = "10.0" +runner = "uikit-legacy" + +[browserstack] +ios_completion_timeout_secs = 7200 +android_benchmark_timeout_secs = 7200 +android_heartbeat_interval_secs = 10 diff --git a/noir-examples/p256_bigcurve/Nargo.toml b/noir-examples/p256_bigcurve/Nargo.toml index c9df7a53a..78429e631 100644 --- a/noir-examples/p256_bigcurve/Nargo.toml +++ b/noir-examples/p256_bigcurve/Nargo.toml @@ -4,5 +4,5 @@ type = "bin" authors = [""] [dependencies] -bignum = { tag = "v0.9.2", git = "https://github.com/noir-lang/noir-bignum" } -bigcurve = { tag = "v0.13.2", git = "https://github.com/noir-lang/noir_bigcurve" } +bignum = { tag = "v0.8.0", git = "https://github.com/noir-lang/noir-bignum" } +bigcurve = { tag = "v0.11.0", git = "https://github.com/noir-lang/noir_bigcurve" } diff --git a/noir-examples/p256_bigcurve/src/main.nr b/noir-examples/p256_bigcurve/src/main.nr index a6e25d439..4628196bc 100644 --- a/noir-examples/p256_bigcurve/src/main.nr +++ b/noir-examples/p256_bigcurve/src/main.nr @@ -4,13 +4,7 @@ use bigcurve::{ }; use bignum::BigNum; -fn main( - hashed_message: [u8; 32], - pub_key_x: [u8; 32], - pub_key_y: [u8; 32], - signature: [u8; 64], - r_point_y: [u8; 32], -) { +fn main(hashed_message: [u8; 32], pub_key_x: [u8; 32], pub_key_y: [u8; 32], signature: [u8; 64]) { let gen = Secp256r1::one(); let public = Secp256r1 { x: Secp256r1_Fq::from_be_bytes(pub_key_x), @@ -41,7 +35,5 @@ fn main( let s_p = Secp256r1Scalar::from_bignum(r / s); let r_point = Secp256r1::evaluate_linear_expression([gen, public], [s_g, s_p], []); - assert(!r_point.is_infinity); assert(r_point.x == r_x); - assert(r_point.y == Secp256r1_Fq::from_be_bytes(r_point_y)); } diff --git a/tooling/provekit-ffi/Cargo.toml b/tooling/provekit-ffi/Cargo.toml index ccfef3330..3adcd69d0 100644 --- a/tooling/provekit-ffi/Cargo.toml +++ b/tooling/provekit-ffi/Cargo.toml @@ -21,7 +21,9 @@ provekit-verifier = { workspace = true } # 3rd party anyhow.workspace = true noirc_abi.workspace = true +noirc_artifacts.workspace = true parking_lot.workspace = true +serde_json.workspace = true [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/tooling/provekit-ffi/src/in_process.rs b/tooling/provekit-ffi/src/in_process.rs new file mode 100644 index 000000000..3446dcb59 --- /dev/null +++ b/tooling/provekit-ffi/src/in_process.rs @@ -0,0 +1,95 @@ +//! Safe in-process helpers built on the same ProveKit implementation as the C +//! FFI entrypoints. + +use { + anyhow::{Context, Result}, + noirc_abi::{input_parser::Format, InputMap}, + noirc_artifacts::program::ProgramArtifact, + provekit_common::{HashConfig, NoirProof, Prover, Verifier}, + provekit_prover::Prove, + provekit_r1cs_compiler::NoirCompiler, + provekit_verifier::Verify, +}; + +/// Prepared proving and verification state for one Noir benchmark program. +#[derive(Clone)] +pub struct PreparedNoirProgram { + name: String, + prover: Prover, + verifier: Verifier, + input_map: InputMap, +} + +impl PreparedNoirProgram { + /// Return the R1CS size exposed by the prepared prover. + pub fn prover_size(&self) -> (usize, usize) { + self.prover.size() + } + + /// Return the number of R1CS constraints in the prepared verifier. + pub fn constraint_count(&self) -> usize { + self.verifier.r1cs.num_constraints() + } + + /// Return the number of parsed ABI input values. + pub fn input_count(&self) -> usize { + self.input_map.len() + } + + /// Generate and bind a proof to the matching verifier state. + pub fn prove(self) -> Result { + let proof = self + .prover + .prove(self.input_map) + .with_context(|| format!("while proving {} benchmark fixture", self.name))?; + + Ok(VerifiedNoirProgram { + name: self.name, + verifier: self.verifier, + proof, + }) + } +} + +/// Verified-ready proof plus verifier state for one Noir benchmark program. +#[derive(Clone)] +pub struct VerifiedNoirProgram { + name: String, + verifier: Verifier, + proof: NoirProof, +} + +impl VerifiedNoirProgram { + /// Verify the proof against its matching verifier state. + pub fn verify(mut self) -> Result { + self.verifier + .verify(&self.proof) + .with_context(|| format!("while verifying {} benchmark fixture", self.name))?; + + Ok(self) + } +} + +/// Prepare a Noir program from an already-compiled artifact JSON string and a +/// TOML witness input string. +pub fn prepare_noir_program_from_json( + name: impl Into, + program_json: &str, + prover_toml: &str, +) -> Result { + let name = name.into(); + let program: ProgramArtifact = serde_json::from_str(program_json) + .with_context(|| format!("while deserializing {name} program artifact"))?; + let scheme = NoirCompiler::from_program(program, HashConfig::default()) + .with_context(|| format!("while preparing {name} noir proof scheme"))?; + let input_map = Format::Toml + .parse(prover_toml, scheme.abi()) + .with_context(|| format!("while parsing {name} prover inputs"))?; + + Ok(PreparedNoirProgram { + name, + prover: Prover::from_noir_proof_scheme(scheme.clone()), + verifier: Verifier::from_noir_proof_scheme(scheme), + input_map, + }) +} diff --git a/tooling/provekit-ffi/src/lib.rs b/tooling/provekit-ffi/src/lib.rs index d41f0b95b..f0084a5cf 100644 --- a/tooling/provekit-ffi/src/lib.rs +++ b/tooling/provekit-ffi/src/lib.rs @@ -29,6 +29,7 @@ pub mod ffi; mod ffi_allocator; +pub mod in_process; pub mod mmap_allocator; pub mod types; pub mod utils;