From 4132f3aeabf3418319a8c9902a70b1bb6643d008 Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Thu, 28 May 2026 02:03:51 +0300 Subject: [PATCH] ci: add live provider e2e workflow --- .github/workflows/live-provider-e2e.yml | 285 ++++++++++++++++++ docs/features/README.md | 1 + docs/features/live-provider-e2e.md | 78 +++++ scripts/live-provider-e2e.sh | 371 ++++++++++++++++++++++++ 4 files changed, 735 insertions(+) create mode 100644 .github/workflows/live-provider-e2e.yml create mode 100644 docs/features/live-provider-e2e.md create mode 100755 scripts/live-provider-e2e.sh diff --git a/.github/workflows/live-provider-e2e.yml b/.github/workflows/live-provider-e2e.yml new file mode 100644 index 00000000..1c13ac0e --- /dev/null +++ b/.github/workflows/live-provider-e2e.yml @@ -0,0 +1,285 @@ +name: Live Provider E2E + +on: + workflow_dispatch: + inputs: + providers: + description: "Comma-separated providers to run, or all" + required: true + default: "all" + type: string + allow_missing: + description: "Skip providers that are missing configured secrets" + required: true + default: true + type: boolean + sync_checkout: + description: "Exercise Crabbox sync during run smoke tests" + required: true + default: false + type: boolean + smoke_repo: + description: "Repo path to use as the smoke workload" + required: true + default: "." + type: string + runner_label: + description: "GitHub runner label for provider jobs" + required: true + default: "ubuntu-latest" + type: string + +permissions: + contents: read + +env: + GOFLAGS: -mod=readonly -trimpath + GOTOOLCHAIN: local + +jobs: + matrix: + name: Provider Matrix + runs-on: ubuntu-latest + outputs: + providers: ${{ steps.providers.outputs.providers }} + + steps: + - name: Build provider list + id: providers + shell: bash + run: | + set -euo pipefail + all='[ + "aws", + "azure", + "gcp", + "hetzner", + "proxmox", + "parallels", + "local-container", + "ssh", + "exe-dev", + "blacksmith-testbox", + "namespace-devbox", + "semaphore", + "sprites", + "daytona", + "islo", + "e2b", + "modal", + "upstash-box", + "tensorlake", + "cloudflare", + "railway", + "runpod", + "wandb" + ]' + providers="$( + jq -c -n --arg raw "${{ inputs.providers }}" --argjson all "$all" ' + def trim: gsub("^\\s+|\\s+$"; ""); + def canon: + if . == "docker" or . == "container" then "local-container" + elif . == "blacksmith" then "blacksmith-testbox" + elif . == "namespace" then "namespace-devbox" + elif . == "static" or . == "static-ssh" then "ssh" + elif . == "cf" then "cloudflare" + else . + end; + ($raw | ascii_downcase | trim) as $input + | if $input == "all" or $input == "" then $all + else + $input + | split(",") + | map(trim | canon) + | map(select(length > 0)) + | unique + end + ' + )" + echo "providers=$providers" >> "$GITHUB_OUTPUT" + printf '%s\n' "$providers" + + provider-e2e: + name: ${{ matrix.provider }} + needs: matrix + if: ${{ needs.matrix.outputs.providers != '[]' }} + runs-on: ${{ inputs.runner_label }} + timeout-minutes: 45 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + provider: ${{ fromJSON(needs.matrix.outputs.providers) }} + + env: + CRABBOX_BIN: ${{ github.workspace }}/bin/crabbox + CRABBOX_LIVE: "1" + CRABBOX_LIVE_REPO: ${{ inputs.smoke_repo }} + CRABBOX_LIVE_SKIP_MISSING: ${{ inputs.allow_missing && '1' || '0' }} + CRABBOX_PROVIDER_E2E_SYNC: ${{ inputs.sync_checkout && '1' || '0' }} + CRABBOX_PROVIDER_E2E_LOG_DIR: ${{ github.workspace }}/.crabbox/live-provider-e2e + CRABBOX_COORDINATOR: ${{ secrets.CRABBOX_COORDINATOR }} + CRABBOX_COORDINATOR_TOKEN: ${{ secrets.CRABBOX_COORDINATOR_TOKEN }} + CRABBOX_COORDINATOR_ADMIN_TOKEN: ${{ secrets.CRABBOX_COORDINATOR_ADMIN_TOKEN }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_SESSION_TOKEN: ${{ secrets.AWS_SESSION_TOKEN }} + AWS_REGION: ${{ secrets.AWS_REGION }} + CRABBOX_AWS_REGION: ${{ secrets.CRABBOX_AWS_REGION }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + CRABBOX_AZURE_LOCATION: ${{ secrets.CRABBOX_AZURE_LOCATION }} + CRABBOX_AZURE_RESOURCE_GROUP: ${{ secrets.CRABBOX_AZURE_RESOURCE_GROUP }} + GOOGLE_CLOUD_PROJECT: ${{ secrets.GOOGLE_CLOUD_PROJECT }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + GCP_SERVICE_ACCOUNT_JSON: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }} + CRABBOX_GCP_ZONE: ${{ secrets.CRABBOX_GCP_ZONE }} + HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + HETZNER_TOKEN: ${{ secrets.HETZNER_TOKEN }} + CRABBOX_PROXMOX_API_URL: ${{ secrets.CRABBOX_PROXMOX_API_URL }} + CRABBOX_PROXMOX_TOKEN_ID: ${{ secrets.CRABBOX_PROXMOX_TOKEN_ID }} + CRABBOX_PROXMOX_TOKEN_SECRET: ${{ secrets.CRABBOX_PROXMOX_TOKEN_SECRET }} + CRABBOX_PROXMOX_NODE: ${{ secrets.CRABBOX_PROXMOX_NODE }} + CRABBOX_PROXMOX_TEMPLATE_ID: ${{ secrets.CRABBOX_PROXMOX_TEMPLATE_ID }} + CRABBOX_PROXMOX_STORAGE: ${{ secrets.CRABBOX_PROXMOX_STORAGE }} + CRABBOX_PROXMOX_BRIDGE: ${{ secrets.CRABBOX_PROXMOX_BRIDGE }} + CRABBOX_PROXMOX_USER: ${{ secrets.CRABBOX_PROXMOX_USER }} + CRABBOX_PROXMOX_INSECURE_TLS: ${{ secrets.CRABBOX_PROXMOX_INSECURE_TLS }} + CRABBOX_PARALLELS_SOURCE: ${{ secrets.CRABBOX_PARALLELS_SOURCE }} + CRABBOX_PARALLELS_SOURCE_SNAPSHOT: ${{ secrets.CRABBOX_PARALLELS_SOURCE_SNAPSHOT }} + CRABBOX_PARALLELS_TEMPLATE: ${{ secrets.CRABBOX_PARALLELS_TEMPLATE }} + CRABBOX_PARALLELS_HOST: ${{ secrets.CRABBOX_PARALLELS_HOST }} + CRABBOX_PARALLELS_HOST_USER: ${{ secrets.CRABBOX_PARALLELS_HOST_USER }} + CRABBOX_PARALLELS_USER: ${{ secrets.CRABBOX_PARALLELS_USER }} + CRABBOX_PARALLELS_WORK_ROOT: ${{ secrets.CRABBOX_PARALLELS_WORK_ROOT }} + CRABBOX_STATIC_HOST: ${{ secrets.CRABBOX_STATIC_HOST }} + CRABBOX_STATIC_USER: ${{ secrets.CRABBOX_STATIC_USER }} + CRABBOX_STATIC_PORT: ${{ secrets.CRABBOX_STATIC_PORT }} + CRABBOX_STATIC_WORK_ROOT: ${{ secrets.CRABBOX_STATIC_WORK_ROOT }} + CRABBOX_TARGET: ${{ secrets.CRABBOX_TARGET }} + CRABBOX_WINDOWS_MODE: ${{ secrets.CRABBOX_WINDOWS_MODE }} + CRABBOX_SSH_PRIVATE_KEY: ${{ secrets.CRABBOX_SSH_PRIVATE_KEY }} + CRABBOX_EXE_DEV_CONTROL_HOST: ${{ secrets.CRABBOX_EXE_DEV_CONTROL_HOST }} + CRABBOX_EXE_DEV_IMAGE: ${{ secrets.CRABBOX_EXE_DEV_IMAGE }} + CRABBOX_BLACKSMITH_ORG: ${{ secrets.CRABBOX_BLACKSMITH_ORG }} + CRABBOX_BLACKSMITH_WORKFLOW: ${{ secrets.CRABBOX_BLACKSMITH_WORKFLOW }} + CRABBOX_BLACKSMITH_JOB: ${{ secrets.CRABBOX_BLACKSMITH_JOB }} + CRABBOX_BLACKSMITH_REF: ${{ secrets.CRABBOX_BLACKSMITH_REF }} + BLACKSMITH_API_TOKEN: ${{ secrets.BLACKSMITH_API_TOKEN }} + CRABBOX_NAMESPACE_IMAGE: ${{ secrets.CRABBOX_NAMESPACE_IMAGE }} + CRABBOX_NAMESPACE_SIZE: ${{ secrets.CRABBOX_NAMESPACE_SIZE }} + CRABBOX_NAMESPACE_REPOSITORY: ${{ secrets.CRABBOX_NAMESPACE_REPOSITORY }} + CRABBOX_NAMESPACE_SITE: ${{ secrets.CRABBOX_NAMESPACE_SITE }} + CRABBOX_NAMESPACE_DELETE_ON_RELEASE: ${{ secrets.CRABBOX_NAMESPACE_DELETE_ON_RELEASE }} + CRABBOX_SEMAPHORE_HOST: ${{ secrets.CRABBOX_SEMAPHORE_HOST }} + CRABBOX_SEMAPHORE_PROJECT: ${{ secrets.CRABBOX_SEMAPHORE_PROJECT }} + CRABBOX_SEMAPHORE_TOKEN: ${{ secrets.CRABBOX_SEMAPHORE_TOKEN }} + CRABBOX_SPRITES_TOKEN: ${{ secrets.CRABBOX_SPRITES_TOKEN }} + CRABBOX_DAYTONA_SNAPSHOT: ${{ secrets.CRABBOX_DAYTONA_SNAPSHOT }} + DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} + DAYTONA_JWT_TOKEN: ${{ secrets.DAYTONA_JWT_TOKEN }} + DAYTONA_ORGANIZATION_ID: ${{ secrets.DAYTONA_ORGANIZATION_ID }} + ISLO_API_KEY: ${{ secrets.ISLO_API_KEY }} + CRABBOX_E2B_API_KEY: ${{ secrets.CRABBOX_E2B_API_KEY }} + E2B_API_KEY: ${{ secrets.E2B_API_KEY }} + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + CRABBOX_UPSTASH_BOX_API_KEY: ${{ secrets.CRABBOX_UPSTASH_BOX_API_KEY }} + UPSTASH_BOX_API_KEY: ${{ secrets.UPSTASH_BOX_API_KEY }} + CRABBOX_TENSORLAKE_API_KEY: ${{ secrets.CRABBOX_TENSORLAKE_API_KEY }} + TENSORLAKE_API_KEY: ${{ secrets.TENSORLAKE_API_KEY }} + TENSORLAKE_ORGANIZATION_ID: ${{ secrets.TENSORLAKE_ORGANIZATION_ID }} + TENSORLAKE_PROJECT_ID: ${{ secrets.TENSORLAKE_PROJECT_ID }} + CRABBOX_CLOUDFLARE_RUNNER_URL: ${{ secrets.CRABBOX_CLOUDFLARE_RUNNER_URL }} + CRABBOX_CLOUDFLARE_RUNNER_TOKEN: ${{ secrets.CRABBOX_CLOUDFLARE_RUNNER_TOKEN }} + CRABBOX_RAILWAY_API_TOKEN: ${{ secrets.CRABBOX_RAILWAY_API_TOKEN }} + RAILWAY_API_TOKEN: ${{ secrets.RAILWAY_API_TOKEN }} + CRABBOX_RAILWAY_SERVICE_ID: ${{ secrets.CRABBOX_RAILWAY_SERVICE_ID }} + CRABBOX_RAILWAY_PROJECT_ID: ${{ secrets.CRABBOX_RAILWAY_PROJECT_ID }} + CRABBOX_RAILWAY_ENVIRONMENT_ID: ${{ secrets.CRABBOX_RAILWAY_ENVIRONMENT_ID }} + CRABBOX_RUNPOD_API_KEY: ${{ secrets.CRABBOX_RUNPOD_API_KEY }} + RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }} + CRABBOX_RUNPOD_INSTANCE_ID: ${{ secrets.CRABBOX_RUNPOD_INSTANCE_ID }} + CRABBOX_RUNPOD_TEMPLATE_ID: ${{ secrets.CRABBOX_RUNPOD_TEMPLATE_ID }} + CRABBOX_WANDB_API_KEY: ${{ secrets.CRABBOX_WANDB_API_KEY }} + WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }} + WANDB_ENTITY_NAME: ${{ secrets.WANDB_ENTITY_NAME }} + WANDB_PROJECT: ${{ secrets.WANDB_PROJECT }} + + steps: + - name: Check out + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: false + + - name: Set up Python + if: ${{ matrix.provider == 'modal' || matrix.provider == 'tensorlake' }} + uses: actions/setup-python@v6 + with: + python-version: "3.x" + + - name: Install common tools + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y jq ripgrep + + - name: Install provider client + if: ${{ matrix.provider == 'modal' || matrix.provider == 'tensorlake' }} + shell: bash + run: | + set -euo pipefail + case "${{ matrix.provider }}" in + modal) + python -m pip install --user modal + ;; + tensorlake) + python -m pip install --user tensorlake + ;; + esac + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Write secret files + shell: bash + run: | + set -euo pipefail + mkdir -p "$HOME/.ssh" "$RUNNER_TEMP/crabbox" + chmod 700 "$HOME/.ssh" + if [[ -n "${CRABBOX_SSH_PRIVATE_KEY:-}" ]]; then + key="$RUNNER_TEMP/crabbox/id_provider_e2e" + printf '%s\n' "$CRABBOX_SSH_PRIVATE_KEY" > "$key" + chmod 600 "$key" + echo "CRABBOX_SSH_KEY=$key" >> "$GITHUB_ENV" + fi + if [[ -n "${GCP_SERVICE_ACCOUNT_JSON:-}" ]]; then + creds="$RUNNER_TEMP/crabbox/gcp-service-account.json" + printf '%s' "$GCP_SERVICE_ACCOUNT_JSON" > "$creds" + chmod 600 "$creds" + echo "GOOGLE_APPLICATION_CREDENTIALS=$creds" >> "$GITHUB_ENV" + fi + + - name: Build Crabbox + run: go build -trimpath -o "$CRABBOX_BIN" ./cmd/crabbox + + - name: Run provider smoke + shell: bash + run: | + set -euo pipefail + mkdir -p "$CRABBOX_PROVIDER_E2E_LOG_DIR" + log="$CRABBOX_PROVIDER_E2E_LOG_DIR/${{ matrix.provider }}.log" + scripts/live-provider-e2e.sh "${{ matrix.provider }}" 2>&1 | tee "$log" + + - name: Upload provider log + if: always() + uses: actions/upload-artifact@v5 + with: + name: live-provider-e2e-${{ matrix.provider }} + path: .crabbox/live-provider-e2e/${{ matrix.provider }}.log + if-no-files-found: ignore diff --git a/docs/features/README.md b/docs/features/README.md index a1364748..1bde9c28 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -31,6 +31,7 @@ Read when: ## Providers - [Providers](providers.md): provider overview, target matrix, classes, and fallback. +- [Live Provider E2E](live-provider-e2e.md): maintainer-run GitHub Actions smoke matrix for every built-in provider. - [Capacity and fallback](capacity-fallback.md): class chains, market spot/on-demand, region/AZ routing. - [Provider backends](../provider-backends.md): contract reference for backend interfaces and registration. - [Authoring a provider](provider-authoring.md): step-by-step guide to writing a new provider. diff --git a/docs/features/live-provider-e2e.md b/docs/features/live-provider-e2e.md new file mode 100644 index 00000000..2e3ab8ac --- /dev/null +++ b/docs/features/live-provider-e2e.md @@ -0,0 +1,78 @@ +# Live Provider E2E + +`.github/workflows/live-provider-e2e.yml` is a maintainer-run workflow for +proving Crabbox against every built-in provider from GitHub Actions. + +The workflow is intentionally manual (`workflow_dispatch`) because it can create +billable provider resources. It builds the current checkout, fans out one job +per selected provider, and uploads a per-provider log artifact. Missing secrets +are skipped by default so maintainers can add providers incrementally. + +## Running It + +1. Open **Actions > Live Provider E2E > Run workflow**. +2. Use `providers=all` to run the full matrix, or pass a comma-separated subset + such as `aws,hetzner,e2b,cloudflare`. +3. Leave `allow_missing=true` while bringing secrets online. Set it to `false` + when the repository should fail if any selected provider is not configured. +4. Leave `sync_checkout=false` for the fastest smoke. Set it to `true` when a + change needs to exercise Crabbox file sync as part of the provider run. +5. Keep `runner_label=ubuntu-latest` for hosted Linux smoke tests, or set it to + a self-hosted runner label for providers that require local tools or private + network access. + +The smoke command is intentionally small: create or resolve a provider lease or +sandbox, run `echo crabbox--e2e-ok`, print basic runtime context, and +clean up where the provider supports cleanup. + +## Required Secrets + +Add these under **Settings > Secrets and variables > Actions**. Values that are +not actually sensitive can still be stored as secrets to keep setup uniform. + +| Provider | Required GitHub secrets | +| --- | --- | +| Brokered AWS/Azure/GCP/Hetzner | `CRABBOX_COORDINATOR`, `CRABBOX_COORDINATOR_TOKEN` | +| AWS direct | `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, optional `AWS_SESSION_TOKEN`, `AWS_REGION` or `CRABBOX_AWS_REGION` | +| Azure direct | `AZURE_SUBSCRIPTION_ID`, `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, optional `CRABBOX_AZURE_LOCATION`, `CRABBOX_AZURE_RESOURCE_GROUP` | +| GCP direct | `GCP_SERVICE_ACCOUNT_JSON`, `GOOGLE_CLOUD_PROJECT` or `GCP_PROJECT_ID`, optional `CRABBOX_GCP_ZONE` | +| Hetzner direct | `HCLOUD_TOKEN` or `HETZNER_TOKEN` | +| Proxmox | `CRABBOX_PROXMOX_API_URL`, `CRABBOX_PROXMOX_TOKEN_ID`, `CRABBOX_PROXMOX_TOKEN_SECRET`, `CRABBOX_PROXMOX_NODE`, `CRABBOX_PROXMOX_TEMPLATE_ID`, optional storage, bridge, user, and TLS secrets | +| Parallels | `CRABBOX_PARALLELS_TEMPLATE` or `CRABBOX_PARALLELS_SOURCE` plus `CRABBOX_PARALLELS_SOURCE_SNAPSHOT`; remote hosts also need `CRABBOX_PARALLELS_HOST`, `CRABBOX_PARALLELS_HOST_USER`, and `CRABBOX_SSH_PRIVATE_KEY` | +| Local Container | No provider secret; the runner must have Docker available | +| Static SSH | `CRABBOX_STATIC_HOST`, optional `CRABBOX_STATIC_USER`, `CRABBOX_STATIC_PORT`, `CRABBOX_STATIC_WORK_ROOT`, `CRABBOX_TARGET`, `CRABBOX_WINDOWS_MODE`, and `CRABBOX_SSH_PRIVATE_KEY` | +| exe.dev | `CRABBOX_SSH_PRIVATE_KEY` for the SSH identity accepted by exe.dev; optional `CRABBOX_EXE_DEV_CONTROL_HOST`, `CRABBOX_EXE_DEV_IMAGE` | +| Blacksmith Testbox | `CRABBOX_BLACKSMITH_ORG`, `CRABBOX_BLACKSMITH_WORKFLOW`, optional `CRABBOX_BLACKSMITH_JOB`, `CRABBOX_BLACKSMITH_REF`, plus the Blacksmith CLI auth secret expected by the runner | +| Namespace Devbox | Optional `CRABBOX_NAMESPACE_IMAGE`, `CRABBOX_NAMESPACE_SIZE`, `CRABBOX_NAMESPACE_REPOSITORY`, `CRABBOX_NAMESPACE_SITE`; the runner must have an authenticated `devbox` CLI | +| Semaphore | `CRABBOX_SEMAPHORE_HOST`, `CRABBOX_SEMAPHORE_PROJECT`, `CRABBOX_SEMAPHORE_TOKEN` | +| Sprites | `CRABBOX_SPRITES_TOKEN`; the runner must have the `sprite` CLI | +| Daytona | `DAYTONA_API_KEY` or `DAYTONA_JWT_TOKEN`, `CRABBOX_DAYTONA_SNAPSHOT`; JWT auth also needs `DAYTONA_ORGANIZATION_ID` | +| Islo | `ISLO_API_KEY` | +| E2B | `CRABBOX_E2B_API_KEY` or `E2B_API_KEY` | +| Modal | `MODAL_TOKEN_ID`, `MODAL_TOKEN_SECRET` | +| Upstash Box | `CRABBOX_UPSTASH_BOX_API_KEY` or `UPSTASH_BOX_API_KEY` | +| Tensorlake | `CRABBOX_TENSORLAKE_API_KEY` or `TENSORLAKE_API_KEY`, optional `TENSORLAKE_ORGANIZATION_ID`, `TENSORLAKE_PROJECT_ID` | +| Cloudflare | `CRABBOX_CLOUDFLARE_RUNNER_URL`, `CRABBOX_CLOUDFLARE_RUNNER_TOKEN` | +| Railway | `CRABBOX_RAILWAY_API_TOKEN` or `RAILWAY_API_TOKEN`, `CRABBOX_RAILWAY_SERVICE_ID`, `CRABBOX_RAILWAY_PROJECT_ID`, `CRABBOX_RAILWAY_ENVIRONMENT_ID` | +| RunPod | `CRABBOX_RUNPOD_API_KEY` or `RUNPOD_API_KEY`; the RunPod account must already trust the SSH public key matching `CRABBOX_SSH_PRIVATE_KEY` | +| W&B Sandboxes | `CRABBOX_WANDB_API_KEY` or `WANDB_API_KEY`, `WANDB_ENTITY_NAME`, optional `WANDB_PROJECT` | + +## Notes + +- The workflow installs `modal` and `tensorlake` Python packages for those + providers. Other providers that rely on a local CLI need that CLI available on + the selected runner before the job starts. +- `GCP_SERVICE_ACCOUNT_JSON` is written to a temporary file and exposed through + `GOOGLE_APPLICATION_CREDENTIALS`. +- `CRABBOX_SSH_PRIVATE_KEY` is written to a temporary `0600` file and exposed + as `CRABBOX_SSH_KEY`. +- Railway is a redeploy-and-stream provider. Its smoke redeploys the configured + existing service rather than executing an arbitrary shell command inside it. +- The workflow never runs on pull requests automatically, so forked PRs cannot + access provider secrets or create provider resources. + +Related docs: + +- [Providers](providers.md) +- [Provider reference](../providers/README.md) +- [Security](../security.md) diff --git a/scripts/live-provider-e2e.sh b/scripts/live-provider-e2e.sh new file mode 100755 index 00000000..512f60e7 --- /dev/null +++ b/scripts/live-provider-e2e.sh @@ -0,0 +1,371 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cb="${CRABBOX_BIN:-$root/bin/crabbox}" +repo="${CRABBOX_LIVE_REPO:-$root}" +skip_missing="${CRABBOX_LIVE_SKIP_MISSING:-0}" +sync_checkout="${CRABBOX_PROVIDER_E2E_SYNC:-0}" +idle_timeout="${CRABBOX_PROVIDER_E2E_IDLE_TIMEOUT:-5m}" +ttl="${CRABBOX_PROVIDER_E2E_TTL:-15m}" +wait_timeout="${CRABBOX_PROVIDER_E2E_WAIT_TIMEOUT:-180s}" + +usage() { + cat >&2 <<'USAGE' +Usage: CRABBOX_LIVE=1 scripts/live-provider-e2e.sh + +Runs one live provider smoke. Set CRABBOX_LIVE_SKIP_MISSING=1 to skip a +provider when its required GitHub Actions secrets or runner tools are absent. +USAGE +} + +provider="${1:-}" +if [[ -z "$provider" || "$provider" == "-h" || "$provider" == "--help" ]]; then + usage + exit 2 +fi + +normalize_provider() { + case "$1" in + blacksmith) printf 'blacksmith-testbox' ;; + cf) printf 'cloudflare' ;; + container | docker) printf 'local-container' ;; + namespace) printf 'namespace-devbox' ;; + static | static-ssh) printf 'ssh' ;; + *) printf '%s' "$1" ;; + esac +} + +provider="$(normalize_provider "$provider")" + +if [[ "${CRABBOX_LIVE:-}" != "1" ]]; then + echo "set CRABBOX_LIVE=1 to run live provider E2E tests" >&2 + exit 2 +fi +if [[ ! -x "$cb" ]]; then + echo "missing crabbox binary: $cb" >&2 + echo "build first: go build -trimpath -o bin/crabbox ./cmd/crabbox" >&2 + exit 2 +fi + +run_in_repo() { + (cd "$repo" && "$@") +} + +skip_or_fail() { + local reason="$1" + if [[ "$skip_missing" == "1" ]]; then + echo "skip provider=$provider reason=$reason" + exit 0 + fi + echo "missing provider=$provider requirement: $reason" >&2 + exit 2 +} + +need_tool() { + command -v "$1" >/dev/null 2>&1 || skip_or_fail "tool $1 on PATH" +} + +need_env() { + local name="$1" + [[ -n "${!name:-}" ]] || skip_or_fail "env $name" +} + +need_any_env() { + local label="$1" + shift + local name + for name in "$@"; do + if [[ -n "${!name:-}" ]]; then + return 0 + fi + done + skip_or_fail "$label (${*})" +} + +need_env_pair() { + local left="$1" + local right="$2" + [[ -n "${!left:-}" && -n "${!right:-}" ]] || skip_or_fail "env $left and $right" +} + +coordinator_ready() { + [[ -n "${CRABBOX_COORDINATOR:-}" && -n "${CRABBOX_COORDINATOR_TOKEN:-}" ]] +} + +managed_provider() { + case "$provider" in + aws | azure | gcp | hetzner) return 0 ;; + *) return 1 ;; + esac +} + +direct_or_coordinator_env() { + if coordinator_ready; then + return 0 + fi + case "$provider" in + aws) + [[ -n "${AWS_ACCESS_KEY_ID:-}" || -n "${AWS_PROFILE:-}" ]] + ;; + azure) + [[ -n "${AZURE_SUBSCRIPTION_ID:-}" && -n "${AZURE_TENANT_ID:-}" && -n "${AZURE_CLIENT_ID:-}" && -n "${AZURE_CLIENT_SECRET:-}" ]] + ;; + gcp) + [[ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" || -n "${GCP_PRIVATE_KEY:-}" ]] + ;; + hetzner) + [[ -n "${HCLOUD_TOKEN:-}" || -n "${HETZNER_TOKEN:-}" ]] + ;; + *) + return 1 + ;; + esac +} + +coordinator_login() { + if ! coordinator_ready; then + return 0 + fi + printf '%s' "$CRABBOX_COORDINATOR_TOKEN" | "$cb" login \ + --url "$CRABBOX_COORDINATOR" \ + --provider "$provider" \ + --token-stdin \ + --json >/dev/null +} + +extract_lease() { + sed -n ' + s/^leased \([^ ]*\).*/\1/p + s/.* lease=\([^ ]*\).*/\1/p + ' | head -1 +} + +extract_slug() { + sed -n 's/.* slug=\([^ ]*\).*/\1/p' | head -1 +} + +provider_token() { + printf '%s' "$provider" | tr -c '[:alnum:]' '-' +} + +smoke_command() { + local token + token="$(provider_token)" + if [[ "${CRABBOX_TARGET:-}" == "windows" && "${CRABBOX_WINDOWS_MODE:-}" != "wsl2" ]]; then + printf "Write-Output 'crabbox-%s-e2e-ok'; Get-Location; [System.Environment]::OSVersion.Platform" "$token" + return 0 + fi + printf 'set -eu; echo crabbox-%s-e2e-ok; pwd; uname -s || true' "$token" +} + +ssh_lease_smoke() { + local out lease slug id + out="$(run_in_repo "$cb" warmup --provider "$provider" --ttl "$ttl" --idle-timeout "$idle_timeout" "$@" 2>&1)" + printf '%s\n' "$out" + lease="$(printf '%s\n' "$out" | extract_lease)" + slug="$(printf '%s\n' "$out" | extract_slug)" + id="${slug:-$lease}" + [[ -n "$id" ]] || { + echo "could not parse lease id or slug from warmup output" >&2 + return 3 + } + + cleanup() { + run_in_repo "$cb" stop --provider "$provider" "$id" || true + } + trap cleanup RETURN + + run_in_repo "$cb" status --provider "$provider" --id "$id" --wait --wait-timeout "$wait_timeout" + run_in_repo "$cb" inspect --provider "$provider" --id "$id" --json || true + run_in_repo "$cb" ssh --provider "$provider" --id "$id" + if [[ "$sync_checkout" == "1" ]]; then + run_in_repo "$cb" run --provider "$provider" --id "$id" --timing-json --shell -- "$(smoke_command)" + else + run_in_repo "$cb" run --provider "$provider" --id "$id" --no-sync --timing-json --shell -- "$(smoke_command)" + fi + run_in_repo "$cb" list --provider "$provider" --json || true + run_in_repo "$cb" stop --provider "$provider" "$id" + trap - RETURN +} + +delegated_run_smoke() { + if [[ "$sync_checkout" == "1" ]]; then + run_in_repo "$cb" run --provider "$provider" --timing-json --shell -- "$(smoke_command)" + else + run_in_repo "$cb" run --provider "$provider" --no-sync --timing-json --shell -- "$(smoke_command)" + fi + run_in_repo "$cb" list --provider "$provider" --json || true +} + +blacksmith_smoke() { + need_tool blacksmith + need_env CRABBOX_BLACKSMITH_ORG + need_env CRABBOX_BLACKSMITH_WORKFLOW + run_in_repo "$cb" run \ + --provider blacksmith-testbox \ + --blacksmith-org "$CRABBOX_BLACKSMITH_ORG" \ + --blacksmith-workflow "$CRABBOX_BLACKSMITH_WORKFLOW" \ + --blacksmith-job "${CRABBOX_BLACKSMITH_JOB:-check}" \ + --blacksmith-ref "${CRABBOX_BLACKSMITH_REF:-main}" \ + --idle-timeout "$idle_timeout" \ + --timing-json \ + --shell -- "$(smoke_command)" + run_in_repo "$cb" list --provider blacksmith-testbox --json || true +} + +daytona_smoke() { + need_any_env "Daytona API auth" DAYTONA_API_KEY DAYTONA_JWT_TOKEN + need_any_env "Daytona snapshot" CRABBOX_DAYTONA_SNAPSHOT DAYTONA_SNAPSHOT + delegated_run_smoke +} + +railway_smoke() { + need_any_env "Railway API token" CRABBOX_RAILWAY_API_TOKEN RAILWAY_API_TOKEN + need_env CRABBOX_RAILWAY_SERVICE_ID + need_env CRABBOX_RAILWAY_PROJECT_ID + need_env CRABBOX_RAILWAY_ENVIRONMENT_ID + run_in_repo "$cb" run \ + --provider railway \ + --no-sync \ + --id "$CRABBOX_RAILWAY_SERVICE_ID" \ + --railway-project "$CRABBOX_RAILWAY_PROJECT_ID" \ + --railway-environment "$CRABBOX_RAILWAY_ENVIRONMENT_ID" \ + --timing-json \ + -- "$(smoke_command)" + run_in_repo "$cb" status \ + --provider railway \ + --id "$CRABBOX_RAILWAY_SERVICE_ID" \ + --railway-project "$CRABBOX_RAILWAY_PROJECT_ID" \ + --railway-environment "$CRABBOX_RAILWAY_ENVIRONMENT_ID" || true +} + +wandb_smoke() { + need_any_env "W&B API key" CRABBOX_WANDB_API_KEY WANDB_API_KEY + need_env WANDB_ENTITY_NAME + run_in_repo "$cb" doctor --provider wandb + run_in_repo "$cb" run \ + --provider wandb \ + --no-sync \ + --wandb-max-lifetime 60 \ + --timing-json \ + -- "$(smoke_command)" + run_in_repo "$cb" list --provider wandb --json || true +} + +preflight_provider() { + case "$provider" in + aws | azure | gcp | hetzner) + direct_or_coordinator_env || skip_or_fail "CRABBOX_COORDINATOR and CRABBOX_COORDINATOR_TOKEN, or direct $provider credentials" + ;; + proxmox) + need_env CRABBOX_PROXMOX_API_URL + need_env CRABBOX_PROXMOX_TOKEN_ID + need_env CRABBOX_PROXMOX_TOKEN_SECRET + need_env CRABBOX_PROXMOX_NODE + need_env CRABBOX_PROXMOX_TEMPLATE_ID + ;; + parallels) + if [[ -z "${CRABBOX_PARALLELS_HOST:-}" ]]; then + need_tool prlctl + fi + if [[ -z "${CRABBOX_PARALLELS_TEMPLATE:-}" ]]; then + need_any_env "Parallels source" CRABBOX_PARALLELS_SOURCE CRABBOX_PARALLELS_SOURCE_ID + need_any_env "Parallels source snapshot" CRABBOX_PARALLELS_SOURCE_SNAPSHOT CRABBOX_PARALLELS_SOURCE_SNAPSHOT_ID + fi + ;; + local-container) + need_tool "${CRABBOX_LOCAL_CONTAINER_RUNTIME:-docker}" + ;; + ssh) + need_env CRABBOX_STATIC_HOST + ;; + exe-dev) + need_tool ssh + ;; + blacksmith-testbox) + ;; + namespace-devbox) + need_tool devbox + ;; + semaphore) + need_env CRABBOX_SEMAPHORE_HOST + need_env CRABBOX_SEMAPHORE_PROJECT + need_any_env "Semaphore API token" CRABBOX_SEMAPHORE_TOKEN SEMAPHORE_API_TOKEN + ;; + sprites) + need_tool sprite + need_any_env "Sprites token" CRABBOX_SPRITES_TOKEN SPRITES_TOKEN SPRITE_TOKEN SETUP_SPRITE_TOKEN + ;; + daytona) + ;; + islo) + need_env ISLO_API_KEY + ;; + e2b) + need_any_env "E2B API key" CRABBOX_E2B_API_KEY E2B_API_KEY + ;; + modal) + need_env_pair MODAL_TOKEN_ID MODAL_TOKEN_SECRET + ;; + upstash-box) + need_any_env "Upstash Box API key" CRABBOX_UPSTASH_BOX_API_KEY UPSTASH_BOX_API_KEY + ;; + tensorlake) + need_tool tensorlake + need_any_env "Tensorlake API key" CRABBOX_TENSORLAKE_API_KEY TENSORLAKE_API_KEY + ;; + cloudflare) + need_env CRABBOX_CLOUDFLARE_RUNNER_URL + need_env CRABBOX_CLOUDFLARE_RUNNER_TOKEN + ;; + railway) + ;; + runpod) + need_any_env "RunPod API key" CRABBOX_RUNPOD_API_KEY RUNPOD_API_KEY + ;; + wandb) + ;; + *) + skip_or_fail "unknown provider" + ;; + esac +} + +preflight_provider +if managed_provider; then + coordinator_login +fi + +case "$provider" in + aws) + ssh_lease_smoke --type "${CRABBOX_LIVE_AWS_TYPE:-t3.small}" + ;; + azure) + ssh_lease_smoke --class "${CRABBOX_LIVE_AZURE_CLASS:-standard}" + ;; + gcp) + ssh_lease_smoke --class "${CRABBOX_LIVE_GCP_CLASS:-standard}" + ;; + hetzner) + ssh_lease_smoke --class "${CRABBOX_LIVE_HETZNER_CLASS:-standard}" + ;; + proxmox | parallels | local-container | ssh | exe-dev | namespace-devbox | semaphore | sprites | runpod) + ssh_lease_smoke + ;; + blacksmith-testbox) + blacksmith_smoke + ;; + daytona) + daytona_smoke + ;; + islo | e2b | modal | upstash-box | tensorlake | cloudflare) + delegated_run_smoke + ;; + railway) + railway_smoke + ;; + wandb) + wandb_smoke + ;; +esac