Live Batch Tests (Daily) #112
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Live Batch Tests (Daily) | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} | |
| AWS_REGION: "us-east-1" | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} | |
| AZURE_OPENAI_EASTUS2_API_KEY: ${{ secrets.AZURE_OPENAI_EASTUS2_API_KEY }} | |
| AZURE_OPENAI_DEPLOYMENT_ID: ${{secrets.AZURE_OPENAI_DEPLOYMENT_ID }} | |
| DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} | |
| FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} | |
| FIREWORKS_ACCOUNT_ID: ${{ secrets.FIREWORKS_ACCOUNT_ID }} | |
| FORCE_COLOR: 1 | |
| GCP_VERTEX_CREDENTIALS_PATH: ${{ github.workspace }}/gcp_jwt_key.json | |
| GOOGLE_APPLICATION_CREDENTIALS: ${{ github.workspace }}/gcp_jwt_key.json | |
| GOOGLE_AI_STUDIO_API_KEY: ${{ secrets.GOOGLE_AI_STUDIO_API_KEY }} | |
| GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} | |
| HYPERBOLIC_API_KEY: ${{secrets.HYPERBOLIC_API_KEY}} | |
| MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| SGLANG_API_KEY: ${{ secrets.SGLANG_API_KEY }} | |
| TGI_API_KEY: ${{ secrets.TGI_API_KEY }} | |
| TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} | |
| VLLM_API_KEY: ${{ secrets.VLLM_API_KEY }} | |
| VLLM_API_BASE: ${{ secrets.VLLM_API_BASE }} | |
| VLLM_MODEL_NAME: "microsoft/Phi-3.5-mini-instruct" | |
| XAI_API_KEY: ${{ secrets.XAI_API_KEY }} | |
| on: | |
| workflow_dispatch: | |
| workflow_call: | |
| schedule: | |
| - cron: "0 0 * * *" # Runs daily at midnight UTC | |
| permissions: | |
| contents: read | |
| actions: read | |
| jobs: | |
| batch-tests: | |
| runs-on: ubuntu-latest | |
| if: github.repository == 'RouterBase/tensorzero' | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 | |
| - name: Cleanup disk space | |
| uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be | |
| - name: Install Rust toolchain | |
| run: | | |
| for attempt in 1 2 3; do | |
| if rustup toolchain install stable && rustup default stable; then | |
| break | |
| fi | |
| if [ $attempt -eq 3 ]; then | |
| echo "Failed to install Rust toolchain after 3 attempts" | |
| exit 1 | |
| fi | |
| sleep $((10 * attempt)) | |
| done | |
| shell: bash | |
| - name: Login to DockerHub | |
| uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Wake up ClickHouse cloud (for batch tests) | |
| run: | | |
| curl "${{ secrets.CLICKHOUSE_CLOUD_URL }}" --retry 4 --retry-all-errors --data-binary 'SHOW DATABASES' | |
| - name: Install cargo-nextest | |
| uses: taiki-e/install-action@60581cd7025e0e855cebd745379013e286d9c787 | |
| with: | |
| tool: cargo-nextest | |
| - name: Write GCP JWT key to file | |
| env: | |
| GCP_JWT_KEY: ${{ secrets.GCP_JWT_KEY }} | |
| run: echo "$GCP_JWT_KEY" > $GITHUB_WORKSPACE/gcp_jwt_key.json | |
| - name: Set up TENSORZERO_CLICKHOUSE_URL for batch tests | |
| run: | | |
| echo "TENSORZERO_CLICKHOUSE_URL=${{ secrets.CLICKHOUSE_CLOUD_URL }}" >> $GITHUB_ENV | |
| - name: Launch the gateway for batch tests | |
| run: | | |
| curl "${{ secrets.CLICKHOUSE_CLOUD_URL }}" --retry 4 --retry-all-errors --data-binary 'SHOW DATABASES' | |
| cargo run-e2e > batch_logs.txt 2>&1 & | |
| echo "GATEWAY_PID=$!" >> $GITHUB_ENV | |
| while ! curl -s -f http://localhost:3000/health >/dev/null 2>&1; do | |
| echo "Waiting for gateway to be healthy..." | |
| sleep 1 | |
| done | |
| - name: Run batch tests | |
| run: | | |
| cargo test-batch --no-fail-fast | |
| - name: Print batch logs | |
| if: always() | |
| run: cat batch_logs.txt |