e2e-nvidia-l40s-x4.yml

# SPDX-License-Identifier: Apache-2.0

name: E2E (NVIDIA L40S x4)

on:
  schedule:
    - cron: '0 16 * * *' # Runs at 4PM UTC every day
  workflow_dispatch:
    inputs:
      pr_or_branch:
        description: 'pull request number or branch name'
        required: true
        default: 'main'

env:
  TMPDIR: /home/tmp

jobs:
  start-large-ec2-runner:
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.launch-ec2-instance-with-fallback.outputs.label }}
      ec2-instance-id: ${{ steps.launch-ec2-instance-with-fallback.outputs.ec2-instance-id }}
      ec2-instance-region: ${{ steps.launch-ec2-instance-with-fallback.outputs.ec2-instance-region }}
    steps:
      - name: Checkout "launch-ec2-runner-with-fallback" in-house CI action
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: instructlab/ci-actions
          # clone the "ci-actions" repo to a local directory called "ci-actions", instead of
          # overwriting the current WORKDIR contents
          path: ci-actions
          ref: release-v0.2
          sparse-checkout: |
            actions/launch-ec2-runner-with-fallback

      - name: Launch EC2 Runner with Fallback
        id: launch-ec2-instance-with-fallback
        uses: ./ci-actions/actions/launch-ec2-runner-with-fallback
        env:
          TMPDIR: "/tmp"
        with:
          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          github_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          regions_config: >
            [
              {
                "region": "us-east-2",
                "subnets": {
                  "us-east-2a": "${{ vars.SUBNET_US_EAST_2A }}",
                  "us-east-2b": "${{ vars.SUBNET_US_EAST_2B }}",
                  "us-east-2c": "${{ vars.SUBNET_US_EAST_2C }}"
                },
                "ec2-ami": "${{ vars.AWS_EC2_AMI_US_EAST_2 }}",
                "security-group-id": "${{ vars.SECURITY_GROUP_ID_US_EAST_2 }}"
              },
              {
                "region": "us-east-1",
                "subnets": {
                  "us-east-1a": "${{ vars.SUBNET_US_EAST_1A }}",
                  "us-east-1b": "${{ vars.SUBNET_US_EAST_1B }}",
                  "us-east-1c": "${{ vars.SUBNET_US_EAST_1C }}",
                  "us-east-1d": "${{ vars.SUBNET_US_EAST_1D }}",
                  "us-east-1e": "${{ vars.SUBNET_US_EAST_1E }}",
                  "us-east-1f": "${{ vars.SUBNET_US_EAST_1F }}"
                },
                "ec2-ami": "${{ vars.AWS_EC2_AMI_US_EAST_1 }}",
                "security-group-id": "${{ vars.SECURITY_GROUP_ID_US_EAST_1 }}"
              }
            ]
          try_spot_instance_first: false
          ec2_instance_type: g6e.12xlarge
          aws_resource_tags: >
            [
              {"Key": "Name", "Value": "instructlab-ci-github-large-runner"},
              {"Key": "GitHubRepository", "Value": "${{ github.repository }}"},
              {"Key": "GitHubRef", "Value": "${{ github.ref }}"},
              {"Key": "GitHubPR", "Value": "${{ github.event.number }}"}
            ]

  e2e-large-test:
    needs:
      - start-large-ec2-runner
    runs-on: ${{ needs.start-large-ec2-runner.outputs.label }}

    permissions:
      pull-requests: write

    steps:
      - name: Install Packages
        run: |
          cat /etc/os-release
          mkdir -p "${TMPDIR}"
          sudo dnf install -y gcc gcc-c++ make git python3.11 python3.11-devel

      - name: Checkout instructlab/instructlab
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: "instructlab/instructlab"
          path: "instructlab"
          # https://github.com/actions/checkout/issues/249
          fetch-depth: 0
  
      - name: Checkout instructlab/eval
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: "instructlab/eval"
          path: "eval"
          # https://github.com/actions/checkout/issues/249
          fetch-depth: 0

      - name: Determine if pr_or_branch is a PR number
        id: check_pr
        run: |
          PR_OR_BRANCH=${{ github.event.inputs.pr_or_branch || 'main' }} # Default to 'main' if not set
          if [[ "$PR_OR_BRANCH" =~ ^[0-9]+$ ]]; then
            echo "is_pr=true" >> "$GITHUB_OUTPUT"
          else
            echo "is_pr=false" >> "$GITHUB_OUTPUT"
          fi
          echo "pr_or_branch=$PR_OR_BRANCH" >> "$GITHUB_OUTPUT"

      - name: Check if gh cli is installed
        id: gh_cli
        run: |
          if command -v gh &> /dev/null ; then
            echo "gh_cli_installed=true" >> "$GITHUB_OUTPUT"
          else
            echo "gh_cli_installed=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Install gh CLI
        if: steps.gh_cli.outputs.gh_cli_installed == 'false'
        run: |
          sudo dnf install 'dnf-command(config-manager)' -y
          sudo dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
          sudo dnf install gh --repo gh-cli -y

      - name: test gh CLI
        run: |
          gh --version

      - name: set default repo
        working-directory: ./eval
        run: |
          gh repo set-default ${{ github.server_url }}/${{ github.repository }}
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Add comment to PR
        if: steps.check_pr.outputs.is_pr == 'true'
        working-directory: ./eval
        run: |
          gh pr comment "${{ steps.check_pr.outputs.pr_or_branch }}" -b "${{ github.workflow }} workflow launched on this PR: [View run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})"
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Fetch and checkout PR
        if: steps.check_pr.outputs.is_pr == 'true'
        working-directory: ./eval
        run: |
          gh pr checkout ${{ steps.check_pr.outputs.pr_or_branch }}
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Checkout branch
        if: steps.check_pr.outputs.is_pr == 'false'
        working-directory: ./eval
        run: |
          git checkout ${{ steps.check_pr.outputs.pr_or_branch }}

      - name: Install ilab
        working-directory: ./instructlab
        run: |
          PYTHON=python3.11 ./scripts/install-ilab-with-cuda.sh

      - name: Update instructlab-eval library
        working-directory: ./eval
        run: |
          . ../instructlab/venv/bin/activate
          # Patch out our own pin from the ilab repo constraints file
          ilab_constraints=../instructlab/constraints-dev.txt
          sed -i '/instructlab-eval==/d' $ilab_constraints

          # Since we reuse the virtual environment prepared using ilab
          # constraints, we should stick to the same constraints when
          # installing latest eval.
          #
          # FIX: this is not ideal; a proper fix would require decoupling the
          # two repos in CI: either by removing the job completely and relying
          # on "sdk" (no ilab) test runs; or by preparing a separate
          # constraints file that would consider both the requirements files
          # for the eval library AND for the ilab - so that they are
          # consistent.
          pip_install="pip install -c $ilab_constraints"
          $pip_install .
          $pip_install .[cuda]

      - name: Check disk before tests
        run: |
          df -h

      - name: Run e2e test
        working-directory: ./instructlab
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          . venv/bin/activate
          ./scripts/e2e-ci.sh -l

      - name: Check disk after tests
        run: |
          df -h

      - name: Add comment to PR if the workflow failed
        if: failure() && steps.check_pr.outputs.is_pr == 'true'
        working-directory: ./eval
        run: |
          gh pr comment "${{ steps.check_pr.outputs.pr_or_branch }}" -b "e2e workflow failed on this PR: [View run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}), please investigate."
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Add comment to PR if the workflow succeeded
        if: success() && steps.check_pr.outputs.is_pr == 'true'
        working-directory: ./eval
        run: |
          gh pr comment "${{ steps.check_pr.outputs.pr_or_branch }}" -b "e2e workflow succeeded on this PR: [View run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}), congrats!"
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Send Discord notification for failure
        if: failure() && steps.check_pr.outputs.is_pr == 'false'
        uses: sarisia/actions-status-discord@5ddd3b114a98457dd80a39b2f00b6a998cd69008 # v1.15.3
        with:
          webhook: ${{ secrets.SON_OF_JEEVES_DISCORD_WEBHOOK }}
          status: ${{ job.status }}
          title: "e2e-nvidia-l40s-x4"
          description: |
            Job in **${{ github.repository }}** running on branch `${{ steps.check_pr.outputs.pr_or_branch }}` completed **with failures** ❌
            Click [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) to view details.
          color: 0xCB2431 # Red color for failure

      - name: Send Discord notification for success
        if: success() && steps.check_pr.outputs.is_pr == 'false'
        uses: sarisia/actions-status-discord@5ddd3b114a98457dd80a39b2f00b6a998cd69008 # v1.15.3
        with:
          webhook: ${{ secrets.SON_OF_JEEVES_DISCORD_WEBHOOK }}
          status: ${{ job.status }}
          title: "e2e-nvidia-l40s-x4"
          description: |
            Job in **${{ github.repository }}** running on branch `${{ steps.check_pr.outputs.pr_or_branch }}` completed **successfully** ✅
            Click [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) to view details.
          color: 0x28A745 # Green color for success

  stop-large-ec2-runner:
    needs:
      - start-large-ec2-runner
      - e2e-large-test
    runs-on: ubuntu-latest
    if: ${{ always() }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ needs.start-large-ec2-runner.outputs.ec2-instance-region }}

      - name: Stop EC2 runner
        uses: machulav/ec2-github-runner@a6dbcefcf8a31a861f5e078bb153ed332130c512 # v2.4.3
        with:
          mode: stop
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          label: ${{ needs.start-large-ec2-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-large-ec2-runner.outputs.ec2-instance-id }}