evaluators/.github/workflows/test-sdk-python.yml at main · learning-commons-org/evaluators · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
name: 🐍 Test Python SDK

on:
  push:
    branches:
      - main
    paths:
      - "sdks/python/**"
      # The package bundles evals/ contracts; a registry change must re-run check-generated.
      - "evals/**"
      - ".github/workflows/test-sdk-python.yml"
      # The coverage job uploads under the flags and thresholds this file defines.
      - "codecov.yml"
  pull_request:
    paths:
      - "sdks/python/**"
      - "evals/**"
      - ".github/workflows/test-sdk-python.yml"
      # The coverage job uploads under the flags and thresholds this file defines.
      - "codecov.yml"

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  verify:
    name: Verify (Python ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: sdks/python
    strategy:
      matrix:
        python-version: ["3.11", "3.12", "3.13", "3.14"]
    steps:
      - name: Checkout
        uses: actions/checkout@v7

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v7
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install package and dev tools
        run: |
          python -m pip install --upgrade pip
          pip install -e ".[dev]"

      - name: make verify
        run: make verify

  coverage:
    name: Coverage
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: sdks/python
    steps:
      - name: Checkout
        uses: actions/checkout@v7

      - name: Set up Python
        uses: actions/setup-python@v7
        with:
          python-version: "3.13"

      - name: Install package and dev tools
        run: |
          python -m pip install --upgrade pip
          pip install -e ".[dev]"

      # The gate. `make coverage` exits non-zero below COV_MIN, so this step — not the
      # Codecov status — is what holds the floor when Codecov is unreachable.
      - name: make coverage
        run: make coverage

      # Reporting only: a Codecov outage must not turn a passing suite red. Runs even when
      # the gate above failed, so a coverage drop still gets its per-line annotation.
      - name: Upload coverage to Codecov
        if: always()
        uses: codecov/codecov-action@v7
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./sdks/python/coverage.xml
          flags: python-sdk
          name: python-sdk-coverage
        continue-on-error: true