From c5654901a2797ed94d032fb746032eed5efc5f5e Mon Sep 17 00:00:00 2001 From: kvmto Date: Thu, 2 Jul 2026 15:26:07 +0000 Subject: [PATCH 1/5] ci: enable NUMA affinity syscalls and add gated multi-GPU QEC test lane Signed-off-by: kvmto --- .github/workflows/lib_qec.yaml | 88 +++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lib_qec.yaml b/.github/workflows/lib_qec.yaml index dea6cd8ba..65f034546 100644 --- a/.github/workflows/lib_qec.yaml +++ b/.github/workflows/lib_qec.yaml @@ -25,7 +25,12 @@ jobs: platform: ['amd64', 'arm64'] cuda_version: ['12.6', '13.0'] runs-on: ${{ startsWith(github.repository, 'NVIDIA/cudaqx') && format('linux-{0}-cpu8', matrix.platform) || 'ubuntu-latest' }} - container: ghcr.io/nvidia/cuda-quantum-devcontainer:${{ matrix.platform }}-cu${{ matrix.cuda_version }}-gcc12-main + container: + image: ghcr.io/nvidia/cuda-quantum-devcontainer:${{ matrix.platform }}-cu${{ matrix.cuda_version }}-gcc12-main + # SYS_NICE: the default seccomp profile only admits the NUMA memory-policy + # syscalls (set_mempolicy/get_mempolicy/mbind) with this capability, which + # the hardware-affinity tests exercise. + options: --cap-add=SYS_NICE permissions: actions: write contents: read @@ -234,3 +239,84 @@ jobs: run: | PYTHONPATH="/cudaq-install:$GITHUB_WORKSPACE/build_qec/python" \ pytest -v libs/qec/python/tests/ + + # ========================================================================== + # Multi-GPU tests - decoder placement across >= 2 GPUs (uses build artifacts + # from build-and-test). + # + # NVIDIA GHA runner labels follow linux--gpu--latest-; + # this job needs the 2-GPU variant. It runs on merges to the default branch + # and on manual dispatch only, to keep multi-GPU capacity out of the per-PR + # critical path; timeout-minutes turns a missing runner label into a visible + # failure instead of an indefinite queue. + # ========================================================================== + multi-gpu-test: + name: Multi-GPU tests + if: github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' + needs: build-and-test + strategy: + fail-fast: false + matrix: + cuda_version: ['12.6', '13.0'] + runs-on: linux-amd64-gpu-a100-latest-2 + container: + image: ghcr.io/nvidia/cuda-quantum-devcontainer:amd64-cu${{ matrix.cuda_version }}-gcc12-main + # SYS_NICE: see build-and-test. + options: --cap-add=SYS_NICE + env: + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + timeout-minutes: 45 + permissions: + actions: read + contents: read + steps: + - name: Install git for LFS + shell: bash + run: | + apt update + apt install -y --no-install-recommends git git-lfs + + - name: Checkout repository + uses: actions/checkout@v4 + with: + set-safe-directory: true + lfs: true # TRT tests read the ONNX asset from the source tree + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: qec-build-amd64-cu${{ matrix.cuda_version }} + path: /tmp + + - name: Extract build artifacts + run: | + tar xzf /tmp/qec-build-artifacts.tar.gz -C / + + - name: Configure + id: config + run: | + cuda_major=$(echo ${{ matrix.cuda_version }} | cut -d . -f1) + echo "cuda_major=$cuda_major" >> $GITHUB_OUTPUT + + - name: Verify at least 2 GPUs + run: | + nvidia-smi + count=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) + echo "GPU count: ${count}" + [ "${count}" -ge 2 ] || { echo "ERROR: expected >=2 GPUs, found ${count}"; exit 1; } + + - name: Install cuStabilizer runtime + run: | + pip install --upgrade "cuquantum-python-cu${{ steps.config.outputs.cuda_major }}>=26.3.0" + + - name: Install TensorRT (C++) + run: | + apt update + apt-cache search tensorrt | awk '{print "Package: "$1"\nPin: version *+cuda${{ matrix.cuda_version }}\nPin-Priority: 1001\n"}' | tee /etc/apt/preferences.d/tensorrt-cuda${{ matrix.cuda_version }}.pref > /dev/null + apt install -y tensorrt-dev + + - name: Run multi-GPU C++ tests + run: | + cd $GITHUB_WORKSPACE/build_qec + ctest --output-on-failure --no-tests=error \ + -R "DecoderPool|HardwarePinningGpu|PoolRunsTwoTrt|CudaDeviceId" From 88c8a0a53b1a237ed62040fe27527ae25feb4b59 Mon Sep 17 00:00:00 2001 From: Angela Burton Date: Thu, 2 Jul 2026 09:59:37 -0700 Subject: [PATCH 2/5] update 2-GPU runner name Signed-off-by: Angela Burton --- .github/workflows/lib_qec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lib_qec.yaml b/.github/workflows/lib_qec.yaml index 65f034546..302c5a9c2 100644 --- a/.github/workflows/lib_qec.yaml +++ b/.github/workflows/lib_qec.yaml @@ -258,7 +258,7 @@ jobs: fail-fast: false matrix: cuda_version: ['12.6', '13.0'] - runs-on: linux-amd64-gpu-a100-latest-2 + runs-on: nv-gpu-amd64-rtxpro6000-2gpu container: image: ghcr.io/nvidia/cuda-quantum-devcontainer:amd64-cu${{ matrix.cuda_version }}-gcc12-main # SYS_NICE: see build-and-test. From 731d911e0fca637b4e688873e9c6f5c1ea45aff4 Mon Sep 17 00:00:00 2001 From: Angela Burton Date: Thu, 2 Jul 2026 10:50:08 -0700 Subject: [PATCH 3/5] remove multi-gpu test conditional Signed-off-by: Angela Burton --- .github/workflows/lib_qec.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/lib_qec.yaml b/.github/workflows/lib_qec.yaml index 302c5a9c2..1642021fb 100644 --- a/.github/workflows/lib_qec.yaml +++ b/.github/workflows/lib_qec.yaml @@ -252,7 +252,6 @@ jobs: # ========================================================================== multi-gpu-test: name: Multi-GPU tests - if: github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' needs: build-and-test strategy: fail-fast: false From 5e3247ea7ee74eaf897741419aa2a0da014aff19 Mon Sep 17 00:00:00 2001 From: Angela Burton Date: Thu, 2 Jul 2026 11:34:54 -0700 Subject: [PATCH 4/5] add ARM runner. Single job on both. Signed-off-by: Angela Burton --- .github/workflows/lib_qec.yaml | 36 +++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/.github/workflows/lib_qec.yaml b/.github/workflows/lib_qec.yaml index 1642021fb..b9345e104 100644 --- a/.github/workflows/lib_qec.yaml +++ b/.github/workflows/lib_qec.yaml @@ -244,22 +244,28 @@ jobs: # Multi-GPU tests - decoder placement across >= 2 GPUs (uses build artifacts # from build-and-test). # - # NVIDIA GHA runner labels follow linux--gpu--latest-; - # this job needs the 2-GPU variant. It runs on merges to the default branch - # and on manual dispatch only, to keep multi-GPU capacity out of the per-PR - # critical path; timeout-minutes turns a missing runner label into a visible - # failure instead of an indefinite queue. + # NVIDIA GHA runner labels are not fully derivable across runner families, so + # keep full labels in the matrix and use runner.arch for matching build + # artifacts and containers. # ========================================================================== multi-gpu-test: - name: Multi-GPU tests + name: Multi-GPU tests (${{ matrix.runner.arch }}, CUDA ${{ matrix.cuda_version }}) needs: build-and-test strategy: fail-fast: false matrix: - cuda_version: ['12.6', '13.0'] - runs-on: nv-gpu-amd64-rtxpro6000-2gpu + include: + - runner: + arch: amd64 + label: nv-gpu-amd64-rtxpro6000-2gpu + cuda_version: '13.0' + - runner: + arch: arm64 + label: linux-arm64-gpu-l4-latest-2 + cuda_version: '13.0' + runs-on: ${{ matrix.runner.label }} container: - image: ghcr.io/nvidia/cuda-quantum-devcontainer:amd64-cu${{ matrix.cuda_version }}-gcc12-main + image: ghcr.io/nvidia/cuda-quantum-devcontainer:${{ matrix.runner.arch }}-cu${{ matrix.cuda_version }}-gcc12-main # SYS_NICE: see build-and-test. options: --cap-add=SYS_NICE env: @@ -284,7 +290,7 @@ jobs: - name: Download build artifacts uses: actions/download-artifact@v4 with: - name: qec-build-amd64-cu${{ matrix.cuda_version }} + name: qec-build-${{ matrix.runner.arch }}-cu${{ matrix.cuda_version }} path: /tmp - name: Extract build artifacts @@ -308,12 +314,20 @@ jobs: run: | pip install --upgrade "cuquantum-python-cu${{ steps.config.outputs.cuda_major }}>=26.3.0" - - name: Install TensorRT (C++) + - name: Install TensorRT (C++, amd64) + if: matrix.runner.arch == 'amd64' run: | apt update apt-cache search tensorrt | awk '{print "Package: "$1"\nPin: version *+cuda${{ matrix.cuda_version }}\nPin-Priority: 1001\n"}' | tee /etc/apt/preferences.d/tensorrt-cuda${{ matrix.cuda_version }}.pref > /dev/null apt install -y tensorrt-dev + - name: Install TensorRT (C++, arm64) + if: matrix.runner.arch == 'arm64' && !startsWith(matrix.cuda_version, '12') + run: | + apt update + apt-cache search tensorrt | awk '{print "Package: "$1"\nPin: version *+cuda13.0\nPin-Priority: 1001\n"}' | tee /etc/apt/preferences.d/tensorrt-cuda13.0.pref > /dev/null + apt install -y tensorrt-dev + - name: Run multi-GPU C++ tests run: | cd $GITHUB_WORKSPACE/build_qec From b1237bdd7c502f2a233d071fbd60c26623a8ec04 Mon Sep 17 00:00:00 2001 From: Angela Burton Date: Thu, 2 Jul 2026 17:25:35 -0700 Subject: [PATCH 5/5] use ARM runner only; turn off failure when no multi-GPU tests are found Signed-off-by: Angela Burton --- .github/workflows/lib_qec.yaml | 38 ++++++++++------------------------ 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/.github/workflows/lib_qec.yaml b/.github/workflows/lib_qec.yaml index b9345e104..40c29284b 100644 --- a/.github/workflows/lib_qec.yaml +++ b/.github/workflows/lib_qec.yaml @@ -244,9 +244,10 @@ jobs: # Multi-GPU tests - decoder placement across >= 2 GPUs (uses build artifacts # from build-and-test). # - # NVIDIA GHA runner labels are not fully derivable across runner families, so - # keep full labels in the matrix and use runner.arch for matching build - # artifacts and containers. + # Keep this lane on the ARM L4 2-GPU runner while amd64 2-GPU runner + # availability is too limited for PR CI. The runner metadata remains + # structured so artifact and container selection stay arch-driven. Run CUDA + # 13.0 only because the ARM CUDA 12.6 lane does not support TensorRT. # ========================================================================== multi-gpu-test: name: Multi-GPU tests (${{ matrix.runner.arch }}, CUDA ${{ matrix.cuda_version }}) @@ -254,15 +255,10 @@ jobs: strategy: fail-fast: false matrix: - include: - - runner: - arch: amd64 - label: nv-gpu-amd64-rtxpro6000-2gpu - cuda_version: '13.0' - - runner: - arch: arm64 - label: linux-arm64-gpu-l4-latest-2 - cuda_version: '13.0' + runner: + - arch: arm64 + label: linux-arm64-gpu-l4-latest-2 + cuda_version: ['13.0'] runs-on: ${{ matrix.runner.label }} container: image: ghcr.io/nvidia/cuda-quantum-devcontainer:${{ matrix.runner.arch }}-cu${{ matrix.cuda_version }}-gcc12-main @@ -303,24 +299,10 @@ jobs: cuda_major=$(echo ${{ matrix.cuda_version }} | cut -d . -f1) echo "cuda_major=$cuda_major" >> $GITHUB_OUTPUT - - name: Verify at least 2 GPUs - run: | - nvidia-smi - count=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) - echo "GPU count: ${count}" - [ "${count}" -ge 2 ] || { echo "ERROR: expected >=2 GPUs, found ${count}"; exit 1; } - - name: Install cuStabilizer runtime run: | pip install --upgrade "cuquantum-python-cu${{ steps.config.outputs.cuda_major }}>=26.3.0" - - name: Install TensorRT (C++, amd64) - if: matrix.runner.arch == 'amd64' - run: | - apt update - apt-cache search tensorrt | awk '{print "Package: "$1"\nPin: version *+cuda${{ matrix.cuda_version }}\nPin-Priority: 1001\n"}' | tee /etc/apt/preferences.d/tensorrt-cuda${{ matrix.cuda_version }}.pref > /dev/null - apt install -y tensorrt-dev - - name: Install TensorRT (C++, arm64) if: matrix.runner.arch == 'arm64' && !startsWith(matrix.cuda_version, '12') run: | @@ -331,5 +313,7 @@ jobs: - name: Run multi-GPU C++ tests run: | cd $GITHUB_WORKSPACE/build_qec - ctest --output-on-failure --no-tests=error \ + # The matching tests will be added separately; until then, this + # command should not fail solely because the filter finds no tests. + ctest --output-on-failure \ -R "DecoderPool|HardwarePinningGpu|PoolRunsTwoTrt|CudaDeviceId"