From eae39db55f42afcb588758552cf11171cc333a4c Mon Sep 17 00:00:00 2001 From: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Sun, 19 Jul 2026 08:40:05 -0700 Subject: [PATCH 1/4] [TRTLLMINF-218][infra] Gate multi-GPU CI stages behind 'ci: full pre-merge approved' label Gate x86_64 and SBSA multi-GPU pre-merge CI stages behind the GitHub label 'ci: full pre-merge approved' so resource-intensive multi-GPU tests only run when explicitly approved by a CI approver. Changes: - L0_MergeRequest.groovy: add hasGithubPRLabel() (fail-open, reuses github-cred-trtllm-ci); insert label check at the top of [Test-x86_64-Multi-GPU] and [Test-SBSA-Multi-GPU] Remote Run stages; write description marker so wrapper can surface call-to-action - full-premerge-approval.yml: new GitHub Actions workflow (modeled after post-merge-approval.yml) to remove the label and post a deny comment when an unauthorized user applies the label - bot-command.yml: document label requirement for --add-multi-gpu-test and --only-multi-gpu-test - ci-overview.md: add Multi-GPU Tests section explaining label gating PostMerge pipelines and GitLab MR builds are exempt from the check. Wrapper (trtllm-github-ci) change tracked in GitLab branch trtllminf-218-full-premerge-label (based on MR #86). Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- .github/workflows/bot-command.yml | 4 +- .github/workflows/full-premerge-approval.yml | 222 +++++++++++++++++++ docs/source/developer-guide/ci-overview.md | 19 ++ jenkins/L0_MergeRequest.groovy | 64 ++++++ 4 files changed, 307 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/full-premerge-approval.yml diff --git a/.github/workflows/bot-command.yml b/.github/workflows/bot-command.yml index 30b5de652f55..cc486e91938e 100644 --- a/.github/workflows/bot-command.yml +++ b/.github/workflows/bot-command.yml @@ -55,9 +55,9 @@ jobs: "`--stage-list \"A10-PyTorch-1, xxx\"` *(OPTIONAL)* : Only run the specified test stages. Supports wildcard `*` for pattern matching (e.g., `\"*PerfSanity*\"` matches all stages containing PerfSanity). Examples: \"A10-PyTorch-1, xxx\", \"*PerfSanity*\". The patterns `\"*\"`, `\"*Post-Merge*\"`, and `\"*PerfSanity*\"`, including equivalent escaped or repeated-star forms and their use in comma-separated lists, require the `ci: post-merge approved` PR label. Note: Does **NOT** update GitHub check status.\n\n" + "`--gpu-type \"A30, H100_PCIe\"` *(OPTIONAL)* : Only run the test stages on the specified GPU types. Examples: \"A30, H100_PCIe\". Note: Does **NOT** update GitHub check status.\n\n" + "`--test-backend \"pytorch, cpp\"` *(OPTIONAL)* : Skip test stages which don't match the specified backends. Only support [pytorch, cpp, tensorrt, triton]. Examples: \"pytorch, cpp\" (does not run test stages with tensorrt or triton backend). Note: Does **NOT** update GitHub pipeline status.\n\n" + - "`--only-multi-gpu-test ` *(OPTIONAL)* : Only run the multi-GPU tests. Note: Does **NOT** update GitHub check status.\n\n" + + "`--only-multi-gpu-test ` *(OPTIONAL)* : Only run the multi-GPU tests. Requires the `ci: full pre-merge approved` label on the PR (ask a member of NVIDIA/trt-llm-ci-approvers). Note: Does **NOT** update GitHub check status.\n\n" + "`--disable-multi-gpu-test ` *(OPTIONAL)* : Disable the multi-GPU tests. Note: Does **NOT** update GitHub check status.\n\n" + - "`--add-multi-gpu-test ` *(OPTIONAL)* : Force run the multi-GPU tests in addition to running L0 pre-merge pipeline.\n\n" + + "`--add-multi-gpu-test ` *(OPTIONAL)* : Force run the multi-GPU tests in addition to running L0 pre-merge pipeline. Requires the `ci: full pre-merge approved` label on the PR (ask a member of NVIDIA/trt-llm-ci-approvers).\n\n" + "`--post-merge ` *(OPTIONAL)* : Run the L0 post-merge pipeline instead of the ordinary L0 pre-merge pipeline. Requires the `ci: post-merge approved` PR label applied by an active member of `NVIDIA/trt-llm-ci-approvers`. The approval label remains in place when new commits are pushed.\n\n" + "`--extra-stage \"H100_PCIe-TensorRT-Post-Merge-1, xxx\"` *(OPTIONAL)* : Run the ordinary L0 pre-merge pipeline and specified test stages. Supports wildcard `*` for pattern matching. Examples: --extra-stage \"H100_PCIe-TensorRT-Post-Merge-1, xxx\", --extra-stage \"*Post-Merge*\". The patterns `\"*\"`, `\"*Post-Merge*\"`, and `\"*PerfSanity*\"`, including equivalent escaped or repeated-star forms and their use in comma-separated lists, require the `ci: post-merge approved` PR label.\n\n" + "`--detailed-log ` *(OPTIONAL)* : Enable flushing out all logs to the Jenkins console. This will significantly increase the log volume and may slow down the job.\n\n" + diff --git a/.github/workflows/full-premerge-approval.yml b/.github/workflows/full-premerge-approval.yml new file mode 100644 index 000000000000..a6e27885bdfb --- /dev/null +++ b/.github/workflows/full-premerge-approval.yml @@ -0,0 +1,222 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Guard Full Pre-Merge Approval Label + +on: + # Intentional: this runs static default-branch API logic only. It never checks + # out or executes PR code; it uses GitHub APIs to validate membership and + # manage this PR's approval label/comment. + pull_request_target: + types: [labeled] + +permissions: + contents: read + pull-requests: write + +jobs: + guard-full-premerge-approval: + concurrency: + group: premerge-label-guard-${{ github.event.pull_request.number }} + cancel-in-progress: false + # Keep a started job eligible to reach fail-closed cleanup after cancellation. + if: >- + always() && + github.repository == 'NVIDIA/TensorRT-LLM' && + github.event.action == 'labeled' && + github.event.label.name == 'ci: full pre-merge approved' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Validate full pre-merge approver + id: validate + if: github.event.action == 'labeled' + uses: actions/github-script@v8 + with: + github-token: ${{ secrets.TRTLLM_AGENT_SHARED_TOKEN }} + result-encoding: string + script: | + const approvalLabel = 'ci: full pre-merge approved'; + const owner = context.repo.owner; + const repo = context.repo.repo; + const issueNumber = context.payload.pull_request.number; + let actor = context.payload.sender?.login || context.actor; + let labelEventId = ''; + let timelineVerified = false; + + try { + const events = await github.paginate( + github.rest.issues.listEventsForTimeline, + { owner, repo, issue_number: issueNumber, per_page: 100 } + ); + const latestApprovalEvent = events + .filter( + (event) => + event.event === 'labeled' && + event.label?.name?.toLowerCase() === approvalLabel.toLowerCase() + ) + .at(-1); + if (latestApprovalEvent) { + actor = latestApprovalEvent.actor?.login || actor; + labelEventId = String(latestApprovalEvent.id); + timelineVerified = true; + } else { + core.warning('Could not identify the latest full pre-merge approval event.'); + } + } catch (error) { + core.warning( + 'Could not read the latest full pre-merge approval event: ' + error.message + ); + } + + core.setOutput('validated_actor', actor); + core.setOutput('label_event_id', labelEventId); + if (!timelineVerified) { + return 'false'; + } + try { + const response = await github.request( + 'GET /orgs/{org}/teams/{team_slug}/memberships/{username}', + { + org: 'NVIDIA', + team_slug: 'trt-llm-ci-approvers', + username: actor, + } + ); + const authorized = response.data.state === 'active'; + console.log( + actor + ' active membership in NVIDIA/trt-llm-ci-approvers: ' + authorized + ); + return authorized ? 'true' : 'false'; + } catch (error) { + if (error.status === 404) { + console.log( + actor + ' is not an active member of NVIDIA/trt-llm-ci-approvers.' + ); + } else { + core.warning( + 'Could not verify full pre-merge approver ' + actor + ': ' + error.message + ); + } + return 'false'; + } + + - name: Clear unauthorized full pre-merge approval + # Let a started job attempt fail-closed cleanup after normal cancellation. + if: always() && steps.validate.outputs.result != 'true' + uses: actions/github-script@v8 + env: + VALIDATED_ACTOR: ${{ steps.validate.outputs.validated_actor }} + VALIDATED_LABEL_EVENT_ID: ${{ steps.validate.outputs.label_event_id }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const approvalLabel = 'ci: full pre-merge approved'; + const actor = + process.env.VALIDATED_ACTOR || + context.payload.sender?.login || + context.actor; + const validatedEventId = process.env.VALIDATED_LABEL_EVENT_ID || ''; + const owner = context.repo.owner; + const repo = context.repo.repo; + const issueNumber = context.payload.pull_request.number; + + try { + const pullRequest = await github.rest.pulls.get({ + owner, + repo, + pull_number: issueNumber, + }); + const labelIsPresent = pullRequest.data.labels.some( + (label) => + label.name?.toLowerCase() === approvalLabel.toLowerCase() + ); + if (!labelIsPresent) { + console.log('Full pre-merge approval label is already absent; no cleanup needed.'); + return; + } + } catch (error) { + core.warning( + 'Could not read the current full pre-merge approval label state; continuing validation: ' + + error.message + ); + } + + try { + const events = await github.paginate( + github.rest.issues.listEventsForTimeline, + { owner, repo, issue_number: issueNumber, per_page: 100 } + ); + const latestApprovalEvent = events + .filter( + (event) => + event.event === 'labeled' && + event.label?.name?.toLowerCase() === approvalLabel.toLowerCase() + ) + .at(-1); + const latestEventId = latestApprovalEvent + ? String(latestApprovalEvent.id) + : ''; + const latestActor = latestApprovalEvent?.actor?.login || ''; + if (!latestApprovalEvent) { + core.warning( + 'Could not identify the latest full pre-merge approval event; removing the label to fail closed.' + ); + } else if (!latestActor) { + core.warning( + 'Could not identify the latest full pre-merge approval actor; removing the label to fail closed.' + ); + } else if (!validatedEventId) { + core.warning( + 'The validation run did not bind an approval event; removing the label to fail closed.' + ); + } else if ( + latestEventId !== validatedEventId || + latestActor !== actor + ) { + console.log( + 'A newer full pre-merge approval event was found; leaving it for its own validation run.' + ); + return; + } + } catch (error) { + core.warning( + 'Could not re-check the latest full pre-merge approval event; removing the label to fail closed: ' + + error.message + ); + } + + try { + await github.rest.issues.removeLabel({ + owner, + repo, + issue_number: issueNumber, + name: approvalLabel, + }); + } catch (error) { + if (error.status !== 404) { + throw error; + } + } + + await github.rest.issues.createComment({ + owner, + repo, + issue_number: issueNumber, + body: + 'Removed the "' + approvalLabel + '" label because @' + actor + + ' could not be verified as an active member of ' + + 'NVIDIA/trt-llm-ci-approvers. Ask a member of that team to apply it.', + }); diff --git a/docs/source/developer-guide/ci-overview.md b/docs/source/developer-guide/ci-overview.md index a272a0018df3..5ea45b297a2b 100644 --- a/docs/source/developer-guide/ci-overview.md +++ b/docs/source/developer-guide/ci-overview.md @@ -105,6 +105,25 @@ full:A100/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_a Changes to `waives.txt` should include a bug link or brief explanation so other developers understand why the test is disabled. +## Multi-GPU Tests + +Running multi-GPU tests (`--add-multi-gpu-test`, `--only-multi-gpu-test`) requires the +`ci: full pre-merge approved` label on the PR. Without this label the +`[Test-x86_64-Multi-GPU] Remote Run` and `[Test-SBSA-Multi-GPU] Remote Run` stages will +fail with an explanatory error. + +To obtain the label, ask a member of `NVIDIA/trt-llm-ci-approvers` to apply it. Only +members of that team can add the label; unauthorized additions are automatically removed +by the `Guard Full Pre-Merge Approval Label` GitHub Actions workflow. + +Once the label is present, re-trigger CI: + +```bash +/bot run --add-multi-gpu-test +``` + +Post-merge pipelines and GitLab MR builds are exempt from this label check. + ## Triggering CI Best Practices ### Triggering Post-merge tests diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 3f276522ee54..254d772105eb 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -644,6 +644,38 @@ def getGithubMRChangedFile(pipeline, githubPrApiUrl, function, filePath="") { return result } +// Returns true if the GitHub PR has the given label, false if absent. +// Fails open (returns true) on network/auth errors so CI is not blocked. +// Only meaningful for GitHub PRs; callers must guard with globalVars[GITHUB_PR_API_URL]. +def hasGithubPRLabel(pipeline, githubPrApiUrl, labelName) { + try { + def rawDataJson = null + withCredentials([ + usernamePassword( + credentialsId: 'github-cred-trtllm-ci', + usernameVariable: 'NOT_USED_YET', + passwordVariable: 'GITHUB_API_TOKEN' + ), + ]) { + rawDataJson = pipeline.sh( + script: """ + curl --silent --header "Authorization: Bearer \${GITHUB_API_TOKEN}" \ + --url "${githubPrApiUrl}" + """, + returnStdout: true + ) + } + def prData = readJSON text: rawDataJson, returnPojo: true + def labels = prData.get("labels", []) ?: [] + def found = labels.any { label -> label?.get("name", "") == labelName } + echo "[hasGithubPRLabel] label '${labelName}' present: ${found}" + return found + } catch (Exception e) { + echo "[hasGithubPRLabel] Warning: failed to check GitHub PR labels: ${e.toString()}. Failing open." + return true // fail-open: do not block CI if the label check itself fails + } +} + def getMergeRequestChangedFileList(pipeline, globalVars) { def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) if (env.alternativeTRT || isOfficialPostMergeJob) { @@ -1573,6 +1605,22 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "x86_64 test job is skipped due to Jenkins configuration" return } + // Gate: require 'ci: full pre-merge approved' label for GitHub PRs. + // PostMerge pipelines and GitLab MR builds are exempt. + if (globalVars[GITHUB_PR_API_URL] && !(env.JOB_NAME ==~ /.*PostMerge.*/)) { + if (!hasGithubPRLabel(pipeline, globalVars[GITHUB_PR_API_URL], + "ci: full pre-merge approved")) { + def existingDesc = currentBuild.description ?: "" + currentBuild.description = existingDesc + (existingDesc ? "
" : "") + + "" + + "Multi-GPU tests require label 'ci: full pre-merge approved'" + + "" + error "x86_64 Multi-GPU tests require the GitHub label " + + "'ci: full pre-merge approved' on this PR. " + + "Ask a member of NVIDIA/trt-llm-ci-approvers to add the label, " + + "then re-trigger CI." + } + } try { def testFilterJson = writeJSON returnText: true, json: testFilter def additionalParameters = [ @@ -1689,6 +1737,22 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "SBSA test job is skipped due to Jenkins configuration" return } + // Gate: require 'ci: full pre-merge approved' label for GitHub PRs. + // PostMerge pipelines and GitLab MR builds are exempt. + if (globalVars[GITHUB_PR_API_URL] && !(env.JOB_NAME ==~ /.*PostMerge.*/)) { + if (!hasGithubPRLabel(pipeline, globalVars[GITHUB_PR_API_URL], + "ci: full pre-merge approved")) { + def existingDesc = currentBuild.description ?: "" + currentBuild.description = existingDesc + (existingDesc ? "
" : "") + + "" + + "Multi-GPU tests require label 'ci: full pre-merge approved'" + + "" + error "SBSA Multi-GPU tests require the GitHub label " + + "'ci: full pre-merge approved' on this PR. " + + "Ask a member of NVIDIA/trt-llm-ci-approvers to add the label, " + + "then re-trigger CI." + } + } try { def testFilterJson = writeJSON returnText: true, json: testFilter def additionalParameters = [ From 7d6b4d998f18f5d2be92c15ce64a0b56a7d568cf Mon Sep 17 00:00:00 2001 From: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Wed, 22 Jul 2026 02:36:29 -0700 Subject: [PATCH 2/4] [TRTLLMINF-218][infra] Address CodeRabbit review feedback - hasGithubPRLabel: add --fail --show-error to curl so HTTP errors (401/403/5xx) exit non-zero and reach the catch/fail-open handler instead of returning empty HTML that blocks CI - Extract duplicated multi-GPU label gate logic into shared requireMultiGpuApprovalLabel() helper - ci-overview.md: add NVIDIA copyright header, add Multi-GPU Tests entry to TOC, clarify rerun instructions for both --add-multi-gpu-test and --only-multi-gpu-test Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- docs/source/developer-guide/ci-overview.md | 27 +++++++++- jenkins/L0_MergeRequest.groovy | 60 ++++++++++------------ 2 files changed, 52 insertions(+), 35 deletions(-) diff --git a/docs/source/developer-guide/ci-overview.md b/docs/source/developer-guide/ci-overview.md index 5ea45b297a2b..b5d9eafe1aa0 100644 --- a/docs/source/developer-guide/ci-overview.md +++ b/docs/source/developer-guide/ci-overview.md @@ -1,3 +1,20 @@ + + # Continuous Integration Overview This page explains how TensorRT‑LLM's CI is organized and how individual tests map to Jenkins stages. Most stages execute integration tests defined in YAML files, while unit tests run as part of a merge‑request pipeline. The sections below describe how to locate a test and trigger the stage that runs it. @@ -9,7 +26,8 @@ This page explains how TensorRT‑LLM's CI is organized and how individual tests 4. [Jenkins stage names](#jenkins-stage-names) 5. [Finding the stage for a test](#finding-the-stage-for-a-test) 6. [Waiving tests](#waiving-tests) -7. [Triggering CI Best Practices](#triggering-ci-best-practices) +7. [Multi-GPU Tests](#multi-gpu-tests) +8. [Triggering CI Best Practices](#triggering-ci-best-practices) ## CI pipelines @@ -116,10 +134,15 @@ To obtain the label, ask a member of `NVIDIA/trt-llm-ci-approvers` to apply it. members of that team can add the label; unauthorized additions are automatically removed by the `Guard Full Pre-Merge Approval Label` GitHub Actions workflow. -Once the label is present, re-trigger CI: +Once the label is present, re-trigger CI with the same command you used +originally. For example: ```bash +# Run the normal pipeline plus multi-GPU stages /bot run --add-multi-gpu-test + +# Run only multi-GPU stages +/bot run --only-multi-gpu-test ``` Post-merge pipelines and GitLab MR builds are exempt from this label check. diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 254d772105eb..f8f0edef2463 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -659,7 +659,8 @@ def hasGithubPRLabel(pipeline, githubPrApiUrl, labelName) { ]) { rawDataJson = pipeline.sh( script: """ - curl --silent --header "Authorization: Bearer \${GITHUB_API_TOKEN}" \ + curl --silent --fail --show-error \ + --header "Authorization: Bearer \${GITHUB_API_TOKEN}" \ --url "${githubPrApiUrl}" """, returnStdout: true @@ -676,6 +677,29 @@ def hasGithubPRLabel(pipeline, githubPrApiUrl, labelName) { } } +// Gate multi-GPU stages behind 'ci: full pre-merge approved' label. +// Returns true if the stage should be blocked (label absent), false otherwise. +// Exempt: PostMerge pipelines and GitLab MR builds (no GITHUB_PR_API_URL). +def requireMultiGpuApprovalLabel(pipeline, globalVars, String arch) { + if (!globalVars[GITHUB_PR_API_URL] || (env.JOB_NAME ==~ /.*PostMerge.*/)) { + return false + } + if (hasGithubPRLabel(pipeline, globalVars[GITHUB_PR_API_URL], + "ci: full pre-merge approved")) { + return false + } + def existingDesc = currentBuild.description ?: "" + currentBuild.description = existingDesc + (existingDesc ? "
" : "") + + "" + + "Multi-GPU tests require label 'ci: full pre-merge approved'" + + "" + error "${arch} Multi-GPU tests require the GitHub label " + + "'ci: full pre-merge approved' on this PR. " + + "Ask a member of NVIDIA/trt-llm-ci-approvers to add the label, " + + "then re-trigger CI." + return true // unreachable, but keeps intent clear +} + def getMergeRequestChangedFileList(pipeline, globalVars) { def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) if (env.alternativeTRT || isOfficialPostMergeJob) { @@ -1605,22 +1629,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "x86_64 test job is skipped due to Jenkins configuration" return } - // Gate: require 'ci: full pre-merge approved' label for GitHub PRs. - // PostMerge pipelines and GitLab MR builds are exempt. - if (globalVars[GITHUB_PR_API_URL] && !(env.JOB_NAME ==~ /.*PostMerge.*/)) { - if (!hasGithubPRLabel(pipeline, globalVars[GITHUB_PR_API_URL], - "ci: full pre-merge approved")) { - def existingDesc = currentBuild.description ?: "" - currentBuild.description = existingDesc + (existingDesc ? "
" : "") + - "" + - "Multi-GPU tests require label 'ci: full pre-merge approved'" + - "" - error "x86_64 Multi-GPU tests require the GitHub label " + - "'ci: full pre-merge approved' on this PR. " + - "Ask a member of NVIDIA/trt-llm-ci-approvers to add the label, " + - "then re-trigger CI." - } - } + requireMultiGpuApprovalLabel(pipeline, globalVars, "x86_64") try { def testFilterJson = writeJSON returnText: true, json: testFilter def additionalParameters = [ @@ -1737,22 +1746,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "SBSA test job is skipped due to Jenkins configuration" return } - // Gate: require 'ci: full pre-merge approved' label for GitHub PRs. - // PostMerge pipelines and GitLab MR builds are exempt. - if (globalVars[GITHUB_PR_API_URL] && !(env.JOB_NAME ==~ /.*PostMerge.*/)) { - if (!hasGithubPRLabel(pipeline, globalVars[GITHUB_PR_API_URL], - "ci: full pre-merge approved")) { - def existingDesc = currentBuild.description ?: "" - currentBuild.description = existingDesc + (existingDesc ? "
" : "") + - "" + - "Multi-GPU tests require label 'ci: full pre-merge approved'" + - "" - error "SBSA Multi-GPU tests require the GitHub label " + - "'ci: full pre-merge approved' on this PR. " + - "Ask a member of NVIDIA/trt-llm-ci-approvers to add the label, " + - "then re-trigger CI." - } - } + requireMultiGpuApprovalLabel(pipeline, globalVars, "SBSA") try { def testFilterJson = writeJSON returnText: true, json: testFilter def additionalParameters = [ From 5471939c0a9446bfc1b5017bb8db175137e6841a Mon Sep 17 00:00:00 2001 From: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Thu, 23 Jul 2026 03:17:31 -0700 Subject: [PATCH 3/4] [TRTLLMINF-218][infra] Use shared lib for label+author validation and add exempt stage list - L0_MergeRequest.groovy: replace hasGithubPRLabel() with trtllm_utils.validatePRLabelApproval() from the shared lib, which verifies both label existence and labeler team membership; error message now distinguishes "label not present" from "labeler not authorized"; fail-open on API errors; echo reason on early return - L0_Test.groovy: add MULTI_GPU_RUN_WITH_SINGLE exempt list; matching multi-GPU stages run inside the single-GPU job instead of waiting for the separate multi-GPU dispatch (which requires the label) Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 70 ++++++++++++++-------------------- jenkins/L0_Test.groovy | 34 ++++++++++++++--- 2 files changed, 58 insertions(+), 46 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index f8f0edef2463..d6c89611a072 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -14,7 +14,7 @@ * limitations under the License. */ -@Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _ +@Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@user/zhanruis/TRTLLMINF-218-pr-label-approval']) _ import java.lang.InterruptedException import groovy.transform.Field @@ -644,60 +644,48 @@ def getGithubMRChangedFile(pipeline, githubPrApiUrl, function, filePath="") { return result } -// Returns true if the GitHub PR has the given label, false if absent. -// Fails open (returns true) on network/auth errors so CI is not blocked. -// Only meaningful for GitHub PRs; callers must guard with globalVars[GITHUB_PR_API_URL]. -def hasGithubPRLabel(pipeline, githubPrApiUrl, labelName) { - try { - def rawDataJson = null - withCredentials([ - usernamePassword( - credentialsId: 'github-cred-trtllm-ci', - usernameVariable: 'NOT_USED_YET', - passwordVariable: 'GITHUB_API_TOKEN' - ), - ]) { - rawDataJson = pipeline.sh( - script: """ - curl --silent --fail --show-error \ - --header "Authorization: Bearer \${GITHUB_API_TOKEN}" \ - --url "${githubPrApiUrl}" - """, - returnStdout: true - ) - } - def prData = readJSON text: rawDataJson, returnPojo: true - def labels = prData.get("labels", []) ?: [] - def found = labels.any { label -> label?.get("name", "") == labelName } - echo "[hasGithubPRLabel] label '${labelName}' present: ${found}" - return found - } catch (Exception e) { - echo "[hasGithubPRLabel] Warning: failed to check GitHub PR labels: ${e.toString()}. Failing open." - return true // fail-open: do not block CI if the label check itself fails - } -} - // Gate multi-GPU stages behind 'ci: full pre-merge approved' label. -// Returns true if the stage should be blocked (label absent), false otherwise. +// Uses trtllm_utils.validatePRLabelApproval() from the shared lib to verify +// both label existence and that the labeler is an active team member. // Exempt: PostMerge pipelines and GitLab MR builds (no GITHUB_PR_API_URL). def requireMultiGpuApprovalLabel(pipeline, globalVars, String arch) { - if (!globalVars[GITHUB_PR_API_URL] || (env.JOB_NAME ==~ /.*PostMerge.*/)) { + if (!globalVars[GITHUB_PR_API_URL]) { + echo "[requireMultiGpuApprovalLabel] Skipping label check: not a GitHub PR (no GITHUB_PR_API_URL)" return false } - if (hasGithubPRLabel(pipeline, globalVars[GITHUB_PR_API_URL], - "ci: full pre-merge approved")) { + if (env.JOB_NAME ==~ /.*PostMerge.*/) { + echo "[requireMultiGpuApprovalLabel] Skipping label check: PostMerge pipeline is exempt" return false } + + def prMatch = (globalVars[GITHUB_PR_API_URL] =~ /\/pulls?\/(\d+)/) + if (!prMatch) { + echo "[requireMultiGpuApprovalLabel] Could not extract PR number from ${globalVars[GITHUB_PR_API_URL]}. Failing open." + return false + } + def prNumber = prMatch[0][1] + + def result = trtllm_utils.validatePRLabelApproval(pipeline, prNumber, "ci: full pre-merge approved") + if (!result.checkCompleted) { + // API error — fail-open: do not block CI if the label check itself fails + echo "[requireMultiGpuApprovalLabel] Label validation incomplete (${result.error}). Failing open." + return false + } + if (result.labelExists && result.authorized) { + return false + } + def existingDesc = currentBuild.description ?: "" currentBuild.description = existingDesc + (existingDesc ? "
" : "") + "" + "Multi-GPU tests require label 'ci: full pre-merge approved'" + "" - error "${arch} Multi-GPU tests require the GitHub label " + - "'ci: full pre-merge approved' on this PR. " + + def reason = !result.labelExists + ? "label not present" + : "label applied by '${result.actor}' who is not an active member of NVIDIA/trt-llm-ci-approvers" + error "${arch} Multi-GPU tests blocked: ${reason}. " + "Ask a member of NVIDIA/trt-llm-ci-approvers to add the label, " + "then re-trigger CI." - return true // unreachable, but keeps intent clear } def getMergeRequestChangedFileList(pipeline, globalVars) { diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index ed2e0a88f113..ad05bf210997 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2289,6 +2289,16 @@ def stageMatchesAnyPattern(String key, List patterns) { } // Test filter flags +// Multi-GPU stages matching any entry here run inside the single-GPU job +// instead of waiting for the separate multi-GPU dispatch (which requires +// the 'ci: full pre-merge approved' label). Supports exact names and +// wildcard (*) patterns. +@Field +def MULTI_GPU_RUN_WITH_SINGLE = [ + // Add stage patterns here, e.g.: + // "DGX_H100-2_GPUs-*", +] + @Field def REUSE_TEST = "reuse_test" @Field @@ -6071,12 +6081,26 @@ pipeline { singleGpuJobs = parallelJobs dgxJobs = [:] - def testPhase2StageName = env.testPhase2StageName - if (testPhase2StageName) { - def multiGpuPattern = /\d+_GPUs/ - singleGpuJobs = parallelJobs.findAll{!(it.key =~ multiGpuPattern)} - dgxJobs = parallelJobs.findAll{it.key =~ multiGpuPattern} + def testPhase2StageName = env.testPhase2StageName + if (testPhase2StageName) { + def multiGpuPattern = /\d+_GPUs/ + singleGpuJobs = parallelJobs.findAll{!(it.key =~ multiGpuPattern)} + dgxJobs = parallelJobs.findAll{it.key =~ multiGpuPattern} + + // Move approval-exempt multi-GPU stages into singleGpuJobs so they + // run without waiting for the multi-GPU dispatch (which requires + // the 'ci: full pre-merge approved' label). + def exemptJobs = dgxJobs.findAll { stageName, stageValue -> + MULTI_GPU_RUN_WITH_SINGLE.any { pattern -> + stageMatchesPattern(stageName, pattern) + } + } + if (exemptJobs) { + echo "[Multi-GPU split] Moving ${exemptJobs.keySet()} to single-GPU job (approval-exempt)" + singleGpuJobs += exemptJobs + dgxJobs -= exemptJobs } + } if (env.JOB_NAME ==~ /.*Single-GPU.*/) { echo "Only run single-GPU tests." From 6ccf5e7a5e334bdf2ecccaea223cbe684102dfab Mon Sep 17 00:00:00 2001 From: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:06:49 -0700 Subject: [PATCH 4/4] [TRTLLMINF-218][infra] Show specific block reason in Blocked stage error message - requireMultiGpuApprovalLabel() now returns the block reason string (instead of boolean true) so callers can display the exact cause - Blocked stage error message now shows whether the label is missing or was applied by an unauthorized user, plus guidance to contact NVIDIA/trt-llm-ci-approvers Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 40 +++++++++++++++++++++++++++------- jenkins/L0_Test.groovy | 36 +++++++++++++++--------------- 2 files changed, 50 insertions(+), 26 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index d6c89611a072..9571d0c58b2f 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -14,7 +14,7 @@ * limitations under the License. */ -@Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@user/zhanruis/TRTLLMINF-218-pr-label-approval']) _ +@Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _ import java.lang.InterruptedException import groovy.transform.Field @@ -675,17 +675,20 @@ def requireMultiGpuApprovalLabel(pipeline, globalVars, String arch) { return false } + // Label missing or unauthorized — write description marker for wrapper + // to surface in PR comment, and return the block reason string. def existingDesc = currentBuild.description ?: "" currentBuild.description = existingDesc + (existingDesc ? "
" : "") + "" + "Multi-GPU tests require label 'ci: full pre-merge approved'" + "" def reason = !result.labelExists - ? "label not present" - : "label applied by '${result.actor}' who is not an active member of NVIDIA/trt-llm-ci-approvers" - error "${arch} Multi-GPU tests blocked: ${reason}. " + - "Ask a member of NVIDIA/trt-llm-ci-approvers to add the label, " + - "then re-trigger CI." + ? "label 'ci: full pre-merge approved' is not present on this PR" + : "label 'ci: full pre-merge approved' was applied by '${result.actor}' who is not an active member of NVIDIA/trt-llm-ci-approvers" + def blockMsg = "${arch} Multi-GPU tests blocked: ${reason}. " + + "Ask a member of NVIDIA/trt-llm-ci-approvers to add the label, then re-trigger CI." + echo "[requireMultiGpuApprovalLabel] ${blockMsg}" + return blockMsg } def getMergeRequestChangedFileList(pipeline, globalVars) { @@ -1611,13 +1614,25 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) } } + // Label gate: check before entering the Remote Run stage so a + // missing/unauthorized label shows as "Blocked" (not a Remote Run + // failure) and does not trigger fail-fast. + def x86LabelBlock = requireMultiGpuApprovalLabel(pipeline, globalVars, "x86_64") + if (x86LabelBlock) { + stage("[Test-x86_64-Multi-GPU] Blocked") { + catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + error x86LabelBlock + } + } + return + } + testStageName = "[Test-x86_64-Multi-GPU] Remote Run" stage(testStageName) { if (X86_TEST_CHOICE == STAGE_CHOICE_SKIP) { echo "x86_64 test job is skipped due to Jenkins configuration" return } - requireMultiGpuApprovalLabel(pipeline, globalVars, "x86_64") try { def testFilterJson = writeJSON returnText: true, json: testFilter def additionalParameters = [ @@ -1728,13 +1743,22 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) } } + def sbsaLabelBlock = requireMultiGpuApprovalLabel(pipeline, globalVars, "SBSA") + if (sbsaLabelBlock) { + stage("[Test-SBSA-Multi-GPU] Blocked") { + catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + error sbsaLabelBlock + } + } + return + } + testStageName = "[Test-SBSA-Multi-GPU] Remote Run" stage(testStageName) { if (SBSA_TEST_CHOICE == STAGE_CHOICE_SKIP) { echo "SBSA test job is skipped due to Jenkins configuration" return } - requireMultiGpuApprovalLabel(pipeline, globalVars, "SBSA") try { def testFilterJson = writeJSON returnText: true, json: testFilter def additionalParameters = [ diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index ad05bf210997..56bbebe8a0e5 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -6081,26 +6081,26 @@ pipeline { singleGpuJobs = parallelJobs dgxJobs = [:] - def testPhase2StageName = env.testPhase2StageName - if (testPhase2StageName) { - def multiGpuPattern = /\d+_GPUs/ - singleGpuJobs = parallelJobs.findAll{!(it.key =~ multiGpuPattern)} - dgxJobs = parallelJobs.findAll{it.key =~ multiGpuPattern} - - // Move approval-exempt multi-GPU stages into singleGpuJobs so they - // run without waiting for the multi-GPU dispatch (which requires - // the 'ci: full pre-merge approved' label). - def exemptJobs = dgxJobs.findAll { stageName, stageValue -> - MULTI_GPU_RUN_WITH_SINGLE.any { pattern -> - stageMatchesPattern(stageName, pattern) + def testPhase2StageName = env.testPhase2StageName + if (testPhase2StageName) { + def multiGpuPattern = /\d+_GPUs/ + singleGpuJobs = parallelJobs.findAll{!(it.key =~ multiGpuPattern)} + dgxJobs = parallelJobs.findAll{it.key =~ multiGpuPattern} + + // Move approval-exempt multi-GPU stages into singleGpuJobs so they + // run without waiting for the multi-GPU dispatch (which requires + // the 'ci: full pre-merge approved' label). + def exemptJobs = dgxJobs.findAll { stageName, stageValue -> + MULTI_GPU_RUN_WITH_SINGLE.any { pattern -> + stageMatchesPattern(stageName, pattern) + } + } + if (exemptJobs) { + echo "[Multi-GPU split] Moving ${exemptJobs.keySet()} to single-GPU job (approval-exempt)" + singleGpuJobs += exemptJobs + dgxJobs -= exemptJobs } } - if (exemptJobs) { - echo "[Multi-GPU split] Moving ${exemptJobs.keySet()} to single-GPU job (approval-exempt)" - singleGpuJobs += exemptJobs - dgxJobs -= exemptJobs - } - } if (env.JOB_NAME ==~ /.*Single-GPU.*/) { echo "Only run single-GPU tests."