From 0c473ac0cbbcf15a1fa8f3fde44eb78d7400346b Mon Sep 17 00:00:00 2001 From: jkim323 Date: Sat, 27 Jun 2026 22:12:00 -0700 Subject: [PATCH 01/11] fix workflow + broken links --- .github/workflows/docusaurus-tests.yml | 11 ++++++----- docs/agents/project-planning/README.md | 10 +++++----- docs/agents/project-planning/arch-diagram-builder.md | 2 +- docs/agents/project-planning/brd-prd-builders.md | 2 +- docs/agents/project-planning/security-plan-creator.md | 4 ++-- 5 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docusaurus-tests.yml b/.github/workflows/docusaurus-tests.yml index 60a5506f5..bc5a316b6 100644 --- a/.github/workflows/docusaurus-tests.yml +++ b/.github/workflows/docusaurus-tests.yml @@ -43,17 +43,18 @@ jobs: Write-Output 'Docusaurus tests requested for all files.' } else { $baseRef = if ($env:GITHUB_BASE_REF) { $env:GITHUB_BASE_REF } else { 'main' } - # Trigger on Docusaurus application changes only: the site app, plugins, - # configs, and accessibility tests all live under docs/docusaurus/. - # Pure markdown content edits elsewhere under docs/ are intentionally - # excluded so content-only PRs skip the expensive a11y/build job. + # Trigger on any change under docs/: the whole-site build compiles all of + # docs/** (docusaurus.config.js sets the docs plugin path to '../', excluding + # only docusaurus/** and announcements/**), so watching docs/ matches the + # build's real input set. This catches broken links in markdown content + # pages on the PR instead of only on the post-merge deploy build. # The collections/ tree is included because docusaurus.config.js reads # collections/*.collection.yml at build time (collectionCounts) and the # tests consume it via COLLECTIONS_DIR, so changes there alter site and # test output. The workflow files themselves are included so test/config # changes still trigger a run. $changed = @(git diff --name-only --diff-filter=ACMR "origin/$baseRef...HEAD" -- ` - docs/docusaurus ` + docs ` collections ` .github/workflows/docusaurus-tests.yml ` .github/workflows/deploy-docs.yml) diff --git a/docs/agents/project-planning/README.md b/docs/agents/project-planning/README.md index cfde1cef9..90ab1646f 100644 --- a/docs/agents/project-planning/README.md +++ b/docs/agents/project-planning/README.md @@ -29,7 +29,7 @@ These agents bring structure and consistency to activities that teams often hand | [BRD Builder](brd-prd-builders) | Requirements | 3-phase Q&A | JSON state | Business requirements document | | [PRD Builder](brd-prd-builders) | Requirements | 7-phase Q&A | JSON state | Product requirements document | | [ADR Creator](adr-creation) | Architecture | 3-phase Frame/Decide/Govern | JSON state | Architecture decision record | -| [Security Planner](../security/README) | Security | 6-phase STRIDE | JSON state | Security model and backlog | +| [Security Planner](../security/README.md) | Security | 6-phase STRIDE | JSON state | Security model and backlog | ## Requirements @@ -57,7 +57,7 @@ The Security Planner applies STRIDE-based security model analysis across seven o > [!IMPORTANT] > Run security planning after architecture decisions stabilize. Changes to infrastructure or service boundaries may invalidate earlier security models. -See the [Security Planning](../security/README) guide for the workflow, operational buckets, and invocation details. +See the [Security Planning](../security/README.md) guide for the workflow, operational buckets, and invocation details. ## Prerequisites @@ -88,9 +88,9 @@ For greenfield projects, follow this order to build artifacts that feed into eac ## Related Documentation -* [RPI Documentation](../../rpi/README): Task research, planning, and implementation workflows -* [GitHub Backlog Manager](../github-backlog/README): Issue lifecycle management for GitHub repositories -* [ADO Backlog Manager](../ado-backlog/README): Work item management for Azure DevOps projects +* [RPI Documentation](../../rpi/README.md): Task research, planning, and implementation workflows +* [GitHub Backlog Manager](../github-backlog/README.md): Issue lifecycle management for GitHub repositories +* [ADO Backlog Manager](../ado-backlog/README.md): Work item management for Azure DevOps projects --- diff --git a/docs/agents/project-planning/arch-diagram-builder.md b/docs/agents/project-planning/arch-diagram-builder.md index 627339d2c..2b1a49ff9 100644 --- a/docs/agents/project-planning/arch-diagram-builder.md +++ b/docs/agents/project-planning/arch-diagram-builder.md @@ -38,7 +38,7 @@ Analyze the Terraform and Bicep files in this repository and create an ASCII arc ## Related Documentation -* [Project Planning Agents](README) +* [Project Planning Agents](README.md) * [ADR Creator](adr-creation) --- diff --git a/docs/agents/project-planning/brd-prd-builders.md b/docs/agents/project-planning/brd-prd-builders.md index 7385760f3..66d7a4455 100644 --- a/docs/agents/project-planning/brd-prd-builders.md +++ b/docs/agents/project-planning/brd-prd-builders.md @@ -229,7 +229,7 @@ Output the PRD with measurable requirements in every section. ## Next Steps 1. Feed your completed BRD or PRD into the [ADR Creator](adr-creation) for architectural decisions -2. See [Project Planning Agents](README) for the full agent catalog +2. See [Project Planning Agents](README.md) for the full agent catalog > [!TIP] > Both agents work best when you provide a clear project name at invocation. The agents can derive a working title from context, but explicit scope accelerates the Assess phase. diff --git a/docs/agents/project-planning/security-plan-creator.md b/docs/agents/project-planning/security-plan-creator.md index 6e7151910..6ee6a211e 100644 --- a/docs/agents/project-planning/security-plan-creator.md +++ b/docs/agents/project-planning/security-plan-creator.md @@ -14,7 +14,7 @@ This page has moved. The former Security Plan Creator agent is now the **Securit ## Where to Go -* [Security Planning](../security/README): Overview, six-phase workflow, operational buckets, and invocation details. +* [Security Planning](../security/README.md): Overview, six-phase workflow, operational buckets, and invocation details. * [Entry Modes](../security/entry-modes): From-PRD and capture entry modes. * [Phase Reference](../security/phase-reference): Phase-by-phase inputs, outputs, and state transitions. * [Handoff Pipeline](../security/handoff-pipeline): Backlog generation and RAI Planner recommendation. @@ -22,7 +22,7 @@ This page has moved. The former Security Plan Creator agent is now the **Securit ## Next Steps 1. Feed security findings into an [ADR](adr-creation) to document security architecture decisions -2. See [Project Planning Agents](README) for the full agent catalog +2. See [Project Planning Agents](README.md) for the full agent catalog > [!TIP] > Run the Security Planner after completing your [BRD or PRD](brd-prd-builders) to align threat analysis with documented requirements and system boundaries. From b85c2f02cb48c4eb9484fd07ddde2fdd14d90c62 Mon Sep 17 00:00:00 2001 From: jkim323 Date: Sat, 27 Jun 2026 23:37:11 -0700 Subject: [PATCH 02/11] add pr validation aggregator --- .../instructions/workflows.instructions.md | 9 + .github/workflows/README.md | 10 +- scripts/security/Test-PrValidationGate.ps1 | 269 ++++++++++++++++++ .../pr-validation-gate/complete-gate.yml | 36 +++ .../pr-validation-gate/missing-job.yml | 35 +++ .../pr-validation-gate/stale-needs.yml | 30 ++ .../security/Test-PrValidationGate.Tests.ps1 | 130 +++++++++ 7 files changed, 514 insertions(+), 5 deletions(-) create mode 100644 scripts/security/Test-PrValidationGate.ps1 create mode 100644 scripts/tests/fixtures/pr-validation-gate/complete-gate.yml create mode 100644 scripts/tests/fixtures/pr-validation-gate/missing-job.yml create mode 100644 scripts/tests/fixtures/pr-validation-gate/stale-needs.yml create mode 100644 scripts/tests/security/Test-PrValidationGate.Tests.ps1 diff --git a/.github/instructions/workflows.instructions.md b/.github/instructions/workflows.instructions.md index 453dc78a6..9de923a56 100644 --- a/.github/instructions/workflows.instructions.md +++ b/.github/instructions/workflows.instructions.md @@ -308,6 +308,14 @@ with: Avoid `format()` workarounds or environment variable indirection when the simpler options above apply. +## PR Validation Gate + +`pr-validation.yml` exposes a single `pr-validation-success` aggregator job as the required status check that gates merge. This job is green only when every other job in the workflow passes. + +Every job defined in `pr-validation.yml`, except `pr-validation-success` itself, MUST appear in the `needs:` list of the `pr-validation-success` job. Omitting a job lets that job fail silently without blocking merge, which defeats the gate. + +The `gate-completeness-check` job enforces this rule in CI, failing the workflow whenever the gate's `needs:` list drifts out of sync with the defined jobs. Contributors can validate the gate locally by running `npm run lint:pr-gate` before pushing. + ## Enforcement Statement The following scripts enforce compliance: @@ -316,5 +324,6 @@ The following scripts enforce compliance: * `scripts/security/Test-SHAStaleness.ps1` - Checks for stale dependencies * `scripts/security/Test-WorkflowPermissions.ps1` - Validates workflow permissions declarations * `scripts/linting/Invoke-YamlLint.ps1` - Runs actionlint validation +* `scripts/security/Test-PrValidationGate.ps1` - Validates the PR validation gate `needs:` completeness All workflows must pass these validation checks to be merged into the repository. diff --git a/.github/workflows/README.md b/.github/workflows/README.md index eeb98d958..25f71d0df 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -2,7 +2,7 @@ title: GitHub Actions Workflows description: Modular CI/CD workflow architecture for validation, security scanning, and automated maintenance author: HVE Core Team -ms.date: 2026-05-01 +ms.date: 2026-06-27 ms.topic: reference keywords: - github actions @@ -49,12 +49,12 @@ Compose multiple reusable workflows for comprehensive validation and security sc | Workflow | Triggers | Jobs | Mode | Purpose | |-----------------------------------|-----------------------------------------|-----------------------------------------------------------------|----------------------------|--------------------------------------| -| `pr-validation.yml` | PR to main/develop (open, push, reopen) | 9 jobs (8 reusable workflows + 1 inline) | Strict validation | Pre-merge quality gate with security | +| `pr-validation.yml` | PR to main/develop (open, push, reopen) | 31 jobs (29 validation jobs + `pr-validation-success` gate + `gate-completeness-check`); `pr-validation-success` is the merge signal | Strict validation | Pre-merge quality gate with security | | `release-stable.yml` | Push to main | 5 jobs (5 reusable workflows) | Strict mode, SARIF uploads | Post-merge validation | | `weekly-security-maintenance.yml` | Schedule (Sun 2AM UTC) | 4 (validate-pinning, check-staleness, codeql-analysis, summary) | Soft-fail warnings | Weekly security posture | | `scorecard.yml` | Push to main, Schedule (Sun 3AM UTC) | 1 (scorecard) | SARIF upload | OpenSSF Scorecard security posture | -pr-validation.yml jobs: codeql-analysis, spell-check, markdown-lint, table-format, psscriptanalyzer, frontmatter-validation, link-lang-check, markdown-link-check, dependency-pinning-check +pr-validation.yml jobs: 29 validation jobs feed a single `pr-validation-success` aggregator gate, which is the only required status check that gates merge; a `gate-completeness-check` job verifies every validation job is wired into that gate's `needs:` list. release-stable.yml jobs: spell-check, markdown-lint, table-format, codeql-analysis, dependency-pinning-scan @@ -250,8 +250,8 @@ Workflow Execution Matrix: | Event | Workflows That Run | CodeQL Included | |--------------------------------------|----------------------------------------------------------|---------------------| -| Open PR to main/develop | `pr-validation.yml` (9 jobs) | ✅ Yes | -| Push to PR branch | `pr-validation.yml` (9 jobs) | ✅ Yes | +| Open PR to main/develop | `pr-validation.yml` (31 jobs) | ✅ Yes | +| Push to PR branch | `pr-validation.yml` (31 jobs) | ✅ Yes | | Merge to main | `release-stable.yml` (5 jobs) | ✅ Yes | | Sunday 4AM UTC | `codeql-analysis.yml`, `weekly-security-maintenance.yml` | ✅ Yes (standalone) | | Feature branch push (no open PR)[^1] | None | ❌ No | diff --git a/scripts/security/Test-PrValidationGate.ps1 b/scripts/security/Test-PrValidationGate.ps1 new file mode 100644 index 000000000..8efa2026a --- /dev/null +++ b/scripts/security/Test-PrValidationGate.ps1 @@ -0,0 +1,269 @@ +#!/usr/bin/env pwsh +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT + +#Requires -Version 7.0 +#Requires -Modules @{ ModuleName='PowerShell-Yaml'; RequiredVersion='0.4.7' } + +<# +.SYNOPSIS + Validates that the PR-validation aggregator gate job depends on every other job. + +.DESCRIPTION + Parses a GitHub Actions workflow (default '.github/workflows/pr-validation.yml') + with ConvertFrom-Yaml, enumerates its top-level job IDs, and asserts that the + aggregator gate job (default 'pr-validation-success') lists every non-gate job + in its 'needs:' array. The check guards against two forms of drift: + + * Missing jobs - a job exists in the workflow but is absent from the gate's + 'needs:' list, so its failure would not block the gate. + * Stale needs - the gate's 'needs:' references a job ID that no longer + exists in the workflow. + + Results are emitted as a JSON object under logs/ and a human-readable summary + is written to the console. With -FailOnViolation, the script exits 1 and names + the offending jobs when any violation (or an absent gate job) is detected; + otherwise it exits 0. + + This validator deliberately parses YAML structurally and does not depend on the + regex-based security helper modules used by sibling validators. + +.PARAMETER WorkflowPath + Path to the workflow YAML file to validate. Defaults to + '.github/workflows/pr-validation.yml'. + +.PARAMETER GateJobId + Job ID of the aggregator gate that must depend on all other jobs. Defaults to + 'pr-validation-success'. + +.PARAMETER OutputPath + Path for the JSON results file. Defaults to + 'logs/pr-validation-gate-results.json'. + +.PARAMETER FailOnViolation + When set, exits with a non-zero code if any job is missing from the gate's + 'needs:' list, any 'needs:' entry is stale, or the gate job is absent. + +.EXAMPLE + ./scripts/security/Test-PrValidationGate.ps1 + +.EXAMPLE + ./scripts/security/Test-PrValidationGate.ps1 -FailOnViolation + +.NOTES + Part of the HVE Core security validation suite. + +.LINK + https://github.com/microsoft/hve-core +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$WorkflowPath = '.github/workflows/pr-validation.yml', + + [Parameter(Mandatory = $false)] + [string]$GateJobId = 'pr-validation-success', + + [Parameter(Mandatory = $false)] + [string]$OutputPath = 'logs/pr-validation-gate-results.json', + + [Parameter(Mandatory = $false)] + [switch]$FailOnViolation +) + +$ErrorActionPreference = 'Stop' + +Import-Module powershell-yaml -ErrorAction Stop + +#region Functions + +function Get-PrValidationGateResult { + <# + .SYNOPSIS + Computes gate-completeness results for a workflow. + + .DESCRIPTION + Parses the workflow YAML, enumerates job IDs, and returns an object + describing which jobs are missing from the gate's 'needs:' list and which + 'needs:' entries are stale. The gate job's presence is reported via the + GateJobPresent property so callers can surface a clear error. + + .PARAMETER WorkflowPath + Path to the workflow YAML file to parse. + + .PARAMETER GateJobId + Job ID of the aggregator gate. + + .OUTPUTS + [pscustomobject] with WorkflowPath, GateJobId, GateJobPresent, AllJobs, + GateNeeds, Missing, and Stale properties. + #> + [CmdletBinding()] + [OutputType([pscustomobject])] + param( + [Parameter(Mandatory = $true)] + [string]$WorkflowPath, + + [Parameter(Mandatory = $true)] + [string]$GateJobId + ) + + if (-not (Test-Path -Path $WorkflowPath)) { + throw "Workflow file not found: $WorkflowPath" + } + + $wf = Get-Content -Raw -Path $WorkflowPath | ConvertFrom-Yaml + + if ($null -eq $wf -or $null -eq $wf.jobs) { + throw "Workflow '$WorkflowPath' does not define a 'jobs:' map." + } + + $allJobs = @($wf.jobs.Keys) + $gateJob = $wf.jobs[$GateJobId] + $gatePresent = $null -ne $gateJob + + # Normalize both flow (needs: [a, b]) and block sequence forms to an array. + $gateNeeds = if ($gatePresent) { @($gateJob.needs) } else { @() } + + $expected = @($allJobs | Where-Object { $_ -ne $GateJobId }) + $missing = @($expected | Where-Object { $_ -notin $gateNeeds }) + $stale = @($gateNeeds | Where-Object { $_ -notin $allJobs }) + + return [pscustomobject]@{ + WorkflowPath = $WorkflowPath + GateJobId = $GateJobId + GateJobPresent = $gatePresent + AllJobs = $allJobs + GateNeeds = $gateNeeds + Missing = $missing + Stale = $stale + } +} + +function Invoke-PrValidationGateCheck { + <# + .SYNOPSIS + Orchestrates the PR-validation gate completeness check. + + .DESCRIPTION + Computes gate-completeness results, writes a JSON results object to the + output path, prints a human-readable summary, and returns an exit code. + + .PARAMETER WorkflowPath + Path to the workflow YAML file to validate. + + .PARAMETER GateJobId + Job ID of the aggregator gate. + + .PARAMETER OutputPath + Path for the JSON results file. + + .PARAMETER FailOnViolation + When set, returns 1 if any violation or an absent gate job is detected. + + .OUTPUTS + [int] Exit code: 0 when clean (or soft-fail mode), 1 on violations. + #> + [CmdletBinding()] + [OutputType([int])] + param( + [Parameter(Mandatory = $false)] + [string]$WorkflowPath = '.github/workflows/pr-validation.yml', + + [Parameter(Mandatory = $false)] + [string]$GateJobId = 'pr-validation-success', + + [Parameter(Mandatory = $false)] + [string]$OutputPath = 'logs/pr-validation-gate-results.json', + + [Parameter(Mandatory = $false)] + [switch]$FailOnViolation + ) + + Write-Host "🔍 Validating PR-validation gate completeness" -ForegroundColor Cyan + Write-Host " Workflow: $WorkflowPath" -ForegroundColor Gray + Write-Host " Gate job: $GateJobId" -ForegroundColor Gray + + $result = Get-PrValidationGateResult -WorkflowPath $WorkflowPath -GateJobId $GateJobId + + $violationCount = $result.Missing.Count + $result.Stale.Count + if (-not $result.GateJobPresent) { + $violationCount++ + } + + $resultObject = [ordered]@{ + workflowPath = $result.WorkflowPath + gateJobId = $result.GateJobId + gateJobPresent = $result.GateJobPresent + totalJobs = $result.AllJobs.Count + gateNeedsCount = $result.GateNeeds.Count + missing = $result.Missing + stale = $result.Stale + violationCount = $violationCount + timestamp = (Get-Date).ToUniversalTime().ToString('o') + } + + # Write JSON results to logs/. + $outputDir = [System.IO.Path]::GetDirectoryName($OutputPath) + if ($outputDir -and -not (Test-Path -Path $outputDir)) { + New-Item -ItemType Directory -Path $outputDir -Force | Out-Null + } + $resultObject | ConvertTo-Json -Depth 10 | Out-File -FilePath $OutputPath -Encoding utf8 -Force + Write-Host " Results written to: $OutputPath" -ForegroundColor Gray + + if (-not $result.GateJobPresent) { + Write-Host "❌ Gate job '$GateJobId' was not found in $WorkflowPath" -ForegroundColor Red + Write-Host " Add a '$GateJobId' job that depends on every other job." -ForegroundColor Red + return 1 + } + + if ($violationCount -eq 0) { + Write-Host "✅ Gate '$GateJobId' depends on all $($result.Missing.Count + $result.GateNeeds.Count) non-gate jobs." -ForegroundColor Green + return 0 + } + + if ($result.Missing.Count -gt 0) { + Write-Host "❌ Jobs missing from '$GateJobId' needs ($($result.Missing.Count)):" -ForegroundColor Red + foreach ($job in $result.Missing) { + Write-Host " - $job" -ForegroundColor Red + } + } + + if ($result.Stale.Count -gt 0) { + Write-Host "❌ Stale '$GateJobId' needs entries referencing missing jobs ($($result.Stale.Count)):" -ForegroundColor Red + foreach ($job in $result.Stale) { + Write-Host " - $job" -ForegroundColor Red + } + } + + if ($FailOnViolation) { + Write-Host "❌ $violationCount gate-completeness violation(s) found - failing." -ForegroundColor Red + return 1 + } + + Write-Host "⚠️ $violationCount gate-completeness violation(s) found - soft fail mode." -ForegroundColor Yellow + return 0 +} + +#endregion Functions + +#region Main Execution + +if ($MyInvocation.InvocationName -ne '.') { + try { + $exitCode = Invoke-PrValidationGateCheck ` + -WorkflowPath $WorkflowPath ` + -GateJobId $GateJobId ` + -OutputPath $OutputPath ` + -FailOnViolation:$FailOnViolation + exit $exitCode + } + catch { + Write-Host "❌ Fatal error: $($_.Exception.Message)" -ForegroundColor Red + Write-Host $_.ScriptStackTrace -ForegroundColor Red + exit 1 + } +} + +#endregion Main Execution diff --git a/scripts/tests/fixtures/pr-validation-gate/complete-gate.yml b/scripts/tests/fixtures/pr-validation-gate/complete-gate.yml new file mode 100644 index 000000000..196fed33b --- /dev/null +++ b/scripts/tests/fixtures/pr-validation-gate/complete-gate.yml @@ -0,0 +1,36 @@ +name: Complete Gate Fixture +on: + pull_request: + +permissions: + contents: read + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - run: echo lint + + test: + name: Test + runs-on: ubuntu-latest + steps: + - run: echo test + + build: + name: Build + runs-on: ubuntu-latest + steps: + - run: echo build + + pr-validation-success: + name: PR Validation Success + runs-on: ubuntu-latest + if: always() + needs: + - lint + - test + - build + steps: + - run: echo gate diff --git a/scripts/tests/fixtures/pr-validation-gate/missing-job.yml b/scripts/tests/fixtures/pr-validation-gate/missing-job.yml new file mode 100644 index 000000000..431c8f956 --- /dev/null +++ b/scripts/tests/fixtures/pr-validation-gate/missing-job.yml @@ -0,0 +1,35 @@ +name: Missing Job Fixture +on: + pull_request: + +permissions: + contents: read + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - run: echo lint + + test: + name: Test + runs-on: ubuntu-latest + steps: + - run: echo test + + build: + name: Build + runs-on: ubuntu-latest + steps: + - run: echo build + + pr-validation-success: + name: PR Validation Success + runs-on: ubuntu-latest + if: always() + needs: + - lint + - test + steps: + - run: echo gate diff --git a/scripts/tests/fixtures/pr-validation-gate/stale-needs.yml b/scripts/tests/fixtures/pr-validation-gate/stale-needs.yml new file mode 100644 index 000000000..e40128704 --- /dev/null +++ b/scripts/tests/fixtures/pr-validation-gate/stale-needs.yml @@ -0,0 +1,30 @@ +name: Stale Needs Fixture +on: + pull_request: + +permissions: + contents: read + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - run: echo lint + + test: + name: Test + runs-on: ubuntu-latest + steps: + - run: echo test + + pr-validation-success: + name: PR Validation Success + runs-on: ubuntu-latest + if: always() + needs: + - lint + - test + - deleted-job + steps: + - run: echo gate diff --git a/scripts/tests/security/Test-PrValidationGate.Tests.ps1 b/scripts/tests/security/Test-PrValidationGate.Tests.ps1 new file mode 100644 index 000000000..f63ba75c4 --- /dev/null +++ b/scripts/tests/security/Test-PrValidationGate.Tests.ps1 @@ -0,0 +1,130 @@ +#Requires -Modules Pester +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT + +BeforeAll { + . (Join-Path $PSScriptRoot '../../security/Test-PrValidationGate.ps1') + + $script:FixturesPath = Join-Path $PSScriptRoot '../fixtures/pr-validation-gate' + $script:CompleteGate = Join-Path $script:FixturesPath 'complete-gate.yml' + $script:MissingJob = Join-Path $script:FixturesPath 'missing-job.yml' + $script:StaleNeeds = Join-Path $script:FixturesPath 'stale-needs.yml' + + Mock Write-Host {} +} + +Describe 'Get-PrValidationGateResult' -Tag 'Unit' { + Context 'when the gate lists every non-gate job' { + BeforeAll { + $script:Result = Get-PrValidationGateResult -WorkflowPath $script:CompleteGate -GateJobId 'pr-validation-success' + } + + It 'Reports the gate job as present' { + $script:Result.GateJobPresent | Should -BeTrue + } + + It 'Reports no missing jobs' { + $script:Result.Missing | Should -BeNullOrEmpty + } + + It 'Reports no stale needs entries' { + $script:Result.Stale | Should -BeNullOrEmpty + } + } + + Context 'when a job is omitted from the gate needs' { + BeforeAll { + $script:Result = Get-PrValidationGateResult -WorkflowPath $script:MissingJob -GateJobId 'pr-validation-success' + } + + It 'Reports the omitted job as missing' { + $script:Result.Missing | Should -Contain 'build' + } + + It 'Reports no stale needs entries' { + $script:Result.Stale | Should -BeNullOrEmpty + } + } + + Context 'when a needs entry references a non-existent job' { + BeforeAll { + $script:Result = Get-PrValidationGateResult -WorkflowPath $script:StaleNeeds -GateJobId 'pr-validation-success' + } + + It 'Reports the stale needs entry' { + $script:Result.Stale | Should -Contain 'deleted-job' + } + + It 'Reports no missing jobs' { + $script:Result.Missing | Should -BeNullOrEmpty + } + } + + Context 'when the gate job is absent' { + BeforeAll { + $script:Result = Get-PrValidationGateResult -WorkflowPath $script:CompleteGate -GateJobId 'nonexistent-gate' + } + + It 'Reports the gate job as not present' { + $script:Result.GateJobPresent | Should -BeFalse + } + } +} + +Describe 'Invoke-PrValidationGateCheck' -Tag 'Unit' { + BeforeEach { + $script:OutputPath = Join-Path $TestDrive 'pr-validation-gate-results.json' + } + + Context 'when the gate is complete' { + It 'Returns exit code 0' { + $exitCode = Invoke-PrValidationGateCheck -WorkflowPath $script:CompleteGate -GateJobId 'pr-validation-success' -OutputPath $script:OutputPath -FailOnViolation + $exitCode | Should -Be 0 + } + + It 'Writes a JSON results file' { + Invoke-PrValidationGateCheck -WorkflowPath $script:CompleteGate -GateJobId 'pr-validation-success' -OutputPath $script:OutputPath | Out-Null + Test-Path -Path $script:OutputPath | Should -BeTrue + $json = Get-Content -Raw -Path $script:OutputPath | ConvertFrom-Json + $json.violationCount | Should -Be 0 + } + } + + Context 'when a job is missing from the gate needs' { + It 'Returns exit code 1 under FailOnViolation' { + $exitCode = Invoke-PrValidationGateCheck -WorkflowPath $script:MissingJob -GateJobId 'pr-validation-success' -OutputPath $script:OutputPath -FailOnViolation + $exitCode | Should -Be 1 + } + + It 'Returns exit code 0 in soft-fail mode' { + $exitCode = Invoke-PrValidationGateCheck -WorkflowPath $script:MissingJob -GateJobId 'pr-validation-success' -OutputPath $script:OutputPath + $exitCode | Should -Be 0 + } + + It 'Records the missing job in the JSON results' { + Invoke-PrValidationGateCheck -WorkflowPath $script:MissingJob -GateJobId 'pr-validation-success' -OutputPath $script:OutputPath | Out-Null + $json = Get-Content -Raw -Path $script:OutputPath | ConvertFrom-Json + $json.missing | Should -Contain 'build' + } + } + + Context 'when a needs entry is stale' { + It 'Returns exit code 1 under FailOnViolation' { + $exitCode = Invoke-PrValidationGateCheck -WorkflowPath $script:StaleNeeds -GateJobId 'pr-validation-success' -OutputPath $script:OutputPath -FailOnViolation + $exitCode | Should -Be 1 + } + + It 'Records the stale entry in the JSON results' { + Invoke-PrValidationGateCheck -WorkflowPath $script:StaleNeeds -GateJobId 'pr-validation-success' -OutputPath $script:OutputPath | Out-Null + $json = Get-Content -Raw -Path $script:OutputPath | ConvertFrom-Json + $json.stale | Should -Contain 'deleted-job' + } + } + + Context 'when the gate job is absent' { + It 'Returns exit code 1 even without FailOnViolation' { + $exitCode = Invoke-PrValidationGateCheck -WorkflowPath $script:CompleteGate -GateJobId 'nonexistent-gate' -OutputPath $script:OutputPath + $exitCode | Should -Be 1 + } + } +} From 9a0bc9398257b24da8ef83dfaad611f81be47b15 Mon Sep 17 00:00:00 2001 From: jkim323 Date: Sat, 27 Jun 2026 23:57:07 -0700 Subject: [PATCH 03/11] add gate completeness check --- .github/workflows/pr-validation.yml | 69 +++++++++++++++++++++++++++++ package.json | 3 +- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 341804e41..eddd3fce8 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -366,3 +366,72 @@ jobs: security-events: write # Required for SARIF upload to Security tab actions: read + gate-completeness-check: + name: PR Gate Completeness + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Setup PowerShell modules + uses: ./.github/actions/setup-ps-modules + + - name: Validate PR gate completeness + shell: pwsh + run: ./scripts/security/Test-PrValidationGate.ps1 -FailOnViolation + + pr-validation-success: + name: PR Validation Success + runs-on: ubuntu-latest + permissions: + contents: read + if: always() + needs: + - spell-check + - markdown-lint + - table-format + - psscriptanalyzer + - discover-python-projects + - python-lint + - copyright-headers + - yaml-lint + - pester-tests + - pytest + - fuzz-tests + - pip-audit + - docusaurus-tests + - frontmatter-validation + - adr-consistency-validation + - ai-artifact-validation + - msdate-freshness + - plugin-validation + - skill-validation + - eval-validation + - link-lang-check + - markdown-link-check + - dependency-pinning-check + - devcontainer-lockfile-check + - workflow-permissions-check + - action-version-consistency-scan + - gitleaks-scan + - npm-audit + - codeql + - gate-completeness-check + steps: + - name: Verify all jobs succeeded + env: + NEEDS_JSON: ${{ toJSON(needs) }} + shell: bash + run: | + failed=$(echo "$NEEDS_JSON" | jq -r 'to_entries[] | select(.value.result == "failure" or .value.result == "cancelled") | .key') + if [ -n "$failed" ]; then + echo "The following jobs did not pass:" + echo "$failed" + exit 1 + fi + echo "All PR validation jobs passed." + diff --git a/package.json b/package.json index 4730839f8..c04c6075e 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "lint:version-consistency": "pwsh -NoProfile -Command \"./scripts/security/Test-ActionVersionConsistency.ps1 -FailOnMismatch -Format Json -OutputPath logs/action-version-consistency-results.json\"", "lint:permissions": "pwsh -NoProfile -Command \"& './scripts/security/Test-WorkflowPermissions.ps1' -FailOnViolation\"", "lint:dependency-pinning": "pwsh -NoProfile -Command \"& './scripts/security/Test-DependencyPinning.ps1' -FailOnUnpinned\"", + "lint:pr-gate": "pwsh -NoProfile -Command \"& './scripts/security/Test-PrValidationGate.ps1' -FailOnViolation\"", "lint:ps-module-pins": "pwsh -NoProfile -File scripts/security/Test-PSModulePins.ps1", "audit:pip": "pwsh -NoProfile -File scripts/security/Invoke-PipAudit.ps1", "audit:npm": "audit-ci --config audit-ci.json", @@ -34,7 +35,7 @@ "lint:ai-artifacts": "pwsh -NoProfile -Command \"& './scripts/linting/Validate-PlannerArtifacts.ps1' -FailOnMissing\"", "lint:models": "pwsh -NoProfile -File scripts/linting/Test-ModelReferences.ps1 -OutputPath logs/model-validation-results.json", "lint:models:refresh": "pwsh -NoProfile -File scripts/linting/Update-ModelCatalog.ps1", - "lint:all": "npm run format:tables && npm run lint:md && npm run lint:ps && npm run lint:yaml && npm run lint:json && npm run lint:links && npm run lint:frontmatter && npm run lint:adr-consistency && npm run lint:collections-metadata && npm run lint:marketplace && npm run lint:hooks && npm run lint:version-consistency && npm run lint:permissions && npm run lint:dependency-pinning && npm run lint:ps-module-pins && npm run lint:py && npm run validate:skills && npm run lint:ai-artifacts && npm run lint:models && npm run validate:devcontainer-lockfile && npm run eval:lint:vally && npm run eval:lint:schema && npm run eval:lint:text && npm run eval:lint:safety", + "lint:all": "npm run format:tables && npm run lint:md && npm run lint:ps && npm run lint:yaml && npm run lint:json && npm run lint:links && npm run lint:frontmatter && npm run lint:adr-consistency && npm run lint:collections-metadata && npm run lint:marketplace && npm run lint:hooks && npm run lint:version-consistency && npm run lint:permissions && npm run lint:dependency-pinning && npm run lint:pr-gate && npm run lint:ps-module-pins && npm run lint:py && npm run validate:skills && npm run lint:ai-artifacts && npm run lint:models && npm run validate:devcontainer-lockfile && npm run eval:lint:vally && npm run eval:lint:schema && npm run eval:lint:text && npm run eval:lint:safety", "format:tables": "pwsh -NoProfile -File scripts/linting/Format-MarkdownTables.ps1", "extension:prepare": "pwsh ./scripts/extension/Prepare-Extension.ps1 && npm run extension:postprocess", "extension:prepare:prerelease": "pwsh ./scripts/extension/Prepare-Extension.ps1 -Channel PreRelease && npm run extension:postprocess", From 8e33197cf5e0688fcdafc5833743758ffa4c86b3 Mon Sep 17 00:00:00 2001 From: jkim323 Date: Sun, 28 Jun 2026 00:18:55 -0700 Subject: [PATCH 04/11] nit --- .github/workflows/README.md | 10 +++++----- docs/agents/project-planning/README.md | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 25f71d0df..bcdee8ca8 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -47,12 +47,12 @@ Modular reusable workflows following Single Responsibility Principle. Each workf Compose multiple reusable workflows for comprehensive validation and security scanning. -| Workflow | Triggers | Jobs | Mode | Purpose | -|-----------------------------------|-----------------------------------------|-----------------------------------------------------------------|----------------------------|--------------------------------------| +| Workflow | Triggers | Jobs | Mode | Purpose | +|-----------------------------------|-----------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------|----------------------------|--------------------------------------| | `pr-validation.yml` | PR to main/develop (open, push, reopen) | 31 jobs (29 validation jobs + `pr-validation-success` gate + `gate-completeness-check`); `pr-validation-success` is the merge signal | Strict validation | Pre-merge quality gate with security | -| `release-stable.yml` | Push to main | 5 jobs (5 reusable workflows) | Strict mode, SARIF uploads | Post-merge validation | -| `weekly-security-maintenance.yml` | Schedule (Sun 2AM UTC) | 4 (validate-pinning, check-staleness, codeql-analysis, summary) | Soft-fail warnings | Weekly security posture | -| `scorecard.yml` | Push to main, Schedule (Sun 3AM UTC) | 1 (scorecard) | SARIF upload | OpenSSF Scorecard security posture | +| `release-stable.yml` | Push to main | 5 jobs (5 reusable workflows) | Strict mode, SARIF uploads | Post-merge validation | +| `weekly-security-maintenance.yml` | Schedule (Sun 2AM UTC) | 4 (validate-pinning, check-staleness, codeql-analysis, summary) | Soft-fail warnings | Weekly security posture | +| `scorecard.yml` | Push to main, Schedule (Sun 3AM UTC) | 1 (scorecard) | SARIF upload | OpenSSF Scorecard security posture | pr-validation.yml jobs: 29 validation jobs feed a single `pr-validation-success` aggregator gate, which is the only required status check that gates merge; a `gate-completeness-check` job verifies every validation job is wired into that gate's `needs:` list. diff --git a/docs/agents/project-planning/README.md b/docs/agents/project-planning/README.md index 90ab1646f..0d27ac0ac 100644 --- a/docs/agents/project-planning/README.md +++ b/docs/agents/project-planning/README.md @@ -24,11 +24,11 @@ These agents bring structure and consistency to activities that teams often hand ## Agent Overview -| Agent | Sub-Category | Workflow | Persistence | Key Output | -|----------------------------------------|--------------|-----------------------------|-------------|--------------------------------| -| [BRD Builder](brd-prd-builders) | Requirements | 3-phase Q&A | JSON state | Business requirements document | -| [PRD Builder](brd-prd-builders) | Requirements | 7-phase Q&A | JSON state | Product requirements document | -| [ADR Creator](adr-creation) | Architecture | 3-phase Frame/Decide/Govern | JSON state | Architecture decision record | +| Agent | Sub-Category | Workflow | Persistence | Key Output | +|-------------------------------------------|--------------|-----------------------------|-------------|--------------------------------| +| [BRD Builder](brd-prd-builders) | Requirements | 3-phase Q&A | JSON state | Business requirements document | +| [PRD Builder](brd-prd-builders) | Requirements | 7-phase Q&A | JSON state | Product requirements document | +| [ADR Creator](adr-creation) | Architecture | 3-phase Frame/Decide/Govern | JSON state | Architecture decision record | | [Security Planner](../security/README.md) | Security | 6-phase STRIDE | JSON state | Security model and backlog | ## Requirements From a3c6733c1834f14e5db850f197619728ff5fdb36 Mon Sep 17 00:00:00 2001 From: jkim323 Date: Sun, 28 Jun 2026 10:23:44 -0700 Subject: [PATCH 05/11] skip repo-root artifacts in stimulus presence check --- scripts/evals/Modules/ArtifactDetection.psm1 | 57 ++++++++++++++- scripts/evals/Test-StimulusPresence.ps1 | 6 ++ .../evals/Test-StimulusPresence.Tests.ps1 | 70 +++++++++++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) diff --git a/scripts/evals/Modules/ArtifactDetection.psm1 b/scripts/evals/Modules/ArtifactDetection.psm1 index 5e1e93c79..760135e6b 100644 --- a/scripts/evals/Modules/ArtifactDetection.psm1 +++ b/scripts/evals/Modules/ArtifactDetection.psm1 @@ -30,6 +30,17 @@ $script:ArtifactPatterns = @( } ) +# Repo-root-only artifact patterns: files placed directly under `.github//` +# (skills: `.github/skills//SKILL.md`) without a collection subdirectory. +# Per `.github/copilot-instructions.md`, these are repo-specific and excluded from +# collection manifests, packaging, and eval coverage enforcement. +$script:RepoRootArtifactPatterns = @{ + agent = '^\.github/agents/[^/]+\.agent\.md$' + prompt = '^\.github/prompts/[^/]+\.prompt\.md$' + instruction = '^\.github/instructions/[^/]+\.instructions\.md$' + skill = '^\.github/skills/[^/]+/SKILL\.md$' +} + function ConvertTo-NormalizedArtifactPath { <# .SYNOPSIS @@ -198,9 +209,53 @@ function Get-ChangedArtifactRecord { } } +function Test-RepoRootArtifact { + <# + .SYNOPSIS + Determines whether an artifact path is a repo-root (repo-specific) artifact. + + .DESCRIPTION + Repo-root artifacts live directly under `.github//` without a collection + subdirectory (skills: `.github/skills//SKILL.md`). Per + `.github/copilot-instructions.md`, these are repo-specific and excluded from + collection manifests, packaging, and eval coverage enforcement. + + .PARAMETER Kind + Artifact kind: agent, prompt, instruction, or skill. + + .PARAMETER Path + Workspace-relative path (forward or backslash separators accepted). + + .OUTPUTS + [bool] True when the path is a repo-root artifact of the given kind. + #> + [CmdletBinding()] + [OutputType([bool])] + param( + [Parameter(Mandatory = $true)] + [string]$Kind, + + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string]$Path + ) + + if (-not $script:RepoRootArtifactPatterns.ContainsKey($Kind)) { + return $false + } + + $normalized = ConvertTo-NormalizedArtifactPath -Path $Path + if ([string]::IsNullOrEmpty($normalized)) { + return $false + } + + return [regex]::IsMatch($normalized, $script:RepoRootArtifactPatterns[$Kind]) +} + Export-ModuleMember -Function @( 'Get-ArtifactDescriptor', 'ConvertFrom-GitDiffNameStatus', 'Get-ChangedArtifactRecord', - 'ConvertTo-NormalizedArtifactPath' + 'ConvertTo-NormalizedArtifactPath', + 'Test-RepoRootArtifact' ) diff --git a/scripts/evals/Test-StimulusPresence.ps1 b/scripts/evals/Test-StimulusPresence.ps1 index 1d2c9c85c..1680d93e7 100644 --- a/scripts/evals/Test-StimulusPresence.ps1 +++ b/scripts/evals/Test-StimulusPresence.ps1 @@ -78,6 +78,7 @@ Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' Import-Module (Join-Path $PSScriptRoot 'Modules/StimulusIndex.psm1') -Force +Import-Module (Join-Path $PSScriptRoot 'Modules/ArtifactDetection.psm1') -Force if (-not (Get-Module -ListAvailable -Name 'powershell-yaml')) { Write-Error "Test-StimulusPresence.ps1 requires the 'powershell-yaml' module." @@ -237,6 +238,11 @@ function Invoke-StimulusPresenceCheck { continue } + if (Test-RepoRootArtifact -Kind $kind -Path $path) { + $skipped.Add(@{ kind = $kind; artifactId = $artifactId; path = $path; reason = 'repo-specific' }) + continue + } + $specs = Test-StimulusCoverage -Index $index -Kind $kind -ArtifactId $artifactId if ($specs.Count -gt 0) { $covered.Add(@{ kind = $kind; artifactId = $artifactId; path = $path; specs = $specs }) diff --git a/scripts/tests/evals/Test-StimulusPresence.Tests.ps1 b/scripts/tests/evals/Test-StimulusPresence.Tests.ps1 index 3ac945d73..381eb023f 100644 --- a/scripts/tests/evals/Test-StimulusPresence.Tests.ps1 +++ b/scripts/tests/evals/Test-StimulusPresence.Tests.ps1 @@ -4,9 +4,11 @@ BeforeAll { $script:ModulePath = Join-Path $PSScriptRoot '../../evals/Modules/StimulusIndex.psm1' + $script:ArtifactModulePath = Join-Path $PSScriptRoot '../../evals/Modules/ArtifactDetection.psm1' $script:ScriptPath = Join-Path $PSScriptRoot '../../evals/Test-StimulusPresence.ps1' Import-Module $script:ModulePath -Force + Import-Module $script:ArtifactModulePath -Force if (-not (Get-Module -ListAvailable -Name 'powershell-yaml')) { throw "Tests require the 'powershell-yaml' module to be installed." } @@ -118,6 +120,38 @@ stimuli: } } +Describe 'ArtifactDetection Test-RepoRootArtifact' -Tag 'Unit' { + It 'Returns true for repo-root artifacts' -ForEach @( + @{ Kind = 'agent'; Path = '.github/agents/example.agent.md' } + @{ Kind = 'prompt'; Path = '.github/prompts/example.prompt.md' } + @{ Kind = 'instruction'; Path = '.github/instructions/workflows.instructions.md' } + @{ Kind = 'skill'; Path = '.github/skills/example/SKILL.md' } + ) { + Test-RepoRootArtifact -Kind $Kind -Path $Path | Should -BeTrue + } + + It 'Returns false for collection-scoped artifacts' -ForEach @( + @{ Kind = 'agent'; Path = '.github/agents/hve-core/task-research.agent.md' } + @{ Kind = 'prompt'; Path = '.github/prompts/hve-core/example.prompt.md' } + @{ Kind = 'instruction'; Path = '.github/instructions/coding-standards/powershell/powershell.instructions.md' } + @{ Kind = 'skill'; Path = '.github/skills/shared/pr-reference/SKILL.md' } + ) { + Test-RepoRootArtifact -Kind $Kind -Path $Path | Should -BeFalse + } + + It 'Normalizes backslash separators' { + Test-RepoRootArtifact -Kind 'instruction' -Path '.github\instructions\workflows.instructions.md' | Should -BeTrue + } + + It 'Returns false for unknown kinds' { + Test-RepoRootArtifact -Kind 'unknown' -Path '.github/instructions/workflows.instructions.md' | Should -BeFalse + } + + It 'Returns false for empty paths' { + Test-RepoRootArtifact -Kind 'instruction' -Path '' | Should -BeFalse + } +} + Describe 'Test-StimulusPresence.ps1 entry script' -Tag 'Integration' { BeforeAll { function New-PresenceFixture { @@ -235,6 +269,42 @@ stimuli: $report.skipped[0].reason | Should -Be 'deleted' } + It 'Skips repo-root artifacts without requiring coverage' { + $artifacts = @( + @{ kind = 'instruction'; artifactId = 'workflows'; path = '.github/instructions/workflows.instructions.md'; status = 'M' } + ) + $spec = "name: empty`nstimuli: []" + $fx = New-PresenceFixture -Artifacts $artifacts -SpecYaml @($spec) + + & pwsh -NoProfile -File $script:ScriptPath ` + -ManifestPath $fx.ManifestPath -EvalRoot $fx.EvalRoot -OutFile $fx.OutFile ` + -RepoRoot $fx.Dir *> $null + $LASTEXITCODE | Should -Be 0 + + $report = Get-Content -LiteralPath $fx.OutFile -Raw | ConvertFrom-Json + $report.skipped.Count | Should -Be 1 + $report.missing.Count | Should -Be 0 + $report.skipped[0].reason | Should -Be 'repo-specific' + $report.skipped[0].artifactId | Should -Be 'workflows' + } + + It 'Still requires coverage for collection-scoped artifacts' { + $artifacts = @( + @{ kind = 'instruction'; artifactId = 'powershell'; path = '.github/instructions/coding-standards/powershell/powershell.instructions.md'; status = 'M' } + ) + $spec = "name: empty`nstimuli: []" + $fx = New-PresenceFixture -Artifacts $artifacts -SpecYaml @($spec) + + & pwsh -NoProfile -File $script:ScriptPath ` + -ManifestPath $fx.ManifestPath -EvalRoot $fx.EvalRoot -OutFile $fx.OutFile ` + -RepoRoot $fx.Dir *> $null + $LASTEXITCODE | Should -Be 1 + + $report = Get-Content -LiteralPath $fx.OutFile -Raw | ConvertFrom-Json + $report.missing.Count | Should -Be 1 + $report.missing[0].artifactId | Should -Be 'powershell' + } + It 'Exits 2 when the manifest does not exist' { $missing = Join-Path $TestDrive ('nope-' + [Guid]::NewGuid() + '.json') $evalRoot = Join-Path $TestDrive ('evals-' + [Guid]::NewGuid()) From 196af7664f718c719037fc802fb125d956fc9f10 Mon Sep 17 00:00:00 2001 From: jkim323 Date: Sun, 28 Jun 2026 10:46:32 -0700 Subject: [PATCH 06/11] skip repo-root artifacts in eval execute path --- scripts/evals/Invoke-VallyEvals.ps1 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/evals/Invoke-VallyEvals.ps1 b/scripts/evals/Invoke-VallyEvals.ps1 index c74810cee..fd0b653c5 100644 --- a/scripts/evals/Invoke-VallyEvals.ps1 +++ b/scripts/evals/Invoke-VallyEvals.ps1 @@ -116,6 +116,7 @@ $ErrorActionPreference = 'Stop' Import-Module (Join-Path $PSScriptRoot 'Modules/StimulusIndex.psm1') -Force Import-Module (Join-Path $PSScriptRoot 'Modules/VallyRunner.psm1') -Force +Import-Module (Join-Path $PSScriptRoot 'Modules/ArtifactDetection.psm1') -Force if (-not (Get-Module -Name powershell-yaml)) { Import-Module powershell-yaml -ErrorAction Stop @@ -369,6 +370,15 @@ if ($null -ne $manifest -and $null -ne $manifest.artifacts) { $artifacts = @($manifest.artifacts | Where-Object { [string]$_.status -ne 'D' }) } +# Repo-root (repo-specific) artifacts live directly under `.github//` +# without a collection subdirectory. Per `.github/copilot-instructions.md` they +# are excluded from collection manifests, packaging, and eval coverage, so they +# carry no eval spec. Drop them here so they do not surface as missing-coverage +# failures, mirroring the skip in `Test-StimulusPresence.ps1`. +$artifacts = @($artifacts | Where-Object { + -not (Test-RepoRootArtifact -Kind ([string]$_.kind) -Path ([string]$_.path)) +}) + # Per-kind shard filter. When -Kind is supplied, the stimulus artifacts[] loop # is narrowed to the matching kind(s) only. Baseline equivalence is cross-kind # (an instruction/skill change can promote a parent agent), so it stays owned by From a26b7a969dde4a8b82aba8e3bd032cf29f83b38b Mon Sep 17 00:00:00 2001 From: Jamie Kim Date: Sun, 28 Jun 2026 16:38:27 -0700 Subject: [PATCH 07/11] fix gate bypass bug --- .github/workflows/pr-validation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index eddd3fce8..225d4de6e 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -427,7 +427,8 @@ jobs: NEEDS_JSON: ${{ toJSON(needs) }} shell: bash run: | - failed=$(echo "$NEEDS_JSON" | jq -r 'to_entries[] | select(.value.result == "failure" or .value.result == "cancelled") | .key') + set -euo pipefail + failed=$(echo "$NEEDS_JSON" | jq -r 'to_entries[] | select(.value.result != "success" and .value.result != "skipped") | .key') if [ -n "$failed" ]; then echo "The following jobs did not pass:" echo "$failed" From 1c5bffa94fa79691d5e6da665f6e3bae78b2e056 Mon Sep 17 00:00:00 2001 From: Jamie Kim Date: Sun, 28 Jun 2026 16:39:38 -0700 Subject: [PATCH 08/11] remove BOM --- scripts/security/Test-PrValidationGate.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/security/Test-PrValidationGate.ps1 b/scripts/security/Test-PrValidationGate.ps1 index 8efa2026a..4712c1879 100644 --- a/scripts/security/Test-PrValidationGate.ps1 +++ b/scripts/security/Test-PrValidationGate.ps1 @@ -1,4 +1,4 @@ -#!/usr/bin/env pwsh +#!/usr/bin/env pwsh # Copyright (c) 2026 Microsoft Corporation. All rights reserved. # SPDX-License-Identifier: MIT From 56b6ed990b5550ac9bd0b7c82b9b9be70d5a1840 Mon Sep 17 00:00:00 2001 From: Jamie Kim Date: Sun, 28 Jun 2026 16:45:30 -0700 Subject: [PATCH 09/11] fix Phantom-stale guard (@($null) --- scripts/security/Test-PrValidationGate.ps1 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/security/Test-PrValidationGate.ps1 b/scripts/security/Test-PrValidationGate.ps1 index 4712c1879..589a83e5d 100644 --- a/scripts/security/Test-PrValidationGate.ps1 +++ b/scripts/security/Test-PrValidationGate.ps1 @@ -123,8 +123,12 @@ function Get-PrValidationGateResult { $gateJob = $wf.jobs[$GateJobId] $gatePresent = $null -ne $gateJob - # Normalize both flow (needs: [a, b]) and block sequence forms to an array. - $gateNeeds = if ($gatePresent) { @($gateJob.needs) } else { @() } + # Normalize both flow (needs: [a, b]) and block sequence forms to an array, then + # drop null/empty elements so a stray YAML null or "" entry cannot inject a phantom stale. + $gateNeeds = if ($gatePresent) { + @($gateJob.needs | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) + } + else { @() } $expected = @($allJobs | Where-Object { $_ -ne $GateJobId }) $missing = @($expected | Where-Object { $_ -notin $gateNeeds }) From f42d3e9667b4a44d7fa57403845a77d465b2a924 Mon Sep 17 00:00:00 2001 From: Jamie Kim Date: Sun, 28 Jun 2026 16:51:04 -0700 Subject: [PATCH 10/11] harden pr-validation gate completeness check --- .../pr-validation-gate/empty-needs.yml | 38 +++++++++++++++++++ .../security/Test-PrValidationGate.Tests.ps1 | 21 ++++++++++ 2 files changed, 59 insertions(+) create mode 100644 scripts/tests/fixtures/pr-validation-gate/empty-needs.yml diff --git a/scripts/tests/fixtures/pr-validation-gate/empty-needs.yml b/scripts/tests/fixtures/pr-validation-gate/empty-needs.yml new file mode 100644 index 000000000..b7c534fe0 --- /dev/null +++ b/scripts/tests/fixtures/pr-validation-gate/empty-needs.yml @@ -0,0 +1,38 @@ +name: Empty Needs Fixture +on: + pull_request: + +permissions: + contents: read + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - run: echo lint + + test: + name: Test + runs-on: ubuntu-latest + steps: + - run: echo test + + build: + name: Build + runs-on: ubuntu-latest + steps: + - run: echo build + + pr-validation-success: + name: PR Validation Success + runs-on: ubuntu-latest + if: always() + needs: + - lint + - test + - ~ + - "" + - build + steps: + - run: echo gate diff --git a/scripts/tests/security/Test-PrValidationGate.Tests.ps1 b/scripts/tests/security/Test-PrValidationGate.Tests.ps1 index f63ba75c4..d3191acea 100644 --- a/scripts/tests/security/Test-PrValidationGate.Tests.ps1 +++ b/scripts/tests/security/Test-PrValidationGate.Tests.ps1 @@ -9,6 +9,7 @@ BeforeAll { $script:CompleteGate = Join-Path $script:FixturesPath 'complete-gate.yml' $script:MissingJob = Join-Path $script:FixturesPath 'missing-job.yml' $script:StaleNeeds = Join-Path $script:FixturesPath 'stale-needs.yml' + $script:EmptyNeeds = Join-Path $script:FixturesPath 'empty-needs.yml' Mock Write-Host {} } @@ -69,6 +70,26 @@ Describe 'Get-PrValidationGateResult' -Tag 'Unit' { $script:Result.GateJobPresent | Should -BeFalse } } + + Context 'when needs contains null or empty elements' { + BeforeAll { + $script:Result = Get-PrValidationGateResult -WorkflowPath $script:EmptyNeeds -GateJobId 'pr-validation-success' + } + + It 'Filters null/empty entries out of GateNeeds' { + $script:Result.GateNeeds | Should -Not -Contain '' + $script:Result.GateNeeds | Should -Not -Contain $null + $script:Result.GateNeeds.Count | Should -Be 3 + } + + It 'Reports no stale needs entries' { + $script:Result.Stale | Should -BeNullOrEmpty + } + + It 'Reports no missing jobs' { + $script:Result.Missing | Should -BeNullOrEmpty + } + } } Describe 'Invoke-PrValidationGateCheck' -Tag 'Unit' { From 461192f7a2a4f00628bcd8a7e0bfa2bc90aa70e5 Mon Sep 17 00:00:00 2001 From: Jamie Kim Date: Sun, 28 Jun 2026 16:56:52 -0700 Subject: [PATCH 11/11] fix lint -> restore BOM --- scripts/security/Test-PrValidationGate.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/security/Test-PrValidationGate.ps1 b/scripts/security/Test-PrValidationGate.ps1 index 589a83e5d..fc6cf2646 100644 --- a/scripts/security/Test-PrValidationGate.ps1 +++ b/scripts/security/Test-PrValidationGate.ps1 @@ -1,4 +1,4 @@ -#!/usr/bin/env pwsh +#!/usr/bin/env pwsh # Copyright (c) 2026 Microsoft Corporation. All rights reserved. # SPDX-License-Identifier: MIT