From 78c9fc78d37b612088e3671f2a3651b54c5123ed Mon Sep 17 00:00:00 2001 From: Guillaume Tauzin <4648633+gtauzin@users.noreply.github.com> Date: Sun, 26 Jul 2026 11:05:31 +0200 Subject: [PATCH 1/2] fix(ci): pin ossf/scorecard-action to a real tag (v2.4.4) The Scorecard workflow shipped with ossf/scorecard-action@v2, but that action publishes no floating vN tag -- only vX.Y.Z. Every generated public repo's Scorecard run failed at load with 'unable to find version v2'. Pin to v2.4.4 (Renovate keeps it current). Also add test_expected_action_pins_actually_resolve_on_github (slow, network-gated): the existing pin test only checks self-consistency + the EXPECTED_ACTION_PINS map, so a non-existent tag passed every test and failed only at runtime. The new test resolves each pinned ref against GitHub so this cannot recur silently. --- ... 'public' %}scorecard.yml{% endif %}.jinja | 2 +- tests/test_template.py | 41 ++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/template/.github/{% if include_actions %}workflows{% endif %}/{% if repo_visibility == 'public' %}scorecard.yml{% endif %}.jinja b/template/.github/{% if include_actions %}workflows{% endif %}/{% if repo_visibility == 'public' %}scorecard.yml{% endif %}.jinja index d7801b9..e4bac50 100644 --- a/template/.github/{% if include_actions %}workflows{% endif %}/{% if repo_visibility == 'public' %}scorecard.yml{% endif %}.jinja +++ b/template/.github/{% if include_actions %}workflows{% endif %}/{% if repo_visibility == 'public' %}scorecard.yml{% endif %}.jinja @@ -27,7 +27,7 @@ jobs: persist-credentials: false - name: Run analysis - uses: ossf/scorecard-action@v2 + uses: ossf/scorecard-action@v2.4.4 with: results_file: results.sarif results_format: sarif diff --git a/tests/test_template.py b/tests/test_template.py index 6f2cf2c..8899dd8 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -4184,7 +4184,7 @@ def test_docs_use_no_em_or_en_dashes(copie_session_default): "github/codeql-action/analyze": "v3", "github/codeql-action/init": "v3", "github/codeql-action/upload-sarif": "v3", - "ossf/scorecard-action": "v2", + "ossf/scorecard-action": "v2.4.4", "peter-evans/create-pull-request": "v8", "pypa/gh-action-pypi-publish": "v1.13.0", "taiki-e/install-action": "v2", @@ -4234,6 +4234,45 @@ def test_action_pins_are_consistent_and_current(copie_session_default): assert not wrong, f"pins disagree with what the fleet runs (action: template -> expected): {wrong}" +@pytest.mark.slow +def test_expected_action_pins_actually_resolve_on_github(): + """Every pin in EXPECTED_ACTION_PINS resolves to a real ref on GitHub. + + ``test_action_pins_are_consistent_and_current`` only checks the template's pins are + self-consistent and match this map -- it cannot tell whether a tag *exists*. That + gap shipped ``ossf/scorecard-action@v2`` (no such tag; the action publishes only + ``vX.Y.Z`` tags), which passed every test and then failed at runtime on every + generated repo with "unable to find version v2". This closes it by resolving each + pinned ref against GitHub. + + Network-gated: skips cleanly when ``gh`` or the network is unavailable, so it is a + CI safety net (CI has an authenticated ``gh`` and no rate limit) rather than an + offline tripwire. Marked ``slow`` so it stays out of the fast feedback loop. + """ + gh = shutil.which("gh") + if gh is None: + pytest.skip("gh CLI not available") + unresolved = {} + for action, ref in EXPECTED_ACTION_PINS.items(): + repo = "/".join(action.split("/")[:2]) # github/codeql-action/init -> github/codeql-action + result = subprocess.run( + [gh, "api", f"repos/{repo}/commits/{ref}", "--jq", ".sha"], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if result.returncode != 0: + err = result.stderr.lower() + if any(s in err for s in ("rate limit", "could not resolve host", "timeout", "connection")): + pytest.skip(f"network/gh unavailable: {result.stderr.strip()[:120]}") + unresolved[f"{action}@{ref}"] = result.stderr.strip()[:120] + assert not unresolved, ( + f"these pinned action refs do not resolve on GitHub (a pin that does not exist passes the " + f"consistency check but fails at runtime on every generated repo): {unresolved}" + ) + + def test_nav_order_is_diataxis_with_reference_last(copie_session_default, copie_minimal): """The nav reads learn -> do -> see -> understand -> look up, in that order. From ec3433194c58fbaede87a8c9272ab5bed5a07b2c Mon Sep 17 00:00:00 2001 From: Guillaume Tauzin <4648633+gtauzin@users.noreply.github.com> Date: Sun, 26 Jul 2026 11:17:22 +0200 Subject: [PATCH 2/2] test: make the pin-existence check skip when gh can't authenticate The new test failed in the full-suite CI job: that env has no GH_TOKEN, so every 'gh api' call returns an auth error, which the skip logic misread as 'pin does not resolve'. Fix: fail ONLY on a definitive 404/not-found; skip when a ref cannot be checked (unauthenticated gh, rate limit, or offline). Verified: passes locally with an authenticated gh, skips cleanly with none. --- tests/test_template.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/tests/test_template.py b/tests/test_template.py index 8899dd8..094d400 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -4252,7 +4252,8 @@ def test_expected_action_pins_actually_resolve_on_github(): gh = shutil.which("gh") if gh is None: pytest.skip("gh CLI not available") - unresolved = {} + missing = {} + uncheckable = [] for action, ref in EXPECTED_ACTION_PINS.items(): repo = "/".join(action.split("/")[:2]) # github/codeql-action/init -> github/codeql-action result = subprocess.run( @@ -4262,15 +4263,26 @@ def test_expected_action_pins_actually_resolve_on_github(): timeout=30, check=False, ) - if result.returncode != 0: - err = result.stderr.lower() - if any(s in err for s in ("rate limit", "could not resolve host", "timeout", "connection")): - pytest.skip(f"network/gh unavailable: {result.stderr.strip()[:120]}") - unresolved[f"{action}@{ref}"] = result.stderr.strip()[:120] - assert not unresolved, ( - f"these pinned action refs do not resolve on GitHub (a pin that does not exist passes the " - f"consistency check but fails at runtime on every generated repo): {unresolved}" + if result.returncode == 0: + continue + err = (result.stderr + result.stdout).lower() + if "not found" in err or "404" in err or "no commit found" in err: + # Definitive: the ref does not exist. This is the failure worth catching. + missing[f"{action}@{ref}"] = result.stderr.strip()[:120] + else: + # Cannot verify: gh unauthenticated (no GH_TOKEN, e.g. the CI test env), + # rate limited, or offline. Do NOT fail on these -- only a real 404 fails. + uncheckable.append(f"{action}@{ref}") + # A definitive 404 fails regardless of any refs we could not check. + assert not missing, ( + f"these pinned action refs do not exist on GitHub (a non-existent pin passes the consistency " + f"check but fails at runtime on every generated repo -- as ossf/scorecard-action@v2 did): {missing}" ) + if uncheckable: + pytest.skip( + f"could not verify {len(uncheckable)}/{len(EXPECTED_ACTION_PINS)} pins " + f"(gh unauthenticated or network unavailable); e.g. {uncheckable[0]}" + ) def test_nav_order_is_diataxis_with_reference_last(copie_session_default, copie_minimal):