diff --git a/template/.github/{% if include_actions %}workflows{% endif %}/{% if repo_visibility == 'public' %}scorecard.yml{% endif %}.jinja b/template/.github/{% if include_actions %}workflows{% endif %}/{% if repo_visibility == 'public' %}scorecard.yml{% endif %}.jinja index d7801b9..e4bac50 100644 --- a/template/.github/{% if include_actions %}workflows{% endif %}/{% if repo_visibility == 'public' %}scorecard.yml{% endif %}.jinja +++ b/template/.github/{% if include_actions %}workflows{% endif %}/{% if repo_visibility == 'public' %}scorecard.yml{% endif %}.jinja @@ -27,7 +27,7 @@ jobs: persist-credentials: false - name: Run analysis - uses: ossf/scorecard-action@v2 + uses: ossf/scorecard-action@v2.4.4 with: results_file: results.sarif results_format: sarif diff --git a/tests/test_template.py b/tests/test_template.py index 6f2cf2c..094d400 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -4184,7 +4184,7 @@ def test_docs_use_no_em_or_en_dashes(copie_session_default): "github/codeql-action/analyze": "v3", "github/codeql-action/init": "v3", "github/codeql-action/upload-sarif": "v3", - "ossf/scorecard-action": "v2", + "ossf/scorecard-action": "v2.4.4", "peter-evans/create-pull-request": "v8", "pypa/gh-action-pypi-publish": "v1.13.0", "taiki-e/install-action": "v2", @@ -4234,6 +4234,57 @@ def test_action_pins_are_consistent_and_current(copie_session_default): assert not wrong, f"pins disagree with what the fleet runs (action: template -> expected): {wrong}" +@pytest.mark.slow +def test_expected_action_pins_actually_resolve_on_github(): + """Every pin in EXPECTED_ACTION_PINS resolves to a real ref on GitHub. + + ``test_action_pins_are_consistent_and_current`` only checks the template's pins are + self-consistent and match this map -- it cannot tell whether a tag *exists*. That + gap shipped ``ossf/scorecard-action@v2`` (no such tag; the action publishes only + ``vX.Y.Z`` tags), which passed every test and then failed at runtime on every + generated repo with "unable to find version v2". This closes it by resolving each + pinned ref against GitHub. + + Network-gated: skips cleanly when ``gh`` or the network is unavailable, so it is a + CI safety net (CI has an authenticated ``gh`` and no rate limit) rather than an + offline tripwire. Marked ``slow`` so it stays out of the fast feedback loop. + """ + gh = shutil.which("gh") + if gh is None: + pytest.skip("gh CLI not available") + missing = {} + uncheckable = [] + for action, ref in EXPECTED_ACTION_PINS.items(): + repo = "/".join(action.split("/")[:2]) # github/codeql-action/init -> github/codeql-action + result = subprocess.run( + [gh, "api", f"repos/{repo}/commits/{ref}", "--jq", ".sha"], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if result.returncode == 0: + continue + err = (result.stderr + result.stdout).lower() + if "not found" in err or "404" in err or "no commit found" in err: + # Definitive: the ref does not exist. This is the failure worth catching. + missing[f"{action}@{ref}"] = result.stderr.strip()[:120] + else: + # Cannot verify: gh unauthenticated (no GH_TOKEN, e.g. the CI test env), + # rate limited, or offline. Do NOT fail on these -- only a real 404 fails. + uncheckable.append(f"{action}@{ref}") + # A definitive 404 fails regardless of any refs we could not check. + assert not missing, ( + f"these pinned action refs do not exist on GitHub (a non-existent pin passes the consistency " + f"check but fails at runtime on every generated repo -- as ossf/scorecard-action@v2 did): {missing}" + ) + if uncheckable: + pytest.skip( + f"could not verify {len(uncheckable)}/{len(EXPECTED_ACTION_PINS)} pins " + f"(gh unauthenticated or network unavailable); e.g. {uncheckable[0]}" + ) + + def test_nav_order_is_diataxis_with_reference_last(copie_session_default, copie_minimal): """The nav reads learn -> do -> see -> understand -> look up, in that order.