Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
persist-credentials: false

- name: Run analysis
uses: ossf/scorecard-action@v2
uses: ossf/scorecard-action@v2.4.4
with:
results_file: results.sarif
results_format: sarif
Expand Down
53 changes: 52 additions & 1 deletion tests/test_template.py
Original file line number Diff line number Diff line change
Expand Up @@ -4184,7 +4184,7 @@ def test_docs_use_no_em_or_en_dashes(copie_session_default):
"github/codeql-action/analyze": "v3",
"github/codeql-action/init": "v3",
"github/codeql-action/upload-sarif": "v3",
"ossf/scorecard-action": "v2",
"ossf/scorecard-action": "v2.4.4",
"peter-evans/create-pull-request": "v8",
"pypa/gh-action-pypi-publish": "v1.13.0",
"taiki-e/install-action": "v2",
Expand Down Expand Up @@ -4234,6 +4234,57 @@ def test_action_pins_are_consistent_and_current(copie_session_default):
assert not wrong, f"pins disagree with what the fleet runs (action: template -> expected): {wrong}"


@pytest.mark.slow
def test_expected_action_pins_actually_resolve_on_github():
"""Every pin in EXPECTED_ACTION_PINS resolves to a real ref on GitHub.

``test_action_pins_are_consistent_and_current`` only checks the template's pins are
self-consistent and match this map -- it cannot tell whether a tag *exists*. That
gap shipped ``ossf/scorecard-action@v2`` (no such tag; the action publishes only
``vX.Y.Z`` tags), which passed every test and then failed at runtime on every
generated repo with "unable to find version v2". This closes it by resolving each
pinned ref against GitHub.

Network-gated: skips cleanly when ``gh`` or the network is unavailable, so it is a
CI safety net (CI has an authenticated ``gh`` and no rate limit) rather than an
offline tripwire. Marked ``slow`` so it stays out of the fast feedback loop.
"""
gh = shutil.which("gh")
if gh is None:
pytest.skip("gh CLI not available")
missing = {}
uncheckable = []
for action, ref in EXPECTED_ACTION_PINS.items():
repo = "/".join(action.split("/")[:2]) # github/codeql-action/init -> github/codeql-action
result = subprocess.run(
[gh, "api", f"repos/{repo}/commits/{ref}", "--jq", ".sha"],
capture_output=True,
text=True,
timeout=30,
check=False,
)
if result.returncode == 0:
continue
err = (result.stderr + result.stdout).lower()
if "not found" in err or "404" in err or "no commit found" in err:
# Definitive: the ref does not exist. This is the failure worth catching.
missing[f"{action}@{ref}"] = result.stderr.strip()[:120]
else:
# Cannot verify: gh unauthenticated (no GH_TOKEN, e.g. the CI test env),
# rate limited, or offline. Do NOT fail on these -- only a real 404 fails.
uncheckable.append(f"{action}@{ref}")
# A definitive 404 fails regardless of any refs we could not check.
assert not missing, (
f"these pinned action refs do not exist on GitHub (a non-existent pin passes the consistency "
f"check but fails at runtime on every generated repo -- as ossf/scorecard-action@v2 did): {missing}"
)
if uncheckable:
pytest.skip(
f"could not verify {len(uncheckable)}/{len(EXPECTED_ACTION_PINS)} pins "
f"(gh unauthenticated or network unavailable); e.g. {uncheckable[0]}"
)


def test_nav_order_is_diataxis_with_reference_last(copie_session_default, copie_minimal):
"""The nav reads learn -> do -> see -> understand -> look up, in that order.

Expand Down