From 23e43d28f31da0b3c65e1831e4f99dd44ccfe35e Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Wed, 22 Jul 2026 11:20:30 -0400 Subject: [PATCH 1/6] Replace workflow/shell automation with a tested pluginctl CLI The registry automation had grown to roughly 5,000 lines of inline YAML and Bash across the plugin workflows, with the app-token block copy-pasted 6 times, the fork-checkout dance 3 times, and the CodeQL SARIF jq program pasted 3 times. This moves the logic into a testable Python package and leaves the workflows as thin wiring. - Add .github/pluginctl: a stdlib-only Python package (core, validate, publish, integrations) exposing one CLI subcommand per task, with a 113-test pytest suite covering the pure logic and golden PR-comment/manifest/README output. - Add two composite actions (gh-app-token, trusted-checkout) and two reusable workflows (_codeql-scan, _clamav-scan) to remove the biggest duplication. - Thin the five core workflows to composite-action + `python -m pluginctl` steps; validate-plugin.yml drops from 1,370 to 442 lines. - Add an opt-in, best-effort signed webhook emitter (HMAC-SHA256) plus docs/webhooks.md, so a downstream consumer can react to registry events. - Delete the V1 validate/publish shell scripts; keep scripts/keys. - Remove the disabled standalone codeql workflow. - Fix a latent bug: deprecated plugins' version count now derives from the manifest's versions[] instead of the removed zips/ path (which rendered 0). User-facing output is kept byte-identical to the old shell, verified by the golden tests. Final fork end-to-end validation is still pending. --- .github/actions/gh-app-token/action.yml | 85 ++ .github/actions/trusted-checkout/action.yml | 64 + .github/pluginctl/README.md | 79 ++ .github/pluginctl/fixtures/sample.sarif | 43 + .github/pluginctl/pyproject.toml | 24 + .github/pluginctl/src/pluginctl/__init__.py | 3 + .github/pluginctl/src/pluginctl/__main__.py | 6 + .github/pluginctl/src/pluginctl/cli.py | 270 ++++ .../pluginctl/src/pluginctl/core/__init__.py | 1 + .../pluginctl/src/pluginctl/core/actions.py | 66 + .github/pluginctl/src/pluginctl/core/gh.py | 183 +++ .github/pluginctl/src/pluginctl/core/git.py | 80 ++ .../pluginctl/src/pluginctl/core/jsonio.py | 38 + .../pluginctl/src/pluginctl/core/models.py | 97 ++ .../pluginctl/src/pluginctl/core/version.py | 89 ++ .../src/pluginctl/integrations/__init__.py | 1 + .../src/pluginctl/integrations/automerge.py | 111 ++ .../pluginctl/integrations/external_readme.py | 157 +++ .../src/pluginctl/integrations/webhooks.py | 118 ++ .../src/pluginctl/publish/__init__.py | 1 + .../src/pluginctl/publish/cleanup.py | 47 + .../src/pluginctl/publish/manifest.py | 399 ++++++ .../src/pluginctl/publish/readmes.py | 440 +++++++ .../pluginctl/src/pluginctl/publish/run.py | 215 +++ .../pluginctl/src/pluginctl/publish/yank.py | 264 ++++ .../pluginctl/src/pluginctl/publish/zips.py | 145 ++ .../src/pluginctl/validate/__init__.py | 1 + .../src/pluginctl/validate/clamav.py | 78 ++ .../src/pluginctl/validate/detect.py | 190 +++ .../pluginctl/src/pluginctl/validate/gate.py | 43 + .../src/pluginctl/validate/labels.py | 35 + .../pluginctl/src/pluginctl/validate/langs.py | 79 ++ .../src/pluginctl/validate/plugin.py | 394 ++++++ .../src/pluginctl/validate/report.py | 425 ++++++ .../pluginctl/src/pluginctl/validate/sarif.py | 259 ++++ .../pluginctl/src/pluginctl/validate/title.py | 69 + .github/pluginctl/tests/conftest.py | 6 + .github/pluginctl/tests/test_clamav.py | 28 + .github/pluginctl/tests/test_detect.py | 48 + .github/pluginctl/tests/test_integrations.py | 60 + .github/pluginctl/tests/test_jsonio.py | 46 + .github/pluginctl/tests/test_labels_gate.py | 55 + .github/pluginctl/tests/test_langs.py | 45 + .github/pluginctl/tests/test_manifest.py | 103 ++ .github/pluginctl/tests/test_readmes.py | 55 + .github/pluginctl/tests/test_report.py | 86 ++ .github/pluginctl/tests/test_sarif.py | 73 ++ .github/pluginctl/tests/test_title.py | 48 + .github/pluginctl/tests/test_validate.py | 109 ++ .github/pluginctl/tests/test_version.py | 36 + .github/pluginctl/tests/test_webhooks.py | 59 + .github/scripts/publish/build-zips.sh | 149 --- .github/scripts/publish/cleanup.sh | 58 - .github/scripts/publish/generate-manifest.sh | 355 ----- .github/scripts/publish/plugin-readmes.sh | 172 --- .github/scripts/publish/releases-readme.sh | 253 ---- .github/scripts/publish/run.sh | 203 --- .github/scripts/publish/yank-version.sh | 283 ---- .github/scripts/validate/detect-changes.sh | 224 ---- .github/scripts/validate/report.sh | 395 ------ .github/scripts/validate/validate.sh | 425 ------ .github/workflows/_clamav-scan.yml | 182 +++ .github/workflows/_codeql-scan.yml | 158 +++ .github/workflows/auto-merge-updates.yml | 160 +-- .github/workflows/codeql.yml | 36 - .github/workflows/publish-plugins.yml | 58 +- .github/workflows/update-external-readme.yml | 183 +-- .github/workflows/validate-plugin.yml | 1168 ++--------------- .github/workflows/yank-plugin-version.yml | 39 +- docs/webhooks.md | 82 ++ 70 files changed, 6066 insertions(+), 3973 deletions(-) create mode 100644 .github/actions/gh-app-token/action.yml create mode 100644 .github/actions/trusted-checkout/action.yml create mode 100644 .github/pluginctl/README.md create mode 100644 .github/pluginctl/fixtures/sample.sarif create mode 100644 .github/pluginctl/pyproject.toml create mode 100644 .github/pluginctl/src/pluginctl/__init__.py create mode 100644 .github/pluginctl/src/pluginctl/__main__.py create mode 100644 .github/pluginctl/src/pluginctl/cli.py create mode 100644 .github/pluginctl/src/pluginctl/core/__init__.py create mode 100644 .github/pluginctl/src/pluginctl/core/actions.py create mode 100644 .github/pluginctl/src/pluginctl/core/gh.py create mode 100644 .github/pluginctl/src/pluginctl/core/git.py create mode 100644 .github/pluginctl/src/pluginctl/core/jsonio.py create mode 100644 .github/pluginctl/src/pluginctl/core/models.py create mode 100644 .github/pluginctl/src/pluginctl/core/version.py create mode 100644 .github/pluginctl/src/pluginctl/integrations/__init__.py create mode 100644 .github/pluginctl/src/pluginctl/integrations/automerge.py create mode 100644 .github/pluginctl/src/pluginctl/integrations/external_readme.py create mode 100644 .github/pluginctl/src/pluginctl/integrations/webhooks.py create mode 100644 .github/pluginctl/src/pluginctl/publish/__init__.py create mode 100644 .github/pluginctl/src/pluginctl/publish/cleanup.py create mode 100644 .github/pluginctl/src/pluginctl/publish/manifest.py create mode 100644 .github/pluginctl/src/pluginctl/publish/readmes.py create mode 100644 .github/pluginctl/src/pluginctl/publish/run.py create mode 100644 .github/pluginctl/src/pluginctl/publish/yank.py create mode 100644 .github/pluginctl/src/pluginctl/publish/zips.py create mode 100644 .github/pluginctl/src/pluginctl/validate/__init__.py create mode 100644 .github/pluginctl/src/pluginctl/validate/clamav.py create mode 100644 .github/pluginctl/src/pluginctl/validate/detect.py create mode 100644 .github/pluginctl/src/pluginctl/validate/gate.py create mode 100644 .github/pluginctl/src/pluginctl/validate/labels.py create mode 100644 .github/pluginctl/src/pluginctl/validate/langs.py create mode 100644 .github/pluginctl/src/pluginctl/validate/plugin.py create mode 100644 .github/pluginctl/src/pluginctl/validate/report.py create mode 100644 .github/pluginctl/src/pluginctl/validate/sarif.py create mode 100644 .github/pluginctl/src/pluginctl/validate/title.py create mode 100644 .github/pluginctl/tests/conftest.py create mode 100644 .github/pluginctl/tests/test_clamav.py create mode 100644 .github/pluginctl/tests/test_detect.py create mode 100644 .github/pluginctl/tests/test_integrations.py create mode 100644 .github/pluginctl/tests/test_jsonio.py create mode 100644 .github/pluginctl/tests/test_labels_gate.py create mode 100644 .github/pluginctl/tests/test_langs.py create mode 100644 .github/pluginctl/tests/test_manifest.py create mode 100644 .github/pluginctl/tests/test_readmes.py create mode 100644 .github/pluginctl/tests/test_report.py create mode 100644 .github/pluginctl/tests/test_sarif.py create mode 100644 .github/pluginctl/tests/test_title.py create mode 100644 .github/pluginctl/tests/test_validate.py create mode 100644 .github/pluginctl/tests/test_version.py create mode 100644 .github/pluginctl/tests/test_webhooks.py delete mode 100644 .github/scripts/publish/build-zips.sh delete mode 100644 .github/scripts/publish/cleanup.sh delete mode 100644 .github/scripts/publish/generate-manifest.sh delete mode 100644 .github/scripts/publish/plugin-readmes.sh delete mode 100644 .github/scripts/publish/releases-readme.sh delete mode 100644 .github/scripts/publish/run.sh delete mode 100644 .github/scripts/publish/yank-version.sh delete mode 100755 .github/scripts/validate/detect-changes.sh delete mode 100755 .github/scripts/validate/report.sh delete mode 100755 .github/scripts/validate/validate.sh create mode 100644 .github/workflows/_clamav-scan.yml create mode 100644 .github/workflows/_codeql-scan.yml delete mode 100644 .github/workflows/codeql.yml create mode 100644 docs/webhooks.md diff --git a/.github/actions/gh-app-token/action.yml b/.github/actions/gh-app-token/action.yml new file mode 100644 index 0000000..4bf5bf4 --- /dev/null +++ b/.github/actions/gh-app-token/action.yml @@ -0,0 +1,85 @@ +name: GitHub App token +description: >- + Mint a GitHub App installation token when GH_APP_ID + GH_APP_PRIVATE_KEY are + configured, otherwise fall back to the workflow GITHUB_TOKEN + (github-actions[bot]) with the same step-summary warning. Replaces the token + block that was copy-pasted across the plugin workflows. + +inputs: + app-id: + description: GitHub App client/app id (pass vars.GH_APP_ID) + required: false + default: "" + private-key: + description: >- + GitHub App private key (pass secrets.GH_APP_PRIVATE_KEY). Composite actions + do not inherit secrets, so it must be provided as an input. + required: false + default: "" + owner: + description: Optional owner to scope the installation token to. + required: false + default: "" + +outputs: + token: + description: The App installation token, or the fallback github.token. + value: ${{ steps.result.outputs.token }} + app-slug: + description: The App slug (empty when falling back). + value: ${{ steps.app-token.outputs.app-slug }} + use_app: + description: "'true' when an App token was minted, 'false' on fallback." + value: ${{ steps.config.outputs.use_app }} + bot-login: + description: The bot login to attribute actions to. + value: ${{ steps.result.outputs.bot_login }} + +runs: + using: composite + steps: + - name: Resolve configuration + id: config + shell: bash + env: + GH_APP_ID: ${{ inputs.app-id }} + GH_APP_PRIVATE_KEY: ${{ inputs.private-key }} + run: | + if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then + echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" + echo "use_app=true" >> "$GITHUB_OUTPUT" + else + echo "use_app=false" >> "$GITHUB_OUTPUT" + fi + + - name: Generate GitHub App token + if: steps.config.outputs.use_app == 'true' + id: app-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ steps.config.outputs.app_id }} + private-key: ${{ inputs.private-key }} + owner: ${{ inputs.owner }} + + - name: Log fallback to actions token + if: steps.config.outputs.use_app != 'true' + shell: bash + run: | + printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" + + - name: Expose token + bot login + id: result + shell: bash + env: + USE_APP: ${{ steps.config.outputs.use_app }} + APP_TOKEN: ${{ steps.app-token.outputs.token }} + APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + FALLBACK_TOKEN: ${{ github.token }} + run: | + if [[ "$USE_APP" == "true" ]]; then + echo "token=$APP_TOKEN" >> "$GITHUB_OUTPUT" + echo "bot_login=${APP_SLUG}[bot]" >> "$GITHUB_OUTPUT" + else + echo "token=$FALLBACK_TOKEN" >> "$GITHUB_OUTPUT" + echo "bot_login=github-actions[bot]" >> "$GITHUB_OUTPUT" + fi diff --git a/.github/actions/trusted-checkout/action.yml b/.github/actions/trusted-checkout/action.yml new file mode 100644 index 0000000..8feda69 --- /dev/null +++ b/.github/actions/trusted-checkout/action.yml @@ -0,0 +1,64 @@ +name: Trusted checkout +description: >- + The pull_request_target checkout "dance": check out the base branch's trusted + code, sparse-checkout the untrusted fork's plugins/ as data only, then restore + the trusted code and re-fetch the base refs. Fork content is never executed; + pluginctl always comes from the base branch. Replaces the sequence duplicated + across detect-changes / validate-plugin. + +inputs: + base-ref: + description: PR base branch (github.event.pull_request.base.ref) + required: true + head-repo: + description: Fork repo full name (github.event.pull_request.head.repo.full_name) + required: true + head-sha: + description: Fork head SHA (github.event.pull_request.head.sha) + required: true + repository: + description: This repository (github.repository) + required: true + +runs: + using: composite + steps: + - name: Checkout base branch code (trusted) + uses: actions/checkout@v6 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.base-ref }} + fetch-depth: 0 + sparse-checkout: .github + sparse-checkout-cone-mode: false + + - name: Fetch base branch + shell: bash + run: git fetch origin ${{ inputs.base-ref }} + + - name: Save trusted code before fork checkout + shell: bash + run: cp -r .github /tmp/trusted-dotgithub + + - name: Checkout PR plugins (untrusted content only) + uses: actions/checkout@v6 + with: + repository: ${{ inputs.head-repo }} + ref: ${{ inputs.head-sha }} + fetch-depth: 0 + sparse-checkout: plugins + sparse-checkout-cone-mode: false + clean: false + # Fork content is only read as data (sparse plugins/), never executed; + # trusted code is saved/restored around this step. + allow-unsafe-pr-checkout: true + + - name: Restore trusted code + shell: bash + run: | + rm -rf .github + cp -r /tmp/trusted-dotgithub .github + + - name: Re-fetch base branch refs + shell: bash + run: git fetch https://github.com/${{ inputs.repository }} +${{ inputs.base-ref }}:refs/remotes/origin/${{ inputs.base-ref }} diff --git a/.github/pluginctl/README.md b/.github/pluginctl/README.md new file mode 100644 index 0000000..3410b6f --- /dev/null +++ b/.github/pluginctl/README.md @@ -0,0 +1,79 @@ +# pluginctl + +The automation CLI behind the plugin-registry workflows. Every piece of logic +that used to live in inline YAML or `.github/scripts/*.sh` now lives here as a +tested Python package, so the workflows are thin wiring and the behavior is +unit-testable. + +Standard library only. `git`, `gh`, `gpg`, and `clamscan` come from the CI +runner; there is no build step. + +## Layout + +``` +src/pluginctl/ + cli.py argparse dispatch (one subcommand per verb) + core/ shared infra: actions, git, gh, jsonio, models, version + validate/ PR-time pipeline: detect, title, labels, gate, sarif, + langs, clamav, plugin (per-plugin checks), report + publish/ releases pipeline: manifest, zips, cleanup, readmes, run, yank + integrations/ webhooks, external_readme, automerge +tests/ pytest suite (pure logic + golden outputs) +fixtures/ sample SARIF and other test inputs +``` + +## Develop + +```bash +cd .github/pluginctl +python -m pytest # run the suite +``` + +Run a command locally against a real checkout (cwd must be the repo root so +`plugins/` resolves): + +```bash +PYTHONPATH=.github/pluginctl/src python -m pluginctl \ + validate --plugin --author --base-ref main --out /tmp/frag.md +``` + +## Command map (old to new) + +| Old | New | +|---|---| +| `validate/detect-changes.sh` + inline blacklist | `pluginctl detect` | +| inline `validate-title` job | `pluginctl check-title` | +| inline `label-pr` job | `pluginctl label` | +| inline CodeQL SARIF jq (x3) | `pluginctl sarif` | +| inline CodeQL language detection | `pluginctl detect-langs` | +| inline ClamAV status/table | `pluginctl clamav-report` | +| `validate/validate.sh` | `pluginctl validate` | +| `validate/report.sh` | `pluginctl report` | +| inline gate ladder | `pluginctl gate` | +| `publish/run.sh` (+ chained scripts) | `pluginctl publish` | +| `publish/yank-version.sh` | `pluginctl yank` | +| inline `auto-merge-updates` | `pluginctl automerge` | +| inline `update-external-readme` | `pluginctl external-readme` | +| (new) signed events | `pluginctl webhook` | + +## How the workflows wire in + +- `.github/actions/gh-app-token` and `.github/actions/trusted-checkout` are + composite actions (reusable *steps*). +- `.github/workflows/_codeql-scan.yml` and `_clamav-scan.yml` are reusable + *workflows* (whole jobs), called by `validate-plugin.yml`. They must live in + `.github/workflows/` per GitHub; the `_` prefix marks them as internal. +- Every `python -m pluginctl` step sets `PYTHONPATH=.github/pluginctl/src`. In + the CodeQL/ClamAV jobs the package is loaded from a separate base-branch + checkout (`_trusted/`) so fork code from the PR merge tree is never executed. + +## Parity notes for maintainers + +- User-facing output (PR comments, manifests, READMEs) must stay byte-identical + to the old shell. `core/jsonio.py` reproduces `jq -c` exactly, and the + markdown renderers are ported line for line, guarded by golden tests. PR-comment + strings are copied verbatim from the old scripts, so match them exactly rather + than reformatting. +- `publish/readmes.py` intentionally corrects one old bug: deprecated plugins' + version count now comes from the manifest's `versions[]` instead of the + removed `zips/` path (which always rendered 0). diff --git a/.github/pluginctl/fixtures/sample.sarif b/.github/pluginctl/fixtures/sample.sarif new file mode 100644 index 0000000..77b431b --- /dev/null +++ b/.github/pluginctl/fixtures/sample.sarif @@ -0,0 +1,43 @@ +{ + "runs": [ + { + "tool": { + "driver": { + "rules": [ + {"id": "py/sql-injection", "properties": {"security-severity": "8.8"}}, + {"id": "py/weak-hash", "properties": {"security-severity": "6.5"}}, + {"id": "py/clear-text-logging", "properties": {"security-severity": "5.0"}}, + {"id": "py/unused-import", "properties": {}} + ] + }, + "extensions": [ + {"rules": [ + {"id": "py/weak-hash", "properties": {"security-severity": "6.9"}} + ]} + ] + }, + "results": [ + { + "ruleId": "py/sql-injection", + "message": {"text": "This SQL query depends on a [user-provided value](123)."}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "plugins/demo/main.py"}, "region": {"startLine": 42}}}] + }, + { + "ruleId": "py/weak-hash", + "message": {"text": "Weak hash used at https://example.com/#4 www.foo"}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "plugins/demo/util.py"}, "region": {"startLine": 7}}}] + }, + { + "ruleId": "py/clear-text-logging", + "message": {"text": "Sensitive data logged."}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "plugins/demo/log.py"}, "region": {"startLine": 3}}}] + }, + { + "ruleId": "py/unused-import", + "message": {"text": "Unused import."}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "plugins/demo/x.py"}, "region": {"startLine": 1}}}] + } + ] + } + ] +} diff --git a/.github/pluginctl/pyproject.toml b/.github/pluginctl/pyproject.toml new file mode 100644 index 0000000..3f27a43 --- /dev/null +++ b/.github/pluginctl/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "pluginctl" +version = "2.0.0" +description = "Dispatcharr plugin registry automation CLI (validate / publish / yank / webhooks)" +requires-python = ">=3.10" +# Standard-library only. git / gh / gpg / clamscan are provided by the CI runner. +dependencies = [] + +[project.optional-dependencies] +test = ["pytest>=7.0"] + +[project.scripts] +pluginctl = "pluginctl.cli:main" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["pluginctl*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/.github/pluginctl/src/pluginctl/__init__.py b/.github/pluginctl/src/pluginctl/__init__.py new file mode 100644 index 0000000..4718b94 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/__init__.py @@ -0,0 +1,3 @@ +"""pluginctl - Dispatcharr plugin registry automation CLI (V2).""" + +__version__ = "2.0.0" diff --git a/.github/pluginctl/src/pluginctl/__main__.py b/.github/pluginctl/src/pluginctl/__main__.py new file mode 100644 index 0000000..c1ba562 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/__main__.py @@ -0,0 +1,6 @@ +"""Enable ``python -m pluginctl``.""" + +from .cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/pluginctl/src/pluginctl/cli.py b/.github/pluginctl/src/pluginctl/cli.py new file mode 100644 index 0000000..1e7a58e --- /dev/null +++ b/.github/pluginctl/src/pluginctl/cli.py @@ -0,0 +1,270 @@ +"""argparse dispatch for ``pluginctl`` / ``python -m pluginctl``. + +Each subcommand is a thin wrapper that reads flags (and falls back to the same +environment variables the workflows already set) and delegates to a handler +module. Handlers are imported lazily so a single missing optional dependency or +runtime tool never breaks unrelated commands. +""" + +from __future__ import annotations + +import argparse +import json +import os +from typing import Optional + + +def _matrix(value: Optional[str]) -> list[str]: + if not value: + return [] + try: + parsed = json.loads(value) + return [str(x) for x in parsed] if isinstance(parsed, list) else [] + except json.JSONDecodeError: + return [] + + +def _bool(value: Optional[str]) -> bool: + return str(value).lower() == "true" + + +def _env(name: str, default: str = "") -> str: + return os.environ.get(name, default) + + +# --------------------------------------------------------------------------- # +# Handlers +# --------------------------------------------------------------------------- # +def cmd_detect(args) -> int: + from .validate import detect + return detect.run( + pr_author=args.author, + base_ref=args.base_ref, + head_ref=args.head_ref or "", + author_blacklist=_env("AUTHOR_BLACKLIST"), + plugin_blacklist=_env("PLUGIN_BLACKLIST"), + repo=args.repo or _env("GITHUB_REPOSITORY"), + ) + + +def cmd_check_title(args) -> int: + from .core import actions + from .validate import title + result = title.check_title(args.title, args.author, args.plugin_count, _matrix(args.matrix)) + actions.set_output("title_valid", "true" if result.valid else "false") + actions.set_output("title_feedback", result.feedback) + actions.set_output("title_suggestion", result.suggestion) + if not result.valid: + actions.error("PR title does not match the required format. See CONTRIBUTING.md for details.") + return 1 + return 0 + + +def cmd_label(args) -> int: + from .validate import labels + return labels.run( + pr_number=args.pr, + has_new_plugin=_bool(args.has_new_plugin), + has_updated_plugin=_bool(args.has_updated_plugin), + outside_files=args.outside_files or "", + outside_violation=_bool(args.outside_violation), + close_pr=_bool(args.close_pr), + ) + + +def cmd_gate(args) -> int: + from .core import actions + from .validate import gate + result = gate.evaluate( + detect_result=_env("DETECT_RESULT"), + close_pr=_env("CLOSE_PR"), + skip_validation=_env("SKIP_VALIDATION"), + outside_violation=_env("OUTSIDE_VIOLATION"), + title_result=_env("TITLE_RESULT"), + codeql_result=_env("CODEQL_RESULT"), + codeql_status=_env("CODEQL_STATUS"), + clamav_result=_env("CLAMAV_RESULT"), + clamav_status=_env("CLAMAV_STATUS"), + validate_result=_env("VALIDATE_RESULT"), + report_result=_env("REPORT_RESULT"), + ) + if result.ok: + actions.log(result.message) + return 0 + actions.error(result.message) + return 1 + + +def cmd_sarif(args) -> int: + from .validate import sarif + return sarif.run( + results_dir=args.results_dir, + repo=args.repo or _env("GITHUB_REPOSITORY"), + sha=args.sha, + matrix=_matrix(args.matrix), + analyze_outcome=args.analyze_outcome, + languages_found=_bool(args.found), + languages=args.languages or "", + unscanned_langs=args.unscanned_langs or "", + job_status=_env("CODEQL_ACTION_JOB_STATUS"), + ) + + +def cmd_validate(args) -> int: + from .validate import plugin as validate + return validate.run( + plugin_name=args.plugin, + pr_author=args.author, + base_ref=args.base_ref, + output_file=args.out, + repo=args.repo or _env("GITHUB_REPOSITORY"), + ) + + +def cmd_report(args) -> int: + from .validate import report + return report.run( + pr_number=args.pr, + pr_author=args.author, + plugin_count=args.plugin_count, + close_pr=_bool(args.close_pr), + fragments_dir=args.fragments_dir, + repo=args.repo or _env("GITHUB_REPOSITORY"), + ) + + +def cmd_detect_langs(args) -> int: + from .validate import langs + return langs.run(root=args.root) + + +def cmd_clamav_report(args) -> int: + from .validate import clamav + return clamav.run(output_file=args.output, scan_exit=args.scan_exit, + findings_out=args.findings_out) + + +def cmd_webhook(args) -> int: + from .integrations import webhooks + data = json.loads(args.data) if args.data else {} + sent = webhooks.emit(args.event, data) + return 0 if sent or True else 1 # emission failures never fail the pipeline + + +def cmd_publish(args) -> int: + from .publish import run as publish_run + return publish_run.run(source_branch=args.source_branch) + + +def cmd_yank(args) -> int: + from .publish import yank + return yank.run() + + +def cmd_automerge(args) -> int: + from .integrations import automerge + return automerge.run(head_sha=args.head_sha or _env("HEAD_SHA")) + + +def cmd_external_readme(args) -> int: + from .integrations import external_readme + return external_readme.run() + + +# --------------------------------------------------------------------------- # +# Parser +# --------------------------------------------------------------------------- # +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="pluginctl", description="Dispatcharr plugin registry automation") + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("detect", help="Detect changed plugins and build the matrix") + p.add_argument("--author", required=True) + p.add_argument("--base-ref", required=True) + p.add_argument("--head-ref", default="") + p.add_argument("--repo", default="") + p.set_defaults(func=cmd_detect) + + p = sub.add_parser("check-title", help="Validate the PR title format") + p.add_argument("--title", required=True) + p.add_argument("--author", required=True) + p.add_argument("--plugin-count", type=int, default=0) + p.add_argument("--matrix", default="[]") + p.set_defaults(func=cmd_check_title) + + p = sub.add_parser("label", help="Reconcile PR classification labels") + p.add_argument("--pr", required=True) + p.add_argument("--has-new-plugin", default="false") + p.add_argument("--has-updated-plugin", default="false") + p.add_argument("--outside-files", default="") + p.add_argument("--outside-violation", default="false") + p.add_argument("--close-pr", default="false") + p.set_defaults(func=cmd_label) + + p = sub.add_parser("gate", help="Evaluate the Plugin PR Check gate (env-driven)") + p.set_defaults(func=cmd_gate) + + p = sub.add_parser("sarif", help="Parse CodeQL SARIF: counts, tables, status") + p.add_argument("--results-dir", default="sarif-results") + p.add_argument("--repo", default="") + p.add_argument("--sha", default="") + p.add_argument("--matrix", default="[]") + p.add_argument("--analyze-outcome", default="") + p.add_argument("--found", default="false") + p.add_argument("--languages", default="") + p.add_argument("--unscanned-langs", default="") + p.set_defaults(func=cmd_sarif) + + p = sub.add_parser("validate", help="Validate one plugin -> markdown fragment") + p.add_argument("--plugin", required=True) + p.add_argument("--author", required=True) + p.add_argument("--base-ref", required=True) + p.add_argument("--out", required=True) + p.add_argument("--repo", default="") + p.set_defaults(func=cmd_validate) + + p = sub.add_parser("report", help="Aggregate fragments, post PR comment") + p.add_argument("--pr", required=True) + p.add_argument("--author", required=True) + p.add_argument("--plugin-count", required=True) + p.add_argument("--close-pr", default="false") + p.add_argument("--fragments-dir", default=".") + p.add_argument("--repo", default="") + p.set_defaults(func=cmd_report) + + p = sub.add_parser("detect-langs", help="Detect CodeQL languages in changed plugins") + p.add_argument("--root", default="plugins") + p.set_defaults(func=cmd_detect_langs) + + p = sub.add_parser("clamav-report", help="Parse clamscan output: status + findings table") + p.add_argument("--output", default="clamav-output.txt") + p.add_argument("--scan-exit", type=int, default=0) + p.add_argument("--findings-out", default="clamav-findings.md") + p.set_defaults(func=cmd_clamav_report) + + p = sub.add_parser("webhook", help="Emit a signed webhook event") + p.add_argument("--event", required=True) + p.add_argument("--data", default="{}") + p.set_defaults(func=cmd_webhook) + + p = sub.add_parser("publish", help="Publish plugins to the releases branch") + p.add_argument("source_branch") + p.set_defaults(func=cmd_publish) + + p = sub.add_parser("yank", help="Yank a plugin version (env-driven)") + p.set_defaults(func=cmd_yank) + + p = sub.add_parser("automerge", help="Auto-merge a pure plugin-update PR") + p.add_argument("--head-sha", default="") + p.set_defaults(func=cmd_automerge) + + p = sub.add_parser("external-readme", help="Open/update the external README PR (env-driven)") + p.set_defaults(func=cmd_external_readme) + + return parser + + +def main(argv: Optional[list[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + return args.func(args) diff --git a/.github/pluginctl/src/pluginctl/core/__init__.py b/.github/pluginctl/src/pluginctl/core/__init__.py new file mode 100644 index 0000000..b9f67b6 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/__init__.py @@ -0,0 +1 @@ +"""Shared infrastructure: JSON, versioning, git/gh wrappers, Actions plumbing.""" diff --git a/.github/pluginctl/src/pluginctl/core/actions.py b/.github/pluginctl/src/pluginctl/core/actions.py new file mode 100644 index 0000000..3ba14f1 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/actions.py @@ -0,0 +1,66 @@ +"""GitHub Actions plumbing: GITHUB_OUTPUT, step summary, workflow commands. + +Thin, side-effecting helpers that write to the files/streams the runner reads. +Kept tiny so command modules stay testable (they build values; these emit them). +""" + +from __future__ import annotations + +import os +import sys +import uuid + + +def set_output(name: str, value: str) -> None: + """Append ``name=value`` to ``$GITHUB_OUTPUT`` (multiline-safe). + + Values containing a newline use the heredoc form GitHub documents, with a + random delimiter to avoid collisions with the payload. + """ + path = os.environ.get("GITHUB_OUTPUT") + if not path: + # Local/dry-run: echo to stdout the same way the shell scripts did. + print(f"{name}={value}") + return + with open(path, "a", encoding="utf-8") as fh: + if "\n" in value: + delim = f"ghadelim_{uuid.uuid4().hex}" + fh.write(f"{name}<<{delim}\n{value}\n{delim}\n") + else: + fh.write(f"{name}={value}\n") + + +def set_outputs(**kwargs: str) -> None: + for name, value in kwargs.items(): + set_output(name, "" if value is None else str(value)) + + +def step_summary(markdown: str) -> None: + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + print(markdown) + return + with open(path, "a", encoding="utf-8") as fh: + fh.write(markdown) + if not markdown.endswith("\n"): + fh.write("\n") + + +def error(message: str) -> None: + print(f"::error::{message}") + + +def warning(message: str) -> None: + print(f"::warning::{message}") + + +def notice(message: str) -> None: + print(f"::notice::{message}") + + +def log(message: str) -> None: + print(message) + + +def eprint(message: str) -> None: + print(message, file=sys.stderr) diff --git a/.github/pluginctl/src/pluginctl/core/gh.py b/.github/pluginctl/src/pluginctl/core/gh.py new file mode 100644 index 0000000..a3d35c2 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/gh.py @@ -0,0 +1,183 @@ +"""Thin wrapper over the ``gh`` CLI and the GitHub REST/GraphQL surface we use. + +Every call shells out to ``gh`` (already authenticated on the runner via +``GH_TOKEN``), matching how the V1 scripts invoked it. Helpers return parsed +values and swallow errors the same way the ``|| true`` / ``2>/dev/null`` idioms +did in Bash, so callers keep the same permissive behavior. +""" + +from __future__ import annotations + +import json +import subprocess +from typing import Any, Optional + + +def _run(args: list[str], check: bool = False, input_text: Optional[str] = None) -> subprocess.CompletedProcess: + return subprocess.run( + ["gh", *args], + check=check, + capture_output=True, + text=True, + input=input_text, + ) + + +def api(endpoint: str, jq: Optional[str] = None, method: Optional[str] = None, + fields: Optional[dict[str, str]] = None) -> Optional[str]: + """``gh api `` returning stripped stdout, or None on failure.""" + args = ["api", endpoint] + if method: + args += ["-X", method] + if jq: + args += ["--jq", jq] + for k, v in (fields or {}).items(): + args += ["-f", f"{k}={v}"] + proc = _run(args) + if proc.returncode != 0: + return None + return proc.stdout.strip() + + +def collaborator_permission(owner: str, repo: str, user: str) -> str: + """Return the collaborator permission string, or ``none`` (matches Bash).""" + perm = api(f"repos/{owner}/{repo}/collaborators/{user}/permission", + jq=".permission") + return perm if perm else "none" + + +def has_write_access(owner: str, repo: str, user: str) -> bool: + """True when the user has admin/maintain/write, mirroring has_write_access().""" + return collaborator_permission(owner, repo, user) in ("admin", "maintain", "write") + + +def graphql(query: str, f_fields: Optional[dict[str, str]] = None, + F_fields: Optional[dict[str, str]] = None, jq: Optional[str] = None) -> Optional[str]: + args = ["api", "graphql", "-f", f"query={query}"] + for k, v in (f_fields or {}).items(): + args += ["-f", f"{k}={v}"] + for k, v in (F_fields or {}).items(): + args += ["-F", f"{k}={v}"] + if jq: + args += ["--jq", jq] + proc = _run(args) + if proc.returncode != 0: + return None + return proc.stdout.strip() + + +def pr_comment(pr_number: str, body: str) -> int: + """``gh pr comment``: returns the exit code (report.sh keys off it).""" + return _run(["pr", "comment", str(pr_number), "--body", body]).returncode + + +def pr_close(pr_number: str) -> int: + return _run(["pr", "close", str(pr_number)]).returncode + + +def pr_comment_repo(pr_number: str, repo: str, body: str) -> int: + return _run(["pr", "comment", str(pr_number), "--repo", repo, "--body", body]).returncode + + +def pr_edit_label(pr_number: str, label: str, add: bool) -> None: + flag = "--add-label" if add else "--remove-label" + _run(["pr", "edit", str(pr_number), flag, label]) + + +def label_create(name: str, color: str, description: str, repo: str) -> None: + _run(["label", "create", name, "--color", color, + "--description", description, "--repo", repo]) + + +def pr_view_json(pr_number: str, fields: str, jq: Optional[str] = None, + repo: Optional[str] = None) -> Optional[str]: + args = ["pr", "view", str(pr_number), "--json", fields] + if jq: + args += ["--jq", jq] + if repo: + args += ["--repo", repo] + proc = _run(args) + if proc.returncode != 0: + return None + return proc.stdout.strip() + + +def loads(text: Optional[str]) -> Any: + if not text: + return None + try: + return json.loads(text) + except json.JSONDecodeError: + return None + + +# ---- GitHub Releases helpers (publish / yank / cleanup) --------------------- +def release_exists(tag: str, repo: str) -> bool: + return _run(["release", "view", tag, "--repo", repo]).returncode == 0 + + +def release_list_tags(repo: str, limit: int = 500) -> list[str]: + proc = _run(["release", "list", "--repo", repo, "--json", "tagName", "--limit", str(limit)]) + if proc.returncode != 0: + return [] + try: + data = json.loads(proc.stdout) + except json.JSONDecodeError: + return [] + return [entry.get("tagName", "") for entry in data if entry.get("tagName")] + + +def release_create(tag: str, repo: str, title: str, notes: str, asset: str) -> int: + return _run(["release", "create", tag, "--repo", repo, "--title", title, + "--notes", notes, asset]).returncode + + +def release_delete(tag: str, repo: str, cleanup_tag: bool = True) -> int: + args = ["release", "delete", tag, "--repo", repo, "--yes"] + if cleanup_tag: + args.append("--cleanup-tag") + return _run(args).returncode + + +def pr_list(repo: str, state: str, fields: str, jq: Optional[str] = None) -> Optional[str]: + args = ["pr", "list", "--repo", repo, "--state", state, "--json", fields] + if jq: + args += ["--jq", jq] + proc = _run(args) + if proc.returncode != 0: + return None + return proc.stdout.strip() + + +def pr_merge_squash(pr: str, repo: str, delete_branch: bool = False) -> int: + return _run(["pr", "merge", str(pr), "--repo", repo, "--squash", + f"--delete-branch={str(delete_branch).lower()}"]).returncode + + +def issue_comment(issue: str, repo: str, body: str) -> int: + return _run(["issue", "comment", str(issue), "--repo", repo, "--body", body]).returncode + + +def issue_close(issue: str, repo: str, reason: str = "completed") -> int: + return _run(["issue", "close", str(issue), "--repo", repo, "--reason", reason]).returncode + + +def pr_create(repo: str, base: str, head: str, title: str, body: str, + labels: Optional[list[str]] = None) -> Optional[str]: + args = ["pr", "create", "--repo", repo, "--base", base, "--head", head, + "--title", title, "--body", body] + for label in (labels or []): + args += ["--label", label] + proc = _run(args) + if proc.returncode != 0: + return None + return proc.stdout.strip() + + +def commit_pull_number(repo: str, sha: str) -> tuple[Optional[str], Optional[str]]: + """The first PR associated with a commit, as (number, html_url) or (None, None).""" + out = api(f"repos/{repo}/commits/{sha}/pulls", + jq='.[0] | {number: .number, url: .html_url}') + data = loads(out) or {} + number = data.get("number") + return (str(number) if number else None, data.get("url")) diff --git a/.github/pluginctl/src/pluginctl/core/git.py b/.github/pluginctl/src/pluginctl/core/git.py new file mode 100644 index 0000000..d347f58 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/git.py @@ -0,0 +1,80 @@ +"""Thin wrappers over the local ``git`` CLI used by detect / validate / publish.""" + +from __future__ import annotations + +import subprocess +from typing import Optional + + +def _run(args: list[str], check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run( + ["git", *args], + check=check, + capture_output=True, + text=True, + ) + + +def merge_base(a: str, b: str) -> str: + return _run(["merge-base", a, b]).stdout.strip() + + +def diff_name_only(base: str, head: str = "HEAD", paths: Optional[list[str]] = None) -> list[str]: + args = ["diff", "--name-only", base, head] + if paths: + args += ["--", *paths] + out = _run(args).stdout + return [line for line in out.splitlines() if line] + + +def diff_name_only_range(range_spec: str, paths: Optional[list[str]] = None) -> list[str]: + """``git diff --name-only [-- paths]`` where range is e.g. ``a...HEAD``.""" + args = ["diff", "--name-only", range_spec] + if paths: + args += ["--", *paths] + out = _run(args).stdout + return [line for line in out.splitlines() if line] + + +def show(ref_path: str) -> Optional[str]: + """``git show :`` returning None when the object does not exist.""" + proc = _run(["show", ref_path], check=False) + if proc.returncode != 0: + return None + return proc.stdout + + +def object_exists(ref_path: str) -> bool: + return _run(["show", ref_path], check=False).returncode == 0 + + +def log_format(fmt: str, ref: str, path: Optional[str] = None) -> Optional[str]: + """``git log -1 --format= [-- ]`` (None if empty/failure).""" + args = ["log", "-1", f"--format={fmt}", ref] + if path is not None: + args += ["--", path] + proc = _run(args, check=False) + if proc.returncode != 0: + return None + out = proc.stdout.strip() + return out or None + + +def rev_parse(rev: str, short: bool = False) -> str: + args = ["rev-parse"] + if short: + args.append("--short") + args.append(rev) + return _run(args).stdout.strip() + + +def fetch(*args: str) -> None: + _run(["fetch", *args], check=False) + + +def sparse_checkout_add(path: str) -> bool: + return _run(["sparse-checkout", "add", path], check=False).returncode == 0 + + +def checkout(*args: str) -> bool: + return _run(["checkout", *args], check=False).returncode == 0 diff --git a/.github/pluginctl/src/pluginctl/core/jsonio.py b/.github/pluginctl/src/pluginctl/core/jsonio.py new file mode 100644 index 0000000..d9a1ff2 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/jsonio.py @@ -0,0 +1,38 @@ +"""Canonical JSON helpers that reproduce the exact byte output of the V1 jq programs. + +The V1 publish pipeline builds manifests with `jq` and emits them with `jq -c`. +To keep the published `releases`-branch output byte-identical, every manifest +object here is built as an insertion-ordered ``dict`` in the same key order as the +corresponding jq program, nulls are dropped the same way jq's +``with_entries(select(.value != null))`` does, and the compact form is produced +with the same separators jq uses. +""" + +from __future__ import annotations + +import json +from typing import Any + + +def dumps(obj: Any) -> str: + """Compact JSON, byte-equivalent to ``jq -c '.'``. + + jq -c uses ``,``/``:`` separators, emits UTF-8 (no ``\\uXXXX`` escaping of + non-ASCII), and does not escape ``/``. ``json.dumps`` with these options + matches on all three counts. + """ + return json.dumps(obj, separators=(",", ":"), ensure_ascii=False) + + +def drop_none(obj: dict) -> dict: + """Reproduce ``with_entries(select(.value != null))`` for a single object. + + Only top-level ``None`` values are removed, insertion order preserved, + matching the jq idiom used throughout ``generate-manifest.sh``. + """ + return {k: v for k, v in obj.items() if v is not None} + + +def compact(obj: Any) -> str: + """Alias for :func:`dumps` for call-site readability.""" + return dumps(obj) diff --git a/.github/pluginctl/src/pluginctl/core/models.py b/.github/pluginctl/src/pluginctl/core/models.py new file mode 100644 index 0000000..d88625b --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/models.py @@ -0,0 +1,97 @@ +"""Typed view over a ``plugin.json`` document. + +Wraps the parsed dict and exposes the handful of accessors the validate/publish +logic needs, with the same ``// ""`` / ``// false`` defaulting the jq programs +used. ``raw`` stays available for the manifest builders that must preserve the +original key set and ordering. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Any, Optional + + +@dataclass +class Plugin: + raw: dict = field(default_factory=dict) + + # ---- constructors ------------------------------------------------------- + @classmethod + def from_json_text(cls, text: str) -> "Plugin": + return cls(raw=json.loads(text)) + + @classmethod + def from_file(cls, path: str) -> "Plugin": + with open(path, encoding="utf-8") as fh: + return cls(raw=json.load(fh)) + + # ---- scalar accessors (jq `// ""` semantics) ---------------------------- + def str_field(self, key: str, default: str = "") -> str: + val = self.raw.get(key) + return default if val is None else str(val) + + @property + def name(self) -> str: + return self.str_field("name") + + @property + def version(self) -> str: + return self.str_field("version") + + @property + def description(self) -> str: + return self.str_field("description") + + @property + def author(self) -> str: + return self.str_field("author") + + @property + def license(self) -> str: + return self.str_field("license") + + @property + def repo_url(self) -> str: + return self.str_field("repo_url") + + @property + def discord_thread(self) -> str: + return self.str_field("discord_thread") + + @property + def source_type(self) -> str: + return self.str_field("source_type", "local") + + @property + def source_url(self) -> str: + return self.str_field("source_url") + + @property + def min_dispatcharr_version(self) -> str: + return self.str_field("min_dispatcharr_version") + + @property + def max_dispatcharr_version(self) -> str: + return self.str_field("max_dispatcharr_version") + + @property + def deprecated(self) -> bool: + return self.raw.get("deprecated") is True + + @property + def unlisted(self) -> bool: + return self.raw.get("unlisted") is True + + @property + def maintainers(self) -> list[str]: + """``[.maintainers[]?]``: always a list, dropping non-list values.""" + val = self.raw.get("maintainers") + return list(val) if isinstance(val, list) else [] + + def maintainers_joined(self, sep: str = " ") -> str: + return sep.join(str(m) for m in self.maintainers) + + def get(self, key: str, default: Any = None) -> Any: + return self.raw.get(key, default) diff --git a/.github/pluginctl/src/pluginctl/core/version.py b/.github/pluginctl/src/pluginctl/core/version.py new file mode 100644 index 0000000..f279b8f --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/version.py @@ -0,0 +1,89 @@ +"""Version parsing / comparison helpers ported from the Bash validate + publish scripts. + +These reproduce the exact acceptance rules of the shell functions: + validate_semver -> ^[0-9]+\\.[0-9]+\\.[0-9]+$ + validate_dispatcharr_version -> ^v?[0-9]+\\.[0-9]+\\.[0-9]+$ + version_greater_than -> field-wise major/minor/patch comparison +and the `sort -V -r` ordering used to pick newest release tags. +""" + +from __future__ import annotations + +import re +from functools import cmp_to_key + +_SEMVER_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") +_DA_VERSION_RE = re.compile(r"^v?[0-9]+\.[0-9]+\.[0-9]+$") + + +def is_semver(version: str) -> bool: + """True for strict ``X.Y.Z`` (matches validate_semver == 1).""" + return bool(_SEMVER_RE.match(version or "")) + + +def is_dispatcharr_version(version: str) -> bool: + """True for ``X.Y.Z`` or ``vX.Y.Z`` (matches validate_dispatcharr_version == 1).""" + return bool(_DA_VERSION_RE.match(version or "")) + + +def version_greater_than(new_version: str, old_version: str) -> bool: + """Return True when new > old, comparing major/minor/patch as integers. + + Mirrors the Bash ``version_greater_than`` exactly: it reads three + dot-separated fields and compares them numerically; equal versions are not + "greater". + """ + def _parts(v: str) -> tuple[int, int, int]: + fields = (v or "").split(".") + # Bash `IFS='.' read -r A B C` leaves missing fields empty -> treated as 0 + nums = [] + for i in range(3): + try: + nums.append(int(fields[i])) + except (IndexError, ValueError): + nums.append(0) + return nums[0], nums[1], nums[2] + + nmaj, nmin, npat = _parts(new_version) + omaj, omin, opat = _parts(old_version) + if nmaj > omaj: + return True + if nmaj < omaj: + return False + if nmin > omin: + return True + if nmin < omin: + return False + if npat > opat: + return True + return False + + +def _version_sort_key(s: str): + """A key that orders like GNU ``sort -V`` for the version strings we handle. + + Release tags are stripped to bare versions such as ``1.2.3`` or ``1.2.3-1`` + (a migration retry suffix). ``sort -V`` treats each maximal run of digits as a + number and each run of non-digits lexically, so we tokenize accordingly. + """ + tokens: list = [] + for part in re.findall(r"\d+|\D+", s or ""): + if part.isdigit(): + tokens.append((1, int(part), "")) + else: + tokens.append((0, 0, part)) + return tokens + + +def _version_cmp(a: str, b: str) -> int: + ka, kb = _version_sort_key(a), _version_sort_key(b) + if ka < kb: + return -1 + if ka > kb: + return 1 + return 0 + + +def sort_versions_desc(versions: list[str]) -> list[str]: + """Sort bare version strings newest-first, like ``sort -V -r``.""" + return sorted(versions, key=cmp_to_key(_version_cmp), reverse=True) diff --git a/.github/pluginctl/src/pluginctl/integrations/__init__.py b/.github/pluginctl/src/pluginctl/integrations/__init__.py new file mode 100644 index 0000000..4410127 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/integrations/__init__.py @@ -0,0 +1 @@ +"""Outward integrations: signed webhooks, external README PR, auto-merge.""" diff --git a/.github/pluginctl/src/pluginctl/integrations/automerge.py b/.github/pluginctl/src/pluginctl/integrations/automerge.py new file mode 100644 index 0000000..bef2611 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/integrations/automerge.py @@ -0,0 +1,111 @@ +"""Auto-merge pure plugin-update PRs (port of auto-merge-updates.yml inline logic). + +Only squash-merges a PR that carries the ``Plugin Update`` label, none of the +blocking labels, is cleanly mergeable, and independently re-verifies from the diff +that it touches only existing plugins under ``plugins/``. Every rejection is a +notice, never a failure. The workflow already gates on the GitHub App token being +configured (a GITHUB_TOKEN merge would not cascade into publish), so this handler +assumes it is invoked only in that case. +""" + +from __future__ import annotations + +import json +import os +import subprocess +from dataclasses import dataclass + +from ..core import actions, gh + +BLOCKING_LABELS = ("New Plugin", "Repo Update", "Invalid", "QUARANTINE") + + +def _git(*args: str, check: bool = False) -> subprocess.CompletedProcess: + return subprocess.run(["git", *args], check=check, capture_output=True, text=True) + + +@dataclass +class LabelDecision: + ok: bool + reason: str = "" + + +def evaluate_labels(labels: list[str], mergeable: str) -> LabelDecision: + """Require Plugin Update, forbid the blocking labels, require MERGEABLE.""" + label_set = set(labels) + if "Plugin Update" not in label_set: + return LabelDecision(False, "Missing 'Plugin Update' label") + for blocked in BLOCKING_LABELS: + if blocked in label_set: + return LabelDecision(False, f"Has '{blocked}' label") + if mergeable != "MERGEABLE": + return LabelDecision(False, f"PR not cleanly mergeable (mergeable={mergeable})") + return LabelDecision(True) + + +def reverify_is_pure_update(pr_number: str, base_branch: str = "main") -> bool: + """Re-derive from the diff that the PR only updates existing plugins.""" + if _git("fetch", "origin", f"pull/{pr_number}/merge:pr-merge-{pr_number}").returncode != 0: + actions.notice(f"Could not fetch merge ref for PR #{pr_number} - skipping.") + return False + _git("checkout", "-q", f"pr-merge-{pr_number}") + merge_base = _git("merge-base", f"origin/{base_branch}", "HEAD").stdout.strip() + changed = _git("diff", "--name-only", merge_base, "HEAD").stdout.splitlines() + + outside = [f for f in changed if not f.startswith("plugins/")] + if outside: + actions.notice(f"Changes outside plugins/ detected - skipping. Files: {' '.join(outside)}") + return False + + slugs = sorted({f.split("/")[1] for f in changed + if f.startswith("plugins/") and len(f.split("/")) > 1}) + if not slugs: + actions.notice("No plugin changes detected in diff - skipping.") + return False + + for slug in slugs: + if _git("show", f"origin/{base_branch}:plugins/{slug}/plugin.json").returncode != 0: + actions.notice(f"Plugin '{slug}' does not exist on {base_branch} - this is a new " + "plugin, not an update. Skipping.") + return False + return True + + +def run(head_sha: str) -> int: + repo = os.environ.get("GH_REPO") or os.environ.get("GITHUB_REPOSITORY", "") + + matches_json = gh.pr_list(repo, "open", "number,headRefOid,isDraft") or "[]" + try: + matches = [pr for pr in json.loads(matches_json) if pr.get("headRefOid") == head_sha] + except json.JSONDecodeError: + matches = [] + if len(matches) != 1: + actions.notice(f"Found {len(matches)} open PR(s) matching head SHA {head_sha} - " + "skipping (expected exactly 1).") + return 0 + pr = matches[0] + pr_number = str(pr.get("number")) + if pr.get("isDraft") is True: + actions.notice(f"PR #{pr_number} is a draft - skipping.") + return 0 + + info = gh.pr_view_json(pr_number, "labels,mergeable,mergeStateStatus", repo=repo) + data = gh.loads(info) or {} + labels = [l.get("name") for l in (data.get("labels") or [])] + mergeable = data.get("mergeable") or "" + actions.log(f"Labels: {','.join(labels)}") + actions.log(f"Mergeable: {mergeable}") + + decision = evaluate_labels(labels, mergeable) + if not decision.ok: + actions.notice(f"{decision.reason} - skipping.") + return 0 + + if not reverify_is_pure_update(pr_number): + return 0 + + gh.pr_merge_squash(pr_number, repo, delete_branch=False) + actions.log(f"Auto-merged PR #{pr_number}") + from . import webhooks + webhooks.emit("pr.auto_merged", webhooks.pr_auto_merged(pr_number, [])) + return 0 diff --git a/.github/pluginctl/src/pluginctl/integrations/external_readme.py b/.github/pluginctl/src/pluginctl/integrations/external_readme.py new file mode 100644 index 0000000..eb95ed9 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/integrations/external_readme.py @@ -0,0 +1,157 @@ +"""Open/update the plugin-listing PR in an external repo (port of update-external-readme.yml). + +Reads the releases-branch README, strips the releases-only preamble, rewrites its +relative links to absolute releases-branch URLs, then creates or updates a PR in +the configured target repo. The text-transform helpers are pure and unit-tested; +the PR plumbing uses the GitHub Contents/PR APIs. Skips when the only change is the +timestamp line. +""" + +from __future__ import annotations + +import base64 +import os +import re +import subprocess +from typing import Optional + +from ..core import actions, gh + +BRANCH = "auto/dispatcharr-plugin-readme" +_SOURCE_COMMIT_RE = re.compile(r"Source commit: ([0-9a-f]+)") +_PLUGIN_LINE_RE = re.compile(r"^- [a-z0-9].*@") + + +def extract_metadata(commit_msg: str) -> tuple[str, str]: + """Return (source_commit, plugin_list_block) from the releases-branch commit.""" + m = _SOURCE_COMMIT_RE.search(commit_msg) + source_commit = m.group(1) if m else "" + plugin_lines = [ln for ln in commit_msg.splitlines() if _PLUGIN_LINE_RE.match(ln)] + return source_commit, "\n".join(plugin_lines) + + +def strip_preamble(readme: str, contrib_url: str) -> str: + """Keep the title + a contribute line, drop the intro/Quick Access, keep from Available Plugins.""" + out: list[str] = [] + found = False + for line in readme.splitlines(): + if line.startswith("# Plugin Releases"): + out.append(line) + out.append("") + out.append(f"Want to get your plugin added to this list? Check out the " + f"[plugin repository]({contrib_url}) to learn how to contribute.") + continue + if line.startswith("## Available Plugins"): + found = True + if found: + out.append(line) + return "\n".join(out) + ("\n" if readme.endswith("\n") else "") + + +def rewrite_links(text: str, base_url: str) -> str: + """Rewrite relative ``](./`` links to absolute releases-branch URLs.""" + return text.replace("](./", f"]({base_url}/") + + +def _strip_timestamp(text: str) -> str: + return "\n".join(ln for ln in text.splitlines() if not ln.startswith("*Last updated:")) + + +def run(releases_ref: str = "origin/releases") -> int: + """Read the releases README from ``releases_ref`` and sync it to the target repo. + + Sourcing from a git ref (rather than a working-tree checkout) lets the caller + keep the base branch checked out, where the composite actions and this package + live, while still reading the releases-branch content and its publish commit. + """ + target_repo = os.environ.get("TARGET_REPO", "") + target_path = os.environ.get("TARGET_PATH", "") + source_repo = os.environ.get("SOURCE_REPO") or os.environ.get("GITHUB_REPOSITORY", "") + reviewer = os.environ.get("PR_REVIEWER", "") + + show = subprocess.run(["git", "show", f"{releases_ref}:README.md"], + capture_output=True, text=True) + if show.returncode != 0: + actions.error("README.md not found on the releases branch - has the Publish Plugins workflow run yet?") + return 1 + readme = show.stdout + + commit_msg = subprocess.run(["git", "log", "-1", "--format=%B", releases_ref], + capture_output=True, text=True).stdout + source_commit, plugin_list = extract_metadata(commit_msg) + releases_commit = subprocess.run(["git", "rev-parse", "--short", releases_ref], + capture_output=True, text=True).stdout.strip() + + contrib_url = f"https://github.com/{source_repo}/blob/main/CONTRIBUTING.md" + readme = strip_preamble(readme, contrib_url) + readme = rewrite_links(readme, f"https://github.com/{source_repo}/tree/releases") + + default_branch = gh.api(f"repos/{target_repo}", jq=".default_branch") + if not default_branch: + actions.error(f"Could not resolve default branch for {target_repo}.") + return 1 + + if gh.api(f"repos/{target_repo}/branches/{BRANCH}") is None: + base_sha = gh.api(f"repos/{target_repo}/git/refs/heads/{default_branch}", jq=".object.sha") + gh.api(f"repos/{target_repo}/git/refs", method="POST", + fields={"ref": f"refs/heads/{BRANCH}", "sha": base_sha or ""}) + actions.log(f"Created branch {BRANCH}") + + existing = gh.api(f"repos/{target_repo}/contents/{target_path}?ref={BRANCH}") + file_sha = "" + if existing: + data = gh.loads(existing) or {} + file_sha = data.get("sha") or "" + try: + existing_content = base64.b64decode(data.get("content", "")).decode("utf-8") + except Exception: + existing_content = "" + if _strip_timestamp(readme) == _strip_timestamp(existing_content): + actions.log("README content unchanged (only timestamp differs) - skipping.") + actions.notice("External README not updated: only the timestamp line changed.") + return 0 + + put_fields = { + "message": f"chore: update plugin releases listing (source commit {source_commit})", + "content": base64.b64encode(readme.encode("utf-8")).decode("ascii"), + "branch": BRANCH, + } + if file_sha: + put_fields["sha"] = file_sha + gh.api(f"repos/{target_repo}/contents/{target_path}", method="PUT", fields=put_fields) + + summary = _build_summary(source_repo, source_commit, releases_commit, plugin_list) + existing_pr = gh.pr_list(target_repo, "open", "number", jq=".[0].number // empty") + if existing_pr: + gh.pr_comment_repo(existing_pr, target_repo, f"### Plugin README update\n\n{summary}") + actions.log(f"Added update comment to existing PR #{existing_pr}") + else: + body = (f"Automated update of the plugin releases listings generated from " + f"[`{source_repo}`](https://github.com/{source_repo}).\n\n{summary}") + labels = None + reviewers = [r.lstrip("@").replace(" ", "") for r in reviewer.split(",") if r.strip()] + _pr_create_with_reviewers(target_repo, BRANCH, default_branch, + "chore: update plugin releases listings", body, reviewers) + actions.log("Created new PR") + return 0 + + +def _build_summary(source_repo, source_commit, releases_commit, plugin_list) -> str: + lines = [ + f"**Source commit:** [`{source_commit}`](https://github.com/{source_repo}/commit/{source_commit})", + f"**Releases commit:** [`{releases_commit}`](https://github.com/{source_repo}/commit/{releases_commit})", + ] + if plugin_list: + lines.append("") + lines.append("**Plugins updated:**") + lines.extend(plugin_list.splitlines()) + return "\n".join(lines) + + +def _pr_create_with_reviewers(repo, head, base, title, body, reviewers) -> None: + args = ["gh", "pr", "create", "--repo", repo, "--head", head, "--base", base, + "--title", title, "--body", body] + for r in reviewers: + if r: + args += ["--reviewer", r] + subprocess.run(args, capture_output=True, text=True) diff --git a/.github/pluginctl/src/pluginctl/integrations/webhooks.py b/.github/pluginctl/src/pluginctl/integrations/webhooks.py new file mode 100644 index 0000000..1bd71e7 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/integrations/webhooks.py @@ -0,0 +1,118 @@ +"""Signed webhook emitter (new capability). + +Emits a compact JSON event to ``WEBHOOK_URL`` signed with ``WEBHOOK_SECRET`` via +HMAC-SHA256 over the raw request body. Both settings are optional: with either +unset, :func:`emit` is a silent no-op. A delivery failure never raises, it logs +a ``::warning::``, so the pipeline is never failed by webhook problems. + +Header scheme (documented in docs/webhooks.md for the consuming bot): + X-PluginCtl-Event + X-PluginCtl-Delivery + X-PluginCtl-Signature sha256= +""" + +from __future__ import annotations + +import datetime +import hashlib +import hmac +import json +import os +import urllib.error +import urllib.request +import uuid +from typing import Any, Optional + +from ..core import actions + +USER_AGENT = "pluginctl-webhook/1" + + +def sign(secret: str, body: bytes) -> str: + """Return ``sha256=`` HMAC of ``body`` using ``secret`` (utf-8).""" + digest = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest() + return f"sha256={digest}" + + +def build_envelope(event: str, data: dict, repository: str, actor: str) -> dict: + return { + "event": event, + "delivered_at": datetime.datetime.now(datetime.timezone.utc) + .strftime("%Y-%m-%dT%H:%M:%SZ"), + "repository": repository, + "actor": actor, + "data": data, + } + + +def emit(event: str, data: dict, *, repository: Optional[str] = None, + actor: Optional[str] = None, url: Optional[str] = None, + secret: Optional[str] = None, _opener=None) -> bool: + """Deliver an event. Returns True if sent, False if skipped or failed. + + Configuration falls back to the environment (``WEBHOOK_URL`` / + ``WEBHOOK_SECRET`` / ``GITHUB_REPOSITORY`` / ``GITHUB_ACTOR``) so callers can + pass nothing in CI. + """ + url = url if url is not None else os.environ.get("WEBHOOK_URL", "") + secret = secret if secret is not None else os.environ.get("WEBHOOK_SECRET", "") + repository = repository if repository is not None else os.environ.get("GITHUB_REPOSITORY", "") + actor = actor if actor is not None else os.environ.get("GITHUB_ACTOR", "") + + if not url or not secret: + return False + + envelope = build_envelope(event, data, repository, actor) + body = json.dumps(envelope, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + delivery = str(uuid.uuid4()) + request = urllib.request.Request( + url, + data=body, + method="POST", + headers={ + "Content-Type": "application/json", + "User-Agent": USER_AGENT, + "X-PluginCtl-Event": event, + "X-PluginCtl-Delivery": delivery, + "X-PluginCtl-Signature": sign(secret, body), + }, + ) + opener = _opener or urllib.request.urlopen + try: + with opener(request, timeout=15) as resp: + status = getattr(resp, "status", None) or resp.getcode() + if status and 200 <= int(status) < 300: + actions.log(f"Webhook '{event}' delivered ({delivery}, HTTP {status}).") + return True + actions.warning(f"Webhook '{event}' returned HTTP {status} - ignoring.") + return False + except (urllib.error.URLError, OSError, ValueError) as exc: # never fail the pipeline + actions.warning(f"Webhook '{event}' delivery failed: {exc}") + return False + + +# ---- typed event constructors (thin, so call sites stay readable) ----------- +def pr_validated(pr, author, result, plugins, checks) -> dict: + return {"pr": pr, "author": author, "result": result, + "plugins": plugins, "checks": checks} + + +def pr_closed_unauthorized(pr, author, reason) -> dict: + return {"pr": pr, "author": author, "reason": reason} + + +def pr_quarantined(pr, author, infected) -> dict: + return {"pr": pr, "author": author, "infected": infected} + + +def pr_auto_merged(pr, plugins) -> dict: + return {"pr": pr, "plugins": plugins} + + +def plugin_published(plugin, version, pr, actor) -> dict: + return {"plugin": plugin, "version": version, "pr": pr, "actor": actor} + + +def plugin_yanked(plugin, version, issue, rollback_pr) -> dict: + return {"plugin": plugin, "version": version, "issue": issue, + "rollback_pr": rollback_pr} diff --git a/.github/pluginctl/src/pluginctl/publish/__init__.py b/.github/pluginctl/src/pluginctl/publish/__init__.py new file mode 100644 index 0000000..7134aec --- /dev/null +++ b/.github/pluginctl/src/pluginctl/publish/__init__.py @@ -0,0 +1 @@ +"""Publish pipeline: build ZIPs + GitHub Releases, manifests, and READMEs.""" diff --git a/.github/pluginctl/src/pluginctl/publish/cleanup.py b/.github/pluginctl/src/pluginctl/publish/cleanup.py new file mode 100644 index 0000000..f090a24 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/publish/cleanup.py @@ -0,0 +1,47 @@ +"""Remove releases for deleted plugins and prune old versions (port of cleanup.sh). + +Runs from the releases-branch checkout directory. +""" + +from __future__ import annotations + +import glob +import os +import shutil + +from ..core import actions, gh +from ..core.version import sort_versions_desc + + +def _versioned_tags(all_tags: list[str], plugin_name: str) -> list[str]: + prefix = f"{plugin_name}-" + latest = f"{plugin_name}-latest" + versions = [t[len(prefix):] for t in all_tags + if t.startswith(prefix) and t != latest] + return [f"{prefix}{v}" for v in sort_versions_desc(versions)] + + +def run(repository: str, max_versioned_zips: int = 10) -> int: + all_tags = gh.release_list_tags(repository) + + # Remove releases for deleted plugins (guarded on zips/ existing, as in Bash). + if os.path.isdir("zips"): + for release_dir in sorted(glob.glob("metadata/*/")): + plugin_name = os.path.basename(release_dir.rstrip("/")) + if not os.path.isdir(f"plugins/{plugin_name}"): + actions.log(f" Removing deleted plugin releases: {plugin_name}") + for tag in [t for t in all_tags if t.startswith(f"{plugin_name}-")]: + actions.log(f" Deleting release {tag}") + gh.release_delete(tag, repository) + shutil.rmtree(release_dir, ignore_errors=True) + + # Prune old versioned releases per plugin (keep the newest max_versioned_zips). + for plugin_dir in sorted(glob.glob("plugins/*/")): + plugin_name = os.path.basename(plugin_dir.rstrip("/")) + tags = _versioned_tags(all_tags, plugin_name) + if len(tags) <= max_versioned_zips: + continue + for old_tag in tags[max_versioned_zips:]: + actions.log(f" Removed release {old_tag} (over limit of {max_versioned_zips})") + gh.release_delete(old_tag, repository) + return 0 diff --git a/.github/pluginctl/src/pluginctl/publish/manifest.py b/.github/pluginctl/src/pluginctl/publish/manifest.py new file mode 100644 index 0000000..5a7e8a1 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/publish/manifest.py @@ -0,0 +1,399 @@ +"""Manifest construction, byte-for-byte equivalent to generate-manifest.sh. + +The pure builders here assemble ``dict`` objects in the *exact* key order of the +jq programs in the Bash script and drop ``None`` the way +``with_entries(select(.value != null))`` does, so ``jsonio.dumps`` reproduces the +compact ``jq -c`` output. GPG signing and the "only re-sign when the payload +changed" logic are preserved in :func:`write_manifest_if_changed` / +:func:`sign_manifest`. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import tempfile +from typing import Optional + +from ..core import jsonio +from ..core.jsonio import drop_none + + +def build_version_entry(metadata: dict, url: str, size, canonical_version: str) -> dict: + """A ``versions[]`` entry. + + With metadata present, all fields are populated (nulls dropped). With empty + metadata the Bash else-branch emits only ``{version,url,size}`` using the + tag-derived canonical version. + """ + if metadata: + return drop_none({ + "version": metadata.get("version"), + "commit_sha": metadata.get("commit_sha"), + "commit_sha_short": metadata.get("commit_sha_short"), + "build_timestamp": metadata.get("build_timestamp"), + "last_updated": metadata.get("last_updated"), + "checksum_md5": metadata.get("checksum_md5"), + "checksum_sha256": metadata.get("checksum_sha256"), + "min_dispatcharr_version": metadata.get("min_dispatcharr_version"), + "max_dispatcharr_version": metadata.get("max_dispatcharr_version"), + "source_url": metadata.get("source_url"), + "url": url, + "size": size, + }) + return {"version": canonical_version, "url": url, "size": size} + + +def override_current_min_max(versioned_zips: list[dict], current_version: str, + min_da: str, max_da: str) -> list[dict]: + """Overlay plugin.json min/max onto the current version's entry (metadata-only updates).""" + out = [] + for entry in versioned_zips: + if entry.get("version") == current_version: + merged = dict(entry) + merged["min_dispatcharr_version"] = min_da if min_da != "" else None + merged["max_dispatcharr_version"] = max_da if max_da != "" else None + out.append(drop_none(merged)) + else: + out.append(entry) + return out + + +def build_plugin_entry(plugin_raw: dict, plugin_name: str, latest_url: str, + registry_url: str, registry_name: str, + versioned_zips: list[dict], latest_metadata: dict, + latest_size_kb) -> dict: + """Per-plugin manifest ``.manifest`` payload (metadata//manifest.json).""" + def g(key): + return plugin_raw.get(key) + + latest = None + if latest_metadata: + latest = drop_none({ + "version": latest_metadata.get("version"), + "commit_sha": latest_metadata.get("commit_sha"), + "commit_sha_short": latest_metadata.get("commit_sha_short"), + "build_timestamp": latest_metadata.get("build_timestamp"), + "last_updated": latest_metadata.get("last_updated"), + "checksum_md5": latest_metadata.get("checksum_md5"), + "checksum_sha256": latest_metadata.get("checksum_sha256"), + "min_dispatcharr_version": g("min_dispatcharr_version"), + "max_dispatcharr_version": g("max_dispatcharr_version"), + "source_url": latest_metadata.get("source_url"), + "latest_url": latest_url, + "url": versioned_zips[0]["url"] if versioned_zips else None, + "size": latest_size_kb, + }) + + return drop_none({ + "slug": plugin_name, + "name": g("name"), + "description": g("description"), + "author": g("author"), + "maintainers": g("maintainers"), + "license": g("license"), + "deprecated": True if g("deprecated") is True else None, + "source_type": "external" if g("source_type") == "external" else None, + "source_url": g("source_url"), + "repo_url": g("repo_url"), + "discord_thread": g("discord_thread"), + "registry_url": registry_url, + "registry_name": registry_name, + "last_updated": latest_metadata.get("last_updated") if latest_metadata else None, + "latest": latest, + "versions": versioned_zips, + }) + + +def trim_description(desc: str) -> str: + """``desc[:197] + "..."`` when longer than 200 chars, else unchanged.""" + return (desc[:197] + "...") if len(desc) > 200 else desc + + +def build_root_entry(plugin_raw: dict, plugin_name: str, latest_metadata: dict, + latest_size_kb, min_da: str, max_da: str, + latest_url: str, manifest_url: str) -> dict: + """Compact entry for the root manifest's ``plugins[]``.""" + license_id = plugin_raw.get("license") or "" + return drop_none({ + "slug": plugin_name, + "name": plugin_raw.get("name") or "", + "description": trim_description(plugin_raw.get("description") or ""), + "manifest_url": manifest_url, + "author": plugin_raw.get("author") or "", + "license": license_id if license_id != "" else None, + "deprecated": True if plugin_raw.get("deprecated") is True else None, + "last_updated": latest_metadata.get("last_updated") if latest_metadata else None, + "latest_version": latest_metadata.get("version") if latest_metadata else None, + "latest_md5": latest_metadata.get("checksum_md5") if latest_metadata else None, + "latest_sha256": latest_metadata.get("checksum_sha256") if latest_metadata else None, + "latest_url": latest_url, + "latest_size": latest_size_kb if _num(latest_size_kb) > 0 else None, + "min_dispatcharr_version": min_da if min_da != "" else None, + "max_dispatcharr_version": max_da if max_da != "" else None, + }) + + +def build_root_manifest(registry_url: str, registry_name: str, root_url: str, + root_entries: list[dict]) -> dict: + return { + "registry_url": registry_url, + "registry_name": registry_name, + "root_url": root_url, + "plugins": root_entries, + } + + +def _num(value) -> float: + try: + return float(value) + except (TypeError, ValueError): + return 0.0 + + +# ---- on-disk wrapper + GPG signing ------------------------------------------ +def write_manifest_if_changed(dest: str, payload: dict, generated_at: str) -> bool: + """Write ``{generated_at, manifest}`` only if the compact payload changed. + + Returns True if written, False if skipped (unchanged), mirroring the Bash + return codes that gate re-signing. + """ + new_compact = jsonio.dumps(payload) + if os.path.isfile(dest): + try: + with open(dest, encoding="utf-8") as fh: + existing = json.load(fh) + if jsonio.dumps(existing.get("manifest")) == new_compact: + return False + except (OSError, ValueError): + pass + wrapper = {"generated_at": generated_at, "manifest": json.loads(new_compact)} + with open(dest, "w", encoding="utf-8") as fh: + fh.write(jsonio.dumps(wrapper)) + return True + + +def sig_is_current(file: str, gpg_key_id: str) -> bool: + """True when the embedded ``.signature`` was issued by ``gpg_key_id``.""" + try: + with open(file, encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError): + return False + sig = data.get("signature") + if not sig: + return False + with tempfile.NamedTemporaryFile("w", suffix=".asc", delete=False) as tmp: + tmp.write(sig + "\n") + tmp_path = tmp.name + try: + out = subprocess.run(["gpg", "--list-packets", tmp_path], + capture_output=True, text=True).stdout + finally: + os.unlink(tmp_path) + m = re.search(r"issuer key ID ([A-Fa-f0-9]{16})", out) + if not m: + return False + return m.group(1).upper() == (gpg_key_id or "").upper() + + +def sign_manifest(file: str, gpg_key_id: str, passphrase: Optional[str]) -> bool: + """Sign the ``.manifest`` payload and embed the armored signature. False on failure.""" + if not gpg_key_id: + return True + with open(file, encoding="utf-8") as fh: + data = json.load(fh) + payload = jsonio.dumps(data["manifest"]) + args = ["gpg", "--batch", "--yes", "--armor", "--detach-sign", + "--local-user", gpg_key_id, "--output", "-"] + if passphrase: + args += ["--passphrase", passphrase, "--pinentry-mode", "loopback"] + proc = subprocess.run(args, input=payload, capture_output=True, text=True) + sig = proc.stdout.rstrip("\n") # command-substitution parity: no trailing newline + if proc.returncode != 0 or not sig: + return False + data["signature"] = sig + with open(file, "w", encoding="utf-8") as fh: + fh.write(jsonio.dumps(data)) + return True + + +# ---- GPG key setup + full generation orchestrator --------------------------- +def import_gpg_key(private_key: str) -> tuple[str, bool]: + """Import the armored key and return (key_id, signing_failed). + + Empty ``private_key`` -> ("", False) (signing disabled, not a failure). + """ + if not private_key: + from ..core import actions + actions.log("GPG_PRIVATE_KEY not set - signatures will be skipped.") + return "", False + subprocess.run(["gpg", "--batch", "--import"], input=private_key, + capture_output=True, text=True) + out = subprocess.run(["gpg", "--list-secret-keys", "--keyid-format", "LONG"], + capture_output=True, text=True).stdout + key_id = "" + for line in out.splitlines(): + if line.startswith("sec"): + parts = line.split() + if len(parts) >= 2 and "/" in parts[1]: + key_id = parts[1].split("/", 1)[1] + break + from ..core import actions + if key_id: + actions.log(f"GPG signing enabled (key: {key_id})") + return key_id, False + actions.warning("GPG key import succeeded but no usable secret key found - signatures will be skipped.") + return "", True + + +def _canonical(version: str) -> str: + return re.sub(r"-[0-9]+$", "", version) + + +def strip_signatures() -> None: + """Remove ``.signature`` from every manifest (no key configured or signing failed).""" + import glob as _glob + for f in _glob.glob("metadata/*/manifest.json") + ["manifest.json"]: + if not os.path.isfile(f): + continue + try: + with open(f, encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError): + continue + if "signature" in data: + del data["signature"] + with open(f, "w", encoding="utf-8") as fh: + fh.write(jsonio.dumps(data)) + + +def generate(source_branch: str, releases_branch: str, repository: str, + build_meta_dir: str) -> int: + """Full generate-manifest.sh: per-plugin manifests, root manifest, signing.""" + import datetime + import glob + from ..core import actions, gh + + generated_at = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + registry_url = f"https://github.com/{repository}" + registry_name = repository + root_url = f"https://github.com/{repository}/releases/download" + raw_releases_url = f"https://raw.githubusercontent.com/{repository}/{releases_branch}" + + key_id, signing_failed = import_gpg_key(os.environ.get("GPG_PRIVATE_KEY", "")) + passphrase = os.environ.get("GPG_PASSPHRASE") or None + + def maybe_sign(path: str) -> None: + nonlocal signing_failed + if not sign_manifest(path, key_id, passphrase): + actions.warning(f"GPG signing failed for {path} - all signatures will be removed.") + signing_failed = True + + def sign_if_needed(path: str, written: bool) -> None: + if written: + maybe_sign(path) + elif key_id and not signing_failed and not sig_is_current(path, key_id): + maybe_sign(path) + + all_tags = gh.release_list_tags(repository) + root_entries: list[dict] = [] + plugin_count = 0 + + for plugin_dir in sorted(glob.glob("plugins/*/")): + plugin_file = os.path.join(plugin_dir, "plugin.json") + if not os.path.isfile(plugin_file): + continue + plugin_name = os.path.basename(plugin_dir.rstrip("/")) + plugin_key = plugin_name.replace("-", "_") + with open(plugin_file, encoding="utf-8") as fh: + raw = json.load(fh) + current_version = str(raw.get("version")) + min_da = str(raw.get("min_dispatcharr_version") or "") + max_da = str(raw.get("max_dispatcharr_version") or "") + unlisted = raw.get("unlisted") is True + actions.log(f" {plugin_name}") + + existing_manifest_file = f"metadata/{plugin_name}/manifest.json" + os.makedirs(f"metadata/{plugin_name}", exist_ok=True) + existing_manifest = None + if os.path.isfile(existing_manifest_file): + try: + with open(existing_manifest_file, encoding="utf-8") as fh: + existing_manifest = json.load(fh) + except ValueError: + existing_manifest = None + + from ..core.version import sort_versions_desc + prefix = f"{plugin_name}-" + raw_versions = [t[len(prefix):] for t in all_tags + if t.startswith(prefix) and t != f"{plugin_name}-latest"] + versioned_tags = [f"{prefix}{v}" for v in sort_versions_desc(raw_versions)] + + versioned_zips: list[dict] = [] + latest_metadata: dict = {} + latest_zip_version = "" + latest_size_kb = 0 + latest_size_set = False + + for release_tag in versioned_tags: + zip_version = release_tag[len(prefix):] + zip_url = f"{plugin_name}-{zip_version}/{plugin_name}-{zip_version}.zip" + canonical_version = _canonical(zip_version) + + metadata: dict = {} + fresh = os.path.join(build_meta_dir, plugin_key, f"{plugin_key}-{canonical_version}.json") + if build_meta_dir and os.path.isfile(fresh): + with open(fresh, encoding="utf-8") as fh: + metadata = json.load(fh) + elif existing_manifest is not None: + for entry in ((existing_manifest.get("manifest") or {}).get("versions") or []): + if entry.get("version") == canonical_version: + metadata = entry + break + + zip_size_kb = 0 + if metadata: + zip_size_kb = metadata.get("size_kb", metadata.get("size", 0)) or 0 + if not latest_size_set: + latest_size_kb = zip_size_kb + latest_size_set = True + + versioned_zips.append(build_version_entry(metadata, zip_url, zip_size_kb, canonical_version)) + if metadata and not latest_metadata: + latest_metadata = metadata + latest_zip_version = zip_version + + latest_url = "" + if latest_zip_version: + latest_url = f"{plugin_name}-{latest_zip_version}/{plugin_name}-{latest_zip_version}.zip" + + versioned_zips = override_current_min_max(versioned_zips, current_version, min_da, max_da) + + plugin_entry = build_plugin_entry(raw, plugin_name, latest_url, registry_url, + registry_name, versioned_zips, latest_metadata, + latest_size_kb) + written = write_manifest_if_changed(existing_manifest_file, plugin_entry, generated_at) + sign_if_needed(existing_manifest_file, written) + + if unlisted: + continue + + manifest_url = f"{raw_releases_url}/metadata/{plugin_name}/manifest.json" + root_entries.append(build_root_entry(raw, plugin_name, latest_metadata, + latest_size_kb, min_da, max_da, + latest_url, manifest_url)) + plugin_count += 1 + + inner_root = build_root_manifest(registry_url, registry_name, root_url, root_entries) + written = write_manifest_if_changed("manifest.json", inner_root, generated_at) + sign_if_needed("manifest.json", written) + + if signing_failed or not key_id: + actions.warning("Removing all manifest signatures (no GPG key configured or signing failed).") + strip_signatures() + + actions.log(f"Generated manifest.json with {plugin_count} plugin(s).") + return 0 diff --git a/.github/pluginctl/src/pluginctl/publish/readmes.py b/.github/pluginctl/src/pluginctl/publish/readmes.py new file mode 100644 index 0000000..9885723 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/publish/readmes.py @@ -0,0 +1,440 @@ +"""Per-plugin and root READMEs for the releases branch. + +Ports plugin-readmes.sh and releases-readme.sh. Output is kept identical to the +Bash except for one intentional correction, called out in the plan: the +deprecated-plugin "All Versions" count now comes from the per-plugin manifest's +``versions[]`` (same source the active section uses) instead of the removed +``zips//*.zip`` path, which always rendered 0. +""" + +from __future__ import annotations + +import datetime +import glob +import json +import os +from typing import Optional + +from ..core import git + + +def fmt_date(value: str) -> str: + """ISO8601 -> "Mon DD YYYY, HH:MM UTC"; returns the input unchanged on failure.""" + if not value: + return value + try: + text = value.replace("Z", "+00:00") + dt = datetime.datetime.fromisoformat(text) + if dt.tzinfo is not None: + dt = dt.astimezone(datetime.timezone.utc) + return dt.strftime("%b %d %Y, %H:%M UTC") + except ValueError: + return value + + +def shields_encode(s: str) -> str: + """shields.io path escaping: _ -> __, - -> --, space -> _ (order matters).""" + s = s.replace("_", "__") + s = s.replace("-", "--") + s = s.replace(" ", "_") + return s + + +def _load_json(path: str) -> Optional[dict]: + try: + with open(path, encoding="utf-8") as fh: + return json.load(fh) + except (OSError, ValueError): + return None + + +def _g(data: Optional[dict], key: str, default: str = "") -> str: + if not data: + return default + val = data.get(key) + return default if val is None else str(val) + + +# --------------------------------------------------------------------------- # +# Per-plugin README (metadata//README.md) +# --------------------------------------------------------------------------- # +def render_plugin_readme(plugin_name: str, plugin_raw: dict, manifest: dict, + root_url: str, repository: str, source_branch: str, + last_updated: str, has_readme: bool, + source_readme: str = "") -> str: + name = _g(plugin_raw, "name") + description = _g(plugin_raw, "description") + author = _g(plugin_raw, "author") + maintainers = ", ".join(str(m) for m in (plugin_raw.get("maintainers") or [])) + repo_url = _g(plugin_raw, "repo_url") + discord_thread = _g(plugin_raw, "discord_thread") + license_id = _g(plugin_raw, "license") + min_da = _g(plugin_raw, "min_dispatcharr_version") + max_da = _g(plugin_raw, "max_dispatcharr_version") + version = _g(plugin_raw, "version") + + manifest_body = manifest.get("manifest", {}) if manifest else {} + latest = manifest_body.get("latest") or {} + latest_url_path = latest.get("latest_url") or "" + latest_full_url = f"{root_url}/{latest_url_path}" if root_url and latest_url_path else "" + + L: list[str] = [] + L.append("[Back to All Plugins](../../README.md)") + L.append("") + L.append(f"# {name}") + L.append("") + L.append(f"**Version:** `{version}` | **Author:** {author} | **Last Updated:** {fmt_date(last_updated)}") + L.append("") + L.append(description) + L.append("") + + badges = _badges(license_id, discord_thread, repo_url) + if badges: + L.append(badges) + L.append("") + compat = _compat_badges(min_da, max_da) + if compat: + L.append(compat) + L.append("") + + L.append("## Downloads") + L.append("") + L.append("### Latest Release") + L.append("") + if latest_full_url: + L.append(f"- **Download:** [`{plugin_name}-latest.zip`]({latest_full_url})") + build_ts = latest.get("build_timestamp") or "" + commit_sha = latest.get("commit_sha") or "" + commit_short = latest.get("commit_sha_short") or "" + md5 = latest.get("checksum_md5") or "" + sha256 = latest.get("checksum_sha256") or "" + if build_ts: + L.append(f"- **Built:** {fmt_date(build_ts)}") + if commit_sha: + L.append(f"- **Source Commit:** [`{commit_short}`](https://github.com/{repository}/commit/{commit_sha})") + if md5 or sha256: + L.append("") + L.append("**Checksums:**") + L.append("```") + if md5: + L.append(f"MD5: {md5}") + if sha256: + L.append(f"SHA256: {sha256}") + L.append("```") + + L.append("") + L.append("### All Versions") + L.append("") + L.append("| Version | Download | Built | Commit | MD5 | SHA256 |") + L.append("|---------|----------|-------|--------|-----|--------|") + for ver in (manifest_body.get("versions") or []): + v = ver.get("version") or "" + if not v: + continue + url_path = ver.get("url") or "" + full_url = f"{root_url}/{url_path}" if root_url and url_path else "" + commit_sha = ver.get("commit_sha") or "" + commit_short = ver.get("commit_sha_short") or "" + build_ts = ver.get("build_timestamp") or "" + md5 = ver.get("checksum_md5") or "" + sha256 = ver.get("checksum_sha256") or "" + build_date = fmt_date(build_ts) + commit_cell = f"[`{commit_short}`](https://github.com/{repository}/commit/{commit_sha})" if commit_sha else "-" + download_cell = f"[Download]({full_url})" if full_url else "-" + L.append(f"| `{v}` | {download_cell} | {build_date or '-'} | {commit_cell} | {md5 or '-'} | {sha256 or '-'} |") + + L.append("") + L.append("---") + L.append("") + footer = "" + if maintainers: + footer = f"**Maintainers:** {maintainers} | " + footer += (f"**Source:** [Browse Plugin]" + f"(https://github.com/{repository}/tree/{source_branch}/plugins/{plugin_name})") + L.append(footer) + L.append("") + L.append("**Metadata:** [View full manifest](./manifest.json)") + + if has_readme: + L.append("") + L.append("---") + L.append("") + L.append("## Plugin README") + L.append("") + L.append(source_readme.rstrip("\n")) + + return "\n".join(L) + "\n" + + +def _badges(license_id: str, discord_thread: str, repo_url: str) -> str: + badges = "" + if license_id: + badges = (f"[![License: {license_id}]" + f"(https://img.shields.io/badge/License-{shields_encode(license_id)}-blue?style=flat-square)]" + f"(https://spdx.org/licenses/{license_id}.html)") + if discord_thread: + if badges: + badges += " " + badges += ("[![Discord](https://img.shields.io/badge/Discord-Discussion-5865F2?style=flat-square&logo=discord&logoColor=white)]" + f"({discord_thread})") + if repo_url: + if badges: + badges += " " + badges += ("[![Repository](https://img.shields.io/badge/GitHub-Repository-181717?style=flat-square&logo=github&logoColor=white)]" + f"({repo_url})") + return badges + + +def _compat_badges(min_da: str, max_da: str) -> str: + if not (min_da or max_da): + return "" + out = "" + if min_da: + out = f"![Dispatcharr min](https://img.shields.io/badge/Dispatcharr_min-{shields_encode(min_da)}-brightgreen?style=flat-square)" + if max_da: + if out: + out += " " + out += f"![Dispatcharr max](https://img.shields.io/badge/Dispatcharr_max-{shields_encode(max_da)}-orange?style=flat-square)" + return out + + +def version_count_from_manifest(manifest_file: str) -> int: + """Deprecated-plugin fix: derive count from the manifest's versions[] (was ls zips/).""" + data = _load_json(manifest_file) + if not data: + return 0 + versions = (data.get("manifest") or {}).get("versions") + return len(versions) if isinstance(versions, list) else 0 + + +# --------------------------------------------------------------------------- # +# Root releases README (README.md on the releases branch) +# --------------------------------------------------------------------------- # +import re as _re + + +def anchor_for(name: str) -> str: + """GitHub-style heading anchor: lowercase, non [a-z0-9-] -> -, collapse runs.""" + a = name.lower() + a = _re.sub(r"[^a-z0-9-]", "-", a) + a = _re.sub(r"-+", "-", a) + return a + + +def table_row(plugin_raw: dict, deprecated_pass: bool) -> str: + name = _g(plugin_raw, "name") + version = _g(plugin_raw, "version") + author = _g(plugin_raw, "author") + description = _g(plugin_raw, "description") + license_cell = _g(plugin_raw, "license", "-") or "-" + suffix = " (deprecated)" if deprecated_pass else "" + return f"| [`{name}`](#{anchor_for(name)}){suffix} | `{version}` | {author} | {license_cell} | {description} |" + + +def render_plugin_block(*, is_deprecated: bool, plugin_name: str, plugin_raw: dict, + manifest: Optional[dict], last_updated: str, commit_sha: str, + commit_sha_short: str, version_count, repository: str, + source_branch: str, releases_branch: str, root_url: str, + has_source_readme: bool) -> str: + name = _g(plugin_raw, "name") + version = _g(plugin_raw, "version") + author = _g(plugin_raw, "author") + description = _g(plugin_raw, "description") + maintainers = ", ".join(str(m) for m in (plugin_raw.get("maintainers") or [])) + license_id = _g(plugin_raw, "license") + min_da = _g(plugin_raw, "min_dispatcharr_version") + max_da = _g(plugin_raw, "max_dispatcharr_version") + repo_url = _g(plugin_raw, "repo_url") + discord_thread = _g(plugin_raw, "discord_thread") + + latest_url_path = "" + if manifest: + latest_url_path = ((manifest.get("manifest") or {}).get("latest") or {}).get("latest_url") or "" + zip_url = f"{root_url}/{latest_url_path}" if root_url and latest_url_path else "" + + source_url = f"https://github.com/{repository}/tree/{source_branch}/plugins/{plugin_name}" + readme_url = f"https://github.com/{repository}/blob/{source_branch}/plugins/{plugin_name}/README.md" + releases_readme_url = f"https://github.com/{repository}/blob/{releases_branch}/metadata/{plugin_name}/README.md" + commit_url = f"https://github.com/{repository}/commit/{commit_sha}" + releases_dir = f"./metadata/{plugin_name}" + + suffix = " (deprecated)" if is_deprecated else "" + + L: list[str] = [] + L.append(f"### [{name}]({releases_readme_url}){suffix}") + L.append("") + L.append(f"**Version:** `{version}` | **Author:** {author} | **Last Updated:** {fmt_date(last_updated)}") + L.append("") + L.append(description) + L.append("") + badges = _badges(license_id, discord_thread, repo_url) + if badges: + L.append(badges) + L.append("") + compat = _compat_badges(min_da, max_da) + if compat: + L.append(compat) + L.append("") + L.append("**Downloads:**") + if zip_url: + L.append(f"- [Latest Release (`{version}`)]({zip_url})") + L.append(f"- [All Versions ({version_count} available)]({releases_dir})") + L.append("") + footer = "" + if maintainers: + footer = f"**Maintainers:** {maintainers} | " + footer += f"**Source:** [Browse]({source_url})" + if has_source_readme: + footer += f" | [README]({readme_url})" + footer += f" | **Last Change:** [`{commit_sha_short}`]({commit_url})" + L.append(footer) + L.append("") + L.append("---") + L.append("") + return "\n".join(L) + "\n" + + +# --------------------------------------------------------------------------- # +# Orchestrators (IO): called from the releases-branch checkout directory +# --------------------------------------------------------------------------- # +def _now_iso() -> str: + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _plugin_last_updated(source_branch: str, plugin_dir: str) -> str: + return git.log_format("%cI", f"origin/{source_branch}", plugin_dir) or _now_iso() + + +def generate_plugin_readmes(source_branch: str, repository: str) -> None: + """Write metadata//README.md for every plugin (plugin-readmes.sh).""" + root = _load_json("manifest.json") or {} + root_url = (root.get("manifest") or {}).get("root_url") or "" + for plugin_dir in sorted(glob.glob("plugins/*/")): + plugin_name = os.path.basename(plugin_dir.rstrip("/")) + plugin_file = os.path.join(plugin_dir, "plugin.json") + plugin_raw = _load_json(plugin_file) + if plugin_raw is None: + continue + manifest_file = f"metadata/{plugin_name}/manifest.json" + manifest = _load_json(manifest_file) + if manifest is None: + continue + last_updated = _plugin_last_updated(source_branch, plugin_dir) + source_readme_path = os.path.join(plugin_dir, "README.md") + has_readme = os.path.isfile(source_readme_path) + source_readme = _read(source_readme_path) if has_readme else "" + content = render_plugin_readme( + plugin_name, plugin_raw, manifest, root_url, repository, source_branch, + last_updated, has_readme, source_readme) + with open(f"metadata/{plugin_name}/README.md", "w", encoding="utf-8") as fh: + fh.write(content) + + +def generate_releases_readme(source_branch: str, releases_branch: str, repository: str) -> None: + """Write the root README.md (releases-readme.sh).""" + root = _load_json("manifest.json") or {} + root_url = (root.get("manifest") or {}).get("root_url") or "" + + plugin_dirs = sorted(glob.glob("plugins/*/")) + raws: dict[str, dict] = {} + for pd in plugin_dirs: + pname = os.path.basename(pd.rstrip("/")) + raw = _load_json(os.path.join(pd, "plugin.json")) + if raw is not None: + raws[pname] = raw + + L: list[str] = [] + L.append("# Plugin Releases") + L.append("") + L.append("This branch contains all published plugin releases.") + L.append("") + L.append("## Quick Access") + L.append("") + L.append("- [manifest.json](./manifest.json) - Complete plugin registry with metadata") + L.append("- [metadata/](./metadata/) - Per-plugin manifests and READMEs") + L.append("") + L.append("## Available Plugins") + L.append("") + L.append("| Plugin | Version | Author | License | Description |") + L.append("|--------|---------|-------|---------|-------------|") + + for pass_name in ("active", "deprecated"): + for pd in plugin_dirs: + pname = os.path.basename(pd.rstrip("/")) + raw = raws.get(pname) + if raw is None: + continue + deprecated = raw.get("deprecated") is True + if raw.get("unlisted") is True: + continue + if pass_name == "active" and deprecated: + continue + if pass_name == "deprecated" and not deprecated: + continue + L.append(table_row(raw, pass_name == "deprecated")) + + L.append("") + L.append("---") + L.append("") + + def emit_block(pname: str, raw: dict, is_deprecated: bool) -> None: + pd = f"plugins/{pname}/" + manifest = _load_json(f"metadata/{pname}/manifest.json") + last_updated = _plugin_last_updated(source_branch, pd) + commit_sha = git.log_format("%H", f"origin/{source_branch}", pd) or "unknown" + commit_short = git.log_format("%h", f"origin/{source_branch}", pd) or "unknown" + version_count = version_count_from_manifest(f"metadata/{pname}/manifest.json") + has_source_readme = os.path.isfile(f"plugins/{pname}/README.md") + L.append(render_plugin_block( + is_deprecated=is_deprecated, plugin_name=pname, plugin_raw=raw, + manifest=manifest, last_updated=last_updated, commit_sha=commit_sha, + commit_sha_short=commit_short, version_count=version_count, + repository=repository, source_branch=source_branch, + releases_branch=releases_branch, root_url=root_url, + has_source_readme=has_source_readme)) + + # Active detailed sections + for pd in plugin_dirs: + pname = os.path.basename(pd.rstrip("/")) + raw = raws.get(pname) + if raw is None or raw.get("deprecated") is True or raw.get("unlisted") is True: + continue + emit_block(pname, raw, False) + + has_deprecated = any( + r.get("deprecated") is True for r in raws.values()) + if has_deprecated: + L.append("") + L.append("## Deprecated Plugins") + L.append("") + L.append("These plugins are deprecated and may be removed in the future.") + L.append("") + for pd in plugin_dirs: + pname = os.path.basename(pd.rstrip("/")) + raw = raws.get(pname) + if raw is None or raw.get("deprecated") is not True or raw.get("unlisted") is True: + continue + emit_block(pname, raw, True) + + L.append("## Using the Manifest") + L.append("") + L.append("Fetch `manifest.json` to programmatically access plugin metadata and download URLs:") + L.append("") + L.append("```bash") + L.append(f"curl https://raw.githubusercontent.com/{repository}/{releases_branch}/manifest.json") + L.append("```") + L.append("") + L.append("---") + L.append("") + L.append(f"*Last updated: {datetime.datetime.now(datetime.timezone.utc).strftime('%b %d %Y, %H:%M UTC')}*") + + with open("README.md", "w", encoding="utf-8") as fh: + fh.write("\n".join(L) + "\n") + + +def _read(path: str) -> str: + try: + with open(path, encoding="utf-8") as fh: + return fh.read() + except OSError: + return "" diff --git a/.github/pluginctl/src/pluginctl/publish/run.py b/.github/pluginctl/src/pluginctl/publish/run.py new file mode 100644 index 0000000..ea002d1 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/publish/run.py @@ -0,0 +1,215 @@ +"""Publish orchestrator (port of run.sh). + +Clones the repo, sets up the releases branch, runs the build/cleanup/manifest/readme +phases, then commits and pushes, skipping commits whose only diff is timestamp +noise. Validated end to end in a fork per the rollout plan; the phase logic it +calls is unit-tested individually. +""" + +from __future__ import annotations + +import os +import subprocess +import tempfile + +from ..core import actions, git as _git_mod +from . import cleanup, manifest, readmes, zips + +RELEASES_BRANCH = "releases" +MAX_VERSIONED_ZIPS = 10 +RELEASES_BRANCH_VERSION = "3" + + +def _git(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run(["git", *args], check=check, capture_output=True, text=True) + + +def _configure_identity(repository: str, app_slug: str) -> None: + if app_slug: + from ..core import gh + bot_user_id = gh.api(f"/users/{app_slug}%5Bbot%5D", jq=".id") or "" + _git("config", "user.name", f"{app_slug}[bot]") + if bot_user_id: + _git("config", "user.email", + f"{bot_user_id}+{app_slug}[bot]@users.noreply.github.com") + else: + _git("config", "user.email", f"{app_slug}[bot]@users.noreply.github.com") + else: + _git("config", "user.name", "github-actions[bot]") + _git("config", "user.email", + "41898282+github-actions[bot]@users.noreply.github.com") + + +def run(source_branch: str) -> int: + repository = os.environ["GITHUB_REPOSITORY"] + token = os.environ["GITHUB_TOKEN"] + app_slug = os.environ.get("APP_SLUG", "") + force_rebuild = os.environ.get("FORCE_REBUILD", "false") == "true" + force_plugin = os.environ.get("FORCE_REBUILD_PLUGIN", "") + remote = f"https://x-access-token:{token}@github.com/{repository}.git" + + actions.log(f"Publishing plugins from {source_branch} to {RELEASES_BRANCH}") + + workdir = tempfile.mkdtemp() + build_meta_dir = tempfile.mkdtemp() + start_cwd = os.getcwd() + try: + _git("clone", "--no-checkout", remote, f"{workdir}/repo") + os.chdir(f"{workdir}/repo") + _configure_identity(repository, app_slug) + _setup_releases_branch(repository, source_branch, token, remote, + force_rebuild, force_plugin) + + # Clean regenerated top-level artifacts and fetch source plugins. + for f in ("manifest.json", "README.md"): + if os.path.exists(f): + os.remove(f) + _git("fetch", "origin", source_branch) + _git("checkout", f"origin/{source_branch}", "--", "plugins") + os.makedirs("zips", exist_ok=True) + + if not force_rebuild: + current = _read("REPO_VER") + if current != RELEASES_BRANCH_VERSION: + actions.error("Releases branch version mismatch.") + actions.error(f" Expected : {RELEASES_BRANCH_VERSION}") + actions.error(f" Found : {current or '(none - migration not run)'}") + actions.error("Run the 'Migrate Releases to GitHub Releases' workflow first, then re-run.") + return 1 + + actions.log("\n=== Building ZIPs ===") + rc = zips.run(source_branch, repository, build_meta_dir) + if rc != 0: + return rc + actions.log("\n=== Cleaning up old releases ===") + cleanup.run(repository, MAX_VERSIONED_ZIPS) + actions.log("\n=== Generating manifests ===") + manifest.generate(source_branch, RELEASES_BRANCH, repository, build_meta_dir) + actions.log("\n=== Generating per-plugin READMEs ===") + readmes.generate_plugin_readmes(source_branch, repository) + actions.log("\n=== Generating releases README ===") + readmes.generate_releases_readme(source_branch, RELEASES_BRANCH, repository) + + return _commit_and_push(repository, source_branch, remote) + finally: + os.chdir(start_cwd) + subprocess.run(["rm", "-rf", workdir, build_meta_dir], check=False) + + +def _setup_releases_branch(repository, source_branch, token, remote, + force_rebuild, force_plugin) -> None: + from ..core import gh + branch_exists = _git("ls-remote", "--exit-code", "--heads", "origin", + RELEASES_BRANCH, check=False).returncode == 0 + + if force_rebuild and force_plugin: + if branch_exists: + _git("checkout", RELEASES_BRANCH) + _git("pull", "origin", RELEASES_BRANCH, check=False) + else: + _orphan_init() + actions.log(f"Targeted force rebuild: deleting GitHub Releases for {force_plugin}") + for tag in gh.release_list_tags(repository): + if tag.startswith(f"{force_plugin}-"): + gh.release_delete(tag, repository) + mf = f"metadata/{force_plugin}/manifest.json" + if os.path.exists(mf): + os.remove(mf) + elif force_rebuild: + actions.log(f"Force rebuild requested - deleting all plugin GitHub Releases and resetting {RELEASES_BRANCH}.") + _git("fetch", "origin", source_branch, check=False) + _git("checkout", f"origin/{source_branch}", "--", "plugins", check=False) + if os.path.isdir("plugins"): + import glob + for plugin_dir in glob.glob("plugins/*/"): + pname = os.path.basename(plugin_dir.rstrip("/")) + for tag in gh.release_list_tags(repository): + if tag.startswith(f"{pname}-"): + gh.release_delete(tag, repository) + subprocess.run(["rm", "-rf", "plugins"], check=False) + _git("checkout", "--orphan", RELEASES_BRANCH) + _git("rm", "-rf", ".", check=False) + _git("commit", "--allow-empty", "-m", f"Initialize {RELEASES_BRANCH} branch (force rebuild)") + _git("push", "--force", remote, RELEASES_BRANCH) + elif branch_exists: + _git("checkout", RELEASES_BRANCH) + _git("pull", "origin", RELEASES_BRANCH, check=False) + else: + _orphan_init() + + +def _orphan_init() -> None: + _git("checkout", "--orphan", RELEASES_BRANCH) + _git("rm", "-rf", ".", check=False) + _git("commit", "--allow-empty", "-m", f"Initialize {RELEASES_BRANCH} branch") + + +def _commit_and_push(repository, source_branch, remote) -> int: + actions.log("\n=== Committing ===") + subprocess.run(["rm", "-rf", "plugins"], check=False) + _git("rm", "-rf", "--cached", "plugins", check=False) + with open("REPO_VER", "w", encoding="utf-8") as fh: + fh.write(RELEASES_BRANCH_VERSION + "\n") + _git("add", "metadata", "manifest.json", "README.md", "REPO_VER") + + if _git("diff", "--cached", "--quiet", check=False).returncode == 0: + actions.log("No changes to commit.") + return 0 + + if _only_timestamps(): + actions.log("No meaningful changes (only timestamps updated) - skipping commit.") + return 0 + + source_commit = _git("rev-parse", "--short", f"origin/{source_branch}").stdout.strip() + plugin_list = "" + changed = _read("changed_plugins.txt") + if changed.strip(): + plugin_list = "\n\n" + "\n".join(f"- {line}" for line in changed.splitlines() if line) + _git("commit", "-m", f"Publish plugin updates from {source_branch}\n\n" + f"Source commit: {source_commit}{plugin_list}") + _git("push", remote, RELEASES_BRANCH) + actions.log(f"Successfully published to {RELEASES_BRANCH}") + _emit_published(repository, changed) + return 0 + + +def _emit_published(repository: str, changed: str) -> None: + """Emit one plugin.published event per changed plugin (best-effort).""" + from ..integrations import webhooks + actor = os.environ.get("GITHUB_ACTOR", "") + for line in changed.splitlines(): + if "@" not in line: + continue + plugin_key, _, version = line.partition("@") + webhooks.emit("plugin.published", + webhooks.plugin_published(plugin_key.replace("_", "-"), version, None, actor)) + + +def _only_timestamps() -> bool: + """True when the staged diff is only README timestamp / manifest generated_at noise.""" + names = _git("diff", "--cached", "--name-only").stdout.splitlines() + for changed_file in names: + if changed_file == "README.md": + if _strip_lines(":README.md", "*Last updated:") != \ + _strip_lines("HEAD:README.md", "*Last updated:"): + return False + elif changed_file == "manifest.json": + if _strip_lines(":manifest.json", '"generated_at"') != \ + _strip_lines("HEAD:manifest.json", '"generated_at"'): + return False + else: + return False + return True + + +def _strip_lines(ref: str, needle: str) -> str: + out = _git("show", ref, check=False).stdout + return "\n".join(ln for ln in out.splitlines() if needle not in ln) + + +def _read(path: str) -> str: + try: + with open(path, encoding="utf-8") as fh: + return fh.read().strip() + except OSError: + return "" diff --git a/.github/pluginctl/src/pluginctl/publish/yank.py b/.github/pluginctl/src/pluginctl/publish/yank.py new file mode 100644 index 0000000..fdc8963 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/publish/yank.py @@ -0,0 +1,264 @@ +"""Yank a plugin version from the releases branch (port of yank-version.sh). + +Removes a specific version's GitHub Release, regenerates manifests/READMEs, and, +when the yanked version was the latest (or only) one, opens a rollback PR against +the source branch so it is not republished. Authorized by an open issue whose +number is cited and (for non-latest yanks) closed. Env-driven, validated by fork +e2e per the rollout plan. +""" + +from __future__ import annotations + +import glob +import json +import os +import subprocess +import tempfile + +from ..core import actions, gh +from ..core.version import sort_versions_desc +from . import manifest, readmes + +RELEASES_BRANCH = "releases" + + +def _git(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run(["git", *args], check=check, capture_output=True, text=True) + + +def run() -> int: + repository = os.environ["GITHUB_REPOSITORY"] + token = os.environ["GITHUB_TOKEN"] + plugin = os.environ["YANK_PLUGIN"] + version = os.environ["YANK_VERSION"] + source_branch = os.environ["SOURCE_BRANCH"] + issue = os.environ["YANK_ISSUE"] + app_slug = os.environ.get("APP_SLUG", "") + remote = f"https://x-access-token:{token}@github.com/{repository}.git" + + actions.log(f"Yanking {plugin} v{version} from {RELEASES_BRANCH} (issue #{issue})") + + issue_state = gh.api(f"repos/{repository}/issues/{issue}", jq=".state") + if not issue_state: + actions.error(f"Issue #{issue} not found in {repository}.") + return 1 + if issue_state != "open": + actions.error(f"Issue #{issue} is already {issue_state}. Only open issues can authorize a yank.") + return 1 + actions.log(f" Issue #{issue} is open - proceeding.") + + workdir = tempfile.mkdtemp() + build_meta_dir = tempfile.mkdtemp() + os.environ["BUILD_META_DIR"] = build_meta_dir # kept for parity; generate() is passed it directly + start_cwd = os.getcwd() + try: + _git("clone", "--no-checkout", remote, f"{workdir}/repo") + os.chdir(f"{workdir}/repo") + _configure_identity(repository, app_slug) + + if _git("ls-remote", "--exit-code", "--heads", "origin", RELEASES_BRANCH, + check=False).returncode != 0: + actions.error("Releases branch does not exist.") + return 1 + _git("checkout", RELEASES_BRANCH) + _git("pull", "origin", RELEASES_BRANCH, check=False) + + release_tag = f"{plugin}-{version}" + plugin_manifest = f"metadata/{plugin}/manifest.json" + + if not gh.release_exists(release_tag, repository): + actions.error(f"GitHub Release {release_tag} not found. Nothing to yank.") + return 1 + + prefix = f"{plugin}-" + remaining = [t for t in gh.release_list_tags(repository) + if t.startswith(prefix) and t != f"{plugin}-latest" and t != release_tag] + is_last_version = len(remaining) == 0 + + current_latest = "" + if os.path.isfile(plugin_manifest): + try: + with open(plugin_manifest, encoding="utf-8") as fh: + data = json.load(fh) + current_latest = ((data.get("manifest") or {}).get("latest") or {}).get("version") or "" + except ValueError: + current_latest = "" + is_latest = version == current_latest + + actions.log(f" Current latest : {current_latest or 'unknown'}") + actions.log(f" Is latest : {str(is_latest).lower()}") + actions.log(f" Remaining releases: {len(remaining)}") + actions.log(f" Is last version : {str(is_last_version).lower()}") + + _git("fetch", "origin", source_branch) + _git("checkout", f"origin/{source_branch}", "--", "plugins") + source_type = "local" + pj = f"plugins/{plugin}/plugin.json" + if os.path.isfile(pj): + try: + with open(pj, encoding="utf-8") as fh: + source_type = (json.load(fh).get("source_type") or "local") + except ValueError: + source_type = "local" + + new_latest_version = "" + if is_last_version: + actions.log(f"Last version - deleting all GitHub Releases for {plugin}.") + gh.release_delete(release_tag, repository) + subprocess.run(["rm", "-rf", f"metadata/{plugin}"], check=False) + else: + actions.log(f"Deleting GitHub Release {release_tag}") + gh.release_delete(release_tag, repository) + if is_latest: + bare = [t[len(prefix):] for t in remaining] + ordered = sort_versions_desc(bare) + new_latest_version = ordered[0] if ordered else "" + if not new_latest_version: + actions.error("Could not find a replacement version to promote to latest.") + return 1 + actions.log(f"Promoting {new_latest_version} to latest (manifest will be updated by generate-manifest).") + + for f in ("manifest.json", "README.md"): + if os.path.exists(f): + os.remove(f) + actions.log("\n=== Regenerating manifests ===") + manifest.generate(source_branch, RELEASES_BRANCH, repository, build_meta_dir) + actions.log("\n=== Regenerating per-plugin READMEs ===") + readmes.generate_plugin_readmes(source_branch, repository) + actions.log("\n=== Regenerating releases README ===") + readmes.generate_releases_readme(source_branch, RELEASES_BRANCH, repository) + + releases_commit = _commit_releases(remote, plugin, version, issue) + rollback_pr_url = "" + if is_latest: + rollback_pr_url = _open_rollback_pr( + repository, source_branch, remote, plugin, version, + new_latest_version, is_last_version, source_type, issue) + + _finalize_issue(repository, plugin, version, issue, releases_commit, + is_latest, rollback_pr_url) + from ..integrations import webhooks + webhooks.emit("plugin.yanked", + webhooks.plugin_yanked(plugin, version, issue, rollback_pr_url or None)) + return 0 + finally: + os.chdir(start_cwd) + subprocess.run(["rm", "-rf", workdir, build_meta_dir], check=False) + + +def _configure_identity(repository: str, app_slug: str) -> None: + if app_slug: + bot_user_id = gh.api(f"/users/{app_slug}%5Bbot%5D", jq=".id") or "" + _git("config", "user.name", f"{app_slug}[bot]") + email = (f"{bot_user_id}+{app_slug}[bot]@users.noreply.github.com" if bot_user_id + else f"{app_slug}[bot]@users.noreply.github.com") + _git("config", "user.email", email) + else: + _git("config", "user.name", "github-actions[bot]") + _git("config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com") + + +def _commit_releases(remote, plugin, version, issue) -> str: + actions.log("\n=== Committing ===") + subprocess.run(["rm", "-rf", "plugins"], check=False) + _git("rm", "-rf", "--cached", "plugins", check=False) + _git("add", "metadata", "manifest.json", "README.md") + if _git("diff", "--cached", "--quiet", check=False).returncode == 0: + actions.log("No changes to commit - was this version already absent?") + return "" + _git("commit", "-m", f"Yank {plugin} v{version}\n\nRefs #{issue}") + releases_commit = _git("rev-parse", "--short", "HEAD").stdout.strip() + _git("push", remote, RELEASES_BRANCH) + actions.log(f"Successfully yanked {plugin} v{version} from {RELEASES_BRANCH}") + return releases_commit + + +def _open_rollback_pr(repository, source_branch, remote, plugin, version, + new_latest_version, is_last_version, source_type, issue) -> str: + actions.log("\n=== Opening rollback PR ===") + rollback_branch = f"yank/{plugin}-{version}" + _git("fetch", "origin", source_branch) + _git("checkout", "-b", rollback_branch, f"origin/{source_branch}") + + if is_last_version: + _git("rm", "-rf", f"plugins/{plugin}") + pr_title = f"[{plugin}]: Remove plugin (all versions yanked)" + pr_body = (f"## Plugin Removed\n\nAll published versions of `{plugin}` have been yanked " + f"from the releases branch.\n\nThis PR removes the plugin from the source branch " + f"to prevent it from being republished on the next run.\n\n" + f"**Yanked version:** `{version}`\n**Authorized by:** #{issue}\n\nCloses #{issue}") + else: + source_note = _rollback_source(plugin, new_latest_version, source_type) + _git("add", f"plugins/{plugin}/") + pr_title = f"[{plugin}]: Roll back to v{new_latest_version} (yank of v{version})" + pr_body = (f"## Version Rollback\n\n`{plugin}` `{version}` has been yanked from the releases " + f"branch. This PR rolls the source back to `{new_latest_version}` so the yanked " + f"version is not republished on the next run.\n\n" + f"**Yanked version:** `{version}`\n**New latest:** `{new_latest_version}`\n" + f"**Authorized by:** #{issue}\n\n{source_note}\n\nCloses #{issue}") + + _git("commit", "-m", pr_title) + _git("push", remote, rollback_branch) + gh.label_create("Rollback", "E11D48", "Version rollback opened by the yank workflow", repository) + url = gh.pr_create(repository, source_branch, rollback_branch, pr_title, pr_body, + labels=["Rollback"]) or "" + actions.log(f"Rollback PR opened: {rollback_branch} -> {source_branch} ({url})") + return url + + +def _rollback_source(plugin, new_latest_version, source_type) -> str: + pj = f"plugins/{plugin}/plugin.json" + if source_type == "local": + restore_commit = "" + shas = _git("log", "--all", "--format=%H", "--", pj).stdout.split() + for sha in shas: + show = _git("show", f"{sha}:{pj}", check=False) + if show.returncode != 0: + continue + try: + if json.loads(show.stdout).get("version") == new_latest_version: + restore_commit = sha + break + except ValueError: + continue + if restore_commit: + actions.log(f"Restoring plugins/{plugin}/ from commit {restore_commit}") + _git("checkout", restore_commit, "--", f"plugins/{plugin}/") + return (f"Plugin source files restored from commit `{restore_commit[:7]}` " + f"(the last commit where `{plugin}` was at `{new_latest_version}`).") + actions.warning(f"Could not find a commit for {plugin} v{new_latest_version} - " + "falling back to version field update only.") + _bump_version(pj, new_latest_version) + return ("⚠️ **Could not restore source files** - no commit found for " + f"`{new_latest_version}` (history may have been squashed). Only the `version` " + "field was updated. Please review the source files manually before merging.") + _bump_version(pj, new_latest_version) + return (f"Version field in `plugin.json` updated to `{new_latest_version}`. The ZIP will be " + "re-fetched from the external source URL on the next publish run.") + + +def _bump_version(pj: str, new_version: str) -> None: + with open(pj, encoding="utf-8") as fh: + data = json.load(fh) + data["version"] = new_version + # jq '.version = $v' rewrites pretty-printed with 2-space indent + trailing newline. + with open(pj, "w", encoding="utf-8") as fh: + json.dump(data, fh, indent=2, ensure_ascii=False) + fh.write("\n") + + +def _finalize_issue(repository, plugin, version, issue, releases_commit, + is_latest, rollback_pr_url) -> None: + commit_label = releases_commit or "unknown" + if is_latest: + gh.issue_comment(issue, repository, + f"`{plugin}` v`{version}` has been removed from the releases branch " + f"(commit `{commit_label}`).\n\nA rollback PR has been opened to update the " + f"source branch: {rollback_pr_url or '(see workflow run for link)'}\n\n" + "This issue will close automatically when that PR is merged.") + else: + gh.issue_comment(issue, repository, + f"`{plugin}` v`{version}` has been removed from the releases branch " + f"(commit `{commit_label}`).\n\nThis was not the latest version so no source " + "branch changes are needed.") + gh.issue_close(issue, repository, reason="completed") diff --git a/.github/pluginctl/src/pluginctl/publish/zips.py b/.github/pluginctl/src/pluginctl/publish/zips.py new file mode 100644 index 0000000..584ba73 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/publish/zips.py @@ -0,0 +1,145 @@ +"""Build versioned ZIPs + GitHub Releases (port of build-zips.sh). + +Runs from the releases-branch checkout directory. For every plugin whose current +version has no GitHub Release yet, it builds (or downloads, for external plugins) +the ZIP, records per-version metadata under ``BUILD_META_DIR`` for +generate-manifest to consume, uploads the release, and appends the plugin to +``changed_plugins.txt``. +""" + +from __future__ import annotations + +import glob +import hashlib +import json +import os +import shutil +import subprocess +import tempfile +import time +import urllib.request +from typing import Optional + +from ..core import actions, gh, git, jsonio +from ..core.jsonio import drop_none + + +def _now() -> str: + import datetime + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _checksums(path: str) -> tuple[str, str]: + md5 = hashlib.md5() + sha = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + md5.update(chunk) + sha.update(chunk) + return md5.hexdigest(), sha.hexdigest() + + +def _download(url: str, dest: str, attempts: int = 3) -> bool: + for attempt in range(1, attempts + 1): + try: + with urllib.request.urlopen(url, timeout=60) as resp, open(dest, "wb") as out: + shutil.copyfileobj(resp, out) + return True + except Exception: + if os.path.exists(dest): + os.remove(dest) + if attempt < attempts: + actions.log(f" Download attempt {attempt} failed, retrying in 15s...") + time.sleep(15) + return False + + +def run(source_branch: str, repository: str, build_meta_dir: str) -> int: + changed: list[str] = [] + open("changed_plugins.txt", "w", encoding="utf-8").close() + + for plugin_dir in sorted(glob.glob("plugins/*/")): + plugin_name = os.path.basename(plugin_dir.rstrip("/")) + plugin_key = plugin_name.replace("-", "_") + with open(os.path.join(plugin_dir, "plugin.json"), encoding="utf-8") as fh: + raw = json.load(fh) + version = str(raw.get("version")) + os.makedirs(f"metadata/{plugin_name}", exist_ok=True) + + zip_path = f"/tmp/{plugin_name}-{version}.zip" + release_tag = f"{plugin_name}-{version}" + + if gh.release_exists(release_tag, repository): + actions.log(f" {plugin_name} v{version} - skipping (release already exists)") + continue + + source_type = raw.get("source_type") or "local" + build_timestamp = _now() + commit_sha = git.log_format("%H", f"origin/{source_branch}", plugin_dir) or "" + commit_sha_short = git.log_format("%h", f"origin/{source_branch}", plugin_dir) or "" + last_updated = git.log_format("%cI", f"origin/{source_branch}", plugin_dir) or _now() + source_url_resolved = "" + + if source_type == "external": + source_url_resolved = str(raw.get("source_url") or "").replace("{version}", version) + actions.log(f" {plugin_name} v{version} - fetching external ZIP from {source_url_resolved}") + changed.append(f"{plugin_key}@{version}") + if not _download(source_url_resolved, zip_path): + actions.error(f"Failed to download external ZIP from {source_url_resolved} after 3 attempts") + return 1 + else: + actions.log(f" {plugin_name} v{version} - building") + changed.append(f"{plugin_key}@{version}") + with tempfile.TemporaryDirectory() as tmpdir: + shutil.copytree(f"plugins/{plugin_name}", os.path.join(tmpdir, plugin_key)) + subprocess.run(["zip", "-r", zip_path, plugin_key], cwd=tmpdir, + check=True, stdout=subprocess.DEVNULL) + + checksum_md5, checksum_sha256 = _checksums(zip_path) + min_da = str(raw.get("min_dispatcharr_version") or "") + max_da = str(raw.get("max_dispatcharr_version") or "") + zip_size_kb = os.path.getsize(zip_path) // 1024 + + meta = drop_none({ + "version": version, + "commit_sha": commit_sha or None, + "commit_sha_short": commit_sha_short or None, + "build_timestamp": build_timestamp, + "last_updated": last_updated, + "checksum_md5": checksum_md5, + "checksum_sha256": checksum_sha256, + "min_dispatcharr_version": min_da or None, + "max_dispatcharr_version": max_da or None, + "source_url": source_url_resolved or None, + "size_kb": zip_size_kb, + }) + os.makedirs(os.path.join(build_meta_dir, plugin_key), exist_ok=True) + with open(os.path.join(build_meta_dir, plugin_key, f"{plugin_key}-{version}.json"), + "w", encoding="utf-8") as fh: + fh.write(jsonio.dumps(meta)) + + notes = _release_notes(repository, plugin_name, commit_sha, commit_sha_short) + actions.log(f" {plugin_name} v{version} - uploading to GitHub Releases") + gh.release_create(release_tag, repository, f"{plugin_name} v{version}", notes, zip_path) + os.remove(zip_path) + + with open("changed_plugins.txt", "w", encoding="utf-8") as fh: + for line in changed: + fh.write(line + "\n") + actions.log(f"Built {len(changed)} new/updated plugin(s).") + return 0 + + +def _release_notes(repository: str, plugin_name: str, commit_sha: str, + commit_sha_short: str) -> str: + readme_url = f"https://github.com/{repository}/blob/releases/metadata/{plugin_name}/README.md" + notes = "" + if commit_sha: + commit_url = f"https://github.com/{repository}/commit/{commit_sha}" + notes = f"**Commit:** [`{commit_sha_short}`]({commit_url})" + pr_number, pr_url = gh.commit_pull_number(repository, commit_sha) + if pr_number: + notes += f"\n**PR:** [#{pr_number}]({pr_url})" + notes += "\n" + notes += f"**README:** [Plugin README]({readme_url})" + return notes diff --git a/.github/pluginctl/src/pluginctl/validate/__init__.py b/.github/pluginctl/src/pluginctl/validate/__init__.py new file mode 100644 index 0000000..80fa5f5 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/__init__.py @@ -0,0 +1 @@ +"""PR-time validation pipeline: detect, checks, SARIF, report, gate.""" diff --git a/.github/pluginctl/src/pluginctl/validate/clamav.py b/.github/pluginctl/src/pluginctl/validate/clamav.py new file mode 100644 index 0000000..880d439 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/clamav.py @@ -0,0 +1,78 @@ +"""``pluginctl clamav-report``: parse clamscan output into status + findings table. + +Ports the "Set ClamAV status outputs" step of the clamav-scan job. ClamAV itself +still runs as the native CLI in the workflow; only the result parsing moves here. +""" + +from __future__ import annotations + +import hashlib +import os +from typing import Callable, Optional + +from ..core import actions + + +def _sha256_file(path: str) -> str: + try: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + except OSError: + return "" + + +def parse_infected_lines(output: str) -> list[str]: + """Lines ending in ' FOUND' (clamscan --infected format).""" + return [ln for ln in output.splitlines() if ln.endswith(" FOUND")] + + +def build_findings_table(infected_lines: list[str], workspace: str, + sha256_fn: Optional[Callable[[str], str]] = None) -> str: + """Render the ``| File | Signature |`` table, linking known hashes to VirusTotal.""" + sha256_fn = sha256_fn or _sha256_file + rows = ["| File | Signature |", "|------|-----------|"] + for line in infected_lines: + # ": FOUND" + full_path = line + idx = line.rfind(": ") + if idx != -1 and line.endswith(" FOUND"): + full_path = line[:idx] + sig = line[idx + 2:-len(" FOUND")] + else: + sig = "" + file_rel = full_path + prefix = (workspace or "") + "/" + if workspace and file_rel.startswith(prefix): + file_rel = file_rel[len(prefix):] + digest = sha256_fn(full_path) + if digest: + rows.append(f"| `{file_rel}` | [`{sig}`](https://www.virustotal.com/gui/file/{digest}) |") + else: + rows.append(f"| `{file_rel}` | `{sig}` |") + return "\n".join(rows) + "\n" + + +def run(output_file: str = "clamav-output.txt", scan_exit: int = 0, + findings_out: str = "clamav-findings.md") -> int: + workspace = os.environ.get("GITHUB_WORKSPACE", "") + output = "" + if os.path.isfile(output_file): + with open(output_file, encoding="utf-8") as fh: + output = fh.read() + infected_lines = parse_infected_lines(output) + actions.set_output("clamav_infected", str(len(infected_lines))) + + if infected_lines: + table = build_findings_table(infected_lines, workspace) + with open(findings_out, "w", encoding="utf-8") as fh: + fh.write(table) + actions.set_output("clamav_status", "failure") + return 0 + if scan_exit >= 2: + actions.set_output("clamav_status", "failure") + return 0 + actions.set_output("clamav_status", "success") + return 0 diff --git a/.github/pluginctl/src/pluginctl/validate/detect.py b/.github/pluginctl/src/pluginctl/validate/detect.py new file mode 100644 index 0000000..63c50c4 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/detect.py @@ -0,0 +1,190 @@ +"""``pluginctl detect``: port of validate/detect-changes.sh + inline blacklist. + +Determines the changed-plugin matrix, whether the PR must be auto-closed, whether +it touches files outside ``plugins/`` without authorization, new/updated flags, +and the signing-key-changed warning. Emits the same ``$GITHUB_OUTPUT`` keys the +Bash script did so the downstream jobs are unchanged. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from typing import Optional + +from ..core import actions, gh, git + +SAFE_NAME_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +PUB_KEY_PATH = ".github/scripts/keys/dispatcharr-plugins.pub" + + +def is_safe_name(name: str) -> bool: + return bool(SAFE_NAME_RE.match(name)) + + +def author_has_plugin_permission(pr_author: str, base_json_text: Optional[str]) -> bool: + """True when pr_author is the base-branch author or in base maintainers. + + Reads from the base branch only, so a PR cannot self-grant permission. + """ + if not base_json_text: + return False + try: + data = json.loads(base_json_text) + except json.JSONDecodeError: + return False + author = data.get("author") or "" + maintainers = [str(m) for m in (data.get("maintainers") or []) if m is not None] + return pr_author == author or pr_author in maintainers + + +@dataclass +class BlacklistResult: + matched: bool = False + reason: str = "" + + +def check_blacklists(pr_author: str, plugins: list[str], + author_blacklist: str, plugin_blacklist: str) -> BlacklistResult: + """Case-insensitive author/plugin blacklist check (whitespace-trimmed entries).""" + author_bl = (author_blacklist or "").strip() + plugin_bl = (plugin_blacklist or "").strip() + if not author_bl and not plugin_bl: + return BlacklistResult() + + if author_bl: + for entry in author_bl.split(","): + slug = entry.replace(" ", "") + if pr_author.lower() == slug.lower(): + actions.warning(f"PR author '{pr_author}' is on the author blacklist.") + return BlacklistResult(True, "author-blacklisted") + + if plugin_bl: + blocked = plugin_bl.split(",") + for plugin in plugins: + if not plugin: + continue + for entry in blocked: + slug = entry.replace(" ", "") + if plugin.lower() == slug.lower(): + actions.warning(f"Plugin '{plugin}' is on the plugin blacklist.") + return BlacklistResult(True, "plugin-blacklisted") + return BlacklistResult() + + +def run(pr_author: str, base_ref: str, head_ref: str = "", + author_blacklist: str = "", plugin_blacklist: str = "", + repo: str = "") -> int: + """Execute detection; write outputs; return process exit code (0 ok, 1 hard block).""" + owner, _, name = repo.partition("/") + + def write_access() -> bool: + return gh.has_write_access(owner, name, pr_author) + + # Bot-authored yank rollback PRs bypass plugin validation entirely. + if pr_author.endswith("[bot]") and head_ref.startswith("yank/"): + actions.log(f"Bot-authored yank rollback PR detected ({pr_author}, {head_ref}) " + "- skipping plugin validation.") + actions.set_outputs(matrix="[]", plugin_count="0", close_pr="false", + close_reason="", skip_validation="true", + outside_violation="false", pub_key_changed="false", + has_new_plugin="false", has_updated_plugin="false") + return 0 + + merge_base = git.merge_base(f"origin/{base_ref}", "HEAD") + changed = git.diff_name_only(merge_base, "HEAD") + outside_changes = [f for f in changed if not f.startswith("plugins/")] + has_outside_violation = bool(outside_changes) and not write_access() + + plugin_list = sorted({f.split("/")[1] for f in changed if f.startswith("plugins/") and len(f.split("/")) > 1}) + + if not plugin_list: + return _no_plugins(pr_author, outside_changes, has_outside_violation, write_access) + + # Allowlist: only safe kebab-case names enter the matrix. + safe_list: list[str] = [] + for plugin in plugin_list: + if is_safe_name(plugin): + safe_list.append(plugin) + else: + actions.warning(f"Skipping plugin with unsafe folder name: '{plugin}'") + plugin_list = safe_list + + if not plugin_list: + actions.error("No valid plugin changes detected in this PR.") + actions.set_outputs(close_pr="true", close_reason="no-valid-plugins", + plugin_count="0", matrix="[]", + has_new_plugin="false", has_updated_plugin="false") + return 0 + + plugin_count = len(plugin_list) + + has_new_plugin = False + has_updated_plugin = False + for plugin in plugin_list: + if git.object_exists(f"origin/{base_ref}:plugins/{plugin}/plugin.json"): + has_updated_plugin = True + else: + has_new_plugin = True + + is_maintainer = write_access() + has_any_permission = is_maintainer + if not is_maintainer: + for plugin in plugin_list: + base_json = git.show(f"origin/{base_ref}:plugins/{plugin}/plugin.json") + if base_json and author_has_plugin_permission(pr_author, base_json): + has_any_permission = True + break + + close_pr = (not has_any_permission) and (not has_new_plugin) + + # Blacklist gate (only meaningful when not already closing). + close_reason = "unauthorized" if close_pr else "" + if not close_pr: + bl = check_blacklists(pr_author, plugin_list, author_blacklist, plugin_blacklist) + if bl.matched: + close_pr = True + close_reason = bl.reason + + matrix_json = json.dumps(plugin_list, separators=(",", ":")) + actions.set_output("matrix", matrix_json) + actions.set_output("plugin_count", str(plugin_count)) + actions.set_output("close_pr", "true" if close_pr else "false") + actions.set_output("skip_validation", "false") + actions.set_output("outside_violation", "true" if has_outside_violation else "false") + actions.set_output("close_reason", close_reason) + if outside_changes: + actions.set_output("outside_files", "\n".join(outside_changes)) + + pub_key_changed = is_maintainer and PUB_KEY_PATH in outside_changes + actions.set_output("pub_key_changed", "true" if pub_key_changed else "false") + actions.set_output("has_new_plugin", "true" if has_new_plugin else "false") + actions.set_output("has_updated_plugin", "true" if has_updated_plugin else "false") + + actions.log(f"Detected {plugin_count} plugin(s): {' '.join(plugin_list)}") + actions.log(f"close_pr={'true' if close_pr else 'false'}") + return 0 + + +def _no_plugins(pr_author, outside_changes, has_outside_violation, write_access) -> int: + if has_outside_violation: + actions.set_outputs(matrix="[]", plugin_count="0", close_pr="false", + close_reason="", skip_validation="false", + outside_violation="true", + has_new_plugin="false", has_updated_plugin="false") + actions.set_output("outside_files", "\n".join(outside_changes)) + return 0 + if write_access(): + pub_key_changed = PUB_KEY_PATH in outside_changes + actions.set_outputs(matrix="[]", plugin_count="0", close_pr="false", + close_reason="", skip_validation="true", + outside_violation="false", + pub_key_changed="true" if pub_key_changed else "false", + has_new_plugin="false", has_updated_plugin="false") + if outside_changes: + actions.set_output("outside_files", "\n".join(outside_changes)) + actions.log("No plugin changes detected - skipping plugin validation (author has write access).") + return 0 + actions.error("No plugin changes detected in this PR.") + return 1 diff --git a/.github/pluginctl/src/pluginctl/validate/gate.py b/.github/pluginctl/src/pluginctl/validate/gate.py new file mode 100644 index 0000000..24506bd --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/gate.py @@ -0,0 +1,43 @@ +"""``pluginctl gate``: port of the ``Plugin PR Check`` evaluation ladder. + +This is the single fixed status check referenced by branch protection. The +decision order and messages are preserved exactly; the quarantine-label check is +a separate step that runs first in the workflow and is modeled here too. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass +class GateResult: + ok: bool + message: str + + +def evaluate(detect_result: str, close_pr: str, skip_validation: str, + outside_violation: str, title_result: str, + codeql_result: str, codeql_status: str, + clamav_result: str, clamav_status: str, + validate_result: str, report_result: str) -> GateResult: + """Return pass/fail with the exact ::error:: message the Bash ladder emits.""" + if detect_result != "success": + return GateResult(False, "Plugin detection failed or no plugin changes found.") + if skip_validation == "true": + return GateResult(True, "No plugin changes detected and author has write access - passing.") + if title_result == "failure": + return GateResult(False, "PR title does not match the required format. Rename the PR and re-run.") + if outside_violation == "true": + return GateResult(False, "PR contains unauthorized changes outside the plugins/ directory.") + if close_pr == "true": + return GateResult(False, "PR is unauthorized - no permission to modify these plugins.") + if codeql_result == "failure" or codeql_status == "failure": + return GateResult(False, "CodeQL security analysis failed. See the Security tab for details.") + if clamav_result == "failure" or clamav_status == "failure": + return GateResult(False, "ClamAV antivirus scan detected threats. See PR comment for details.") + if validate_result in ("failure", "cancelled"): + return GateResult(False, "One or more plugin validations failed. See PR comment for details.") + if report_result != "success": + return GateResult(False, "Plugin validation failed. See PR comment for details.") + return GateResult(True, "All plugins validated successfully.") diff --git a/.github/pluginctl/src/pluginctl/validate/labels.py b/.github/pluginctl/src/pluginctl/validate/labels.py new file mode 100644 index 0000000..bf877b0 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/labels.py @@ -0,0 +1,35 @@ +"""``pluginctl label``: port of the inline ``label-pr`` job. + +Reconciles the classification labels on a PR. The decision is pure and unit +tested; applying it just calls ``gh pr edit --add-label/--remove-label`` with the +same add-when-true / remove-when-false behavior as the Bash ``apply_label``. +""" + +from __future__ import annotations + +from ..core import gh + +LABELS = ["New Plugin", "Plugin Update", "Repo Update", "Invalid"] + + +def decide_labels(has_new_plugin: bool, has_updated_plugin: bool, + outside_files: str, outside_violation: bool, + close_pr: bool) -> dict[str, bool]: + """Return the desired on/off state for each managed label.""" + is_repo_update = bool(outside_files) and not outside_violation + is_invalid = outside_violation or close_pr + return { + "New Plugin": has_new_plugin, + "Plugin Update": has_updated_plugin, + "Repo Update": is_repo_update, + "Invalid": is_invalid, + } + + +def run(pr_number: str, has_new_plugin: bool, has_updated_plugin: bool, + outside_files: str, outside_violation: bool, close_pr: bool) -> int: + desired = decide_labels(has_new_plugin, has_updated_plugin, + outside_files, outside_violation, close_pr) + for label in LABELS: + gh.pr_edit_label(pr_number, label, desired[label]) + return 0 diff --git a/.github/pluginctl/src/pluginctl/validate/langs.py b/.github/pluginctl/src/pluginctl/validate/langs.py new file mode 100644 index 0000000..622eb4d --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/langs.py @@ -0,0 +1,79 @@ +"""``pluginctl detect-langs``: which CodeQL languages are present in changed plugins. + +Ports the "Detect supported languages" bash step. Emits ``found`` / ``languages`` +/ ``unscanned_langs`` the same way, in the same order, so the CodeQL matrix and +the "unscanned file types" notice are unchanged. +""" + +from __future__ import annotations + +import os + +from ..core import actions + +# CodeQL-supported languages, in the Bash detection order. +_SUPPORTED = [ + ("python", {".py"}), + ("javascript", None), # special-cased (prunes vendored dirs) + ("go", {".go"}), + ("ruby", {".rb"}), + ("java-kotlin", {".java", ".kt", ".kts"}), + ("c-cpp", {".c", ".cpp", ".cc", ".h", ".hpp"}), +] + +# Present-but-unsupported file types, in the Bash order. +_UNSCANNED = [ + ("shell", {".sh", ".bash"}), + ("php", {".php"}), + ("lua", {".lua"}), + ("perl", {".pl", ".pm"}), + ("rust", {".rs"}), +] + +# Directories pruned from the JavaScript search (vendored / build output). +_JS_PRUNE = {"node_modules", "dist", "build", "static"} +_JS_EXTS = {".js", ".ts", ".jsx", ".tsx"} + + +def _has_ext(root: str, exts: set[str]) -> bool: + for _dirpath, _dirs, files in os.walk(root): + for fn in files: + if os.path.splitext(fn)[1] in exts: + return True + return False + + +def _has_js(root: str) -> bool: + for dirpath, dirs, files in os.walk(root): + dirs[:] = [d for d in dirs if d not in _JS_PRUNE] + for fn in files: + if os.path.splitext(fn)[1] in _JS_EXTS: + return True + return False + + +def detect(root: str = "plugins") -> tuple[bool, str, str]: + """Return (found, languages_csv, unscanned_csv).""" + langs: list[str] = [] + for name, exts in _SUPPORTED: + if name == "javascript": + if _has_js(root): + langs.append(name) + elif _has_ext(root, exts): + langs.append(name) + unscanned = [name for name, exts in _UNSCANNED if _has_ext(root, exts)] + return bool(langs), ",".join(langs), ",".join(unscanned) + + +def run(root: str = "plugins") -> int: + found, languages, unscanned = detect(root) + actions.set_output("unscanned_langs", unscanned) + actions.set_output("found", "true" if found else "false") + actions.set_output("languages", languages) + if found: + actions.log(f"Detected languages for CodeQL: {languages}") + else: + actions.log("No supported language files found in changed plugins, skipping CodeQL.") + if unscanned: + actions.log(f"Unscanned file types (no CodeQL support): {unscanned}") + return 0 diff --git a/.github/pluginctl/src/pluginctl/validate/plugin.py b/.github/pluginctl/src/pluginctl/validate/plugin.py new file mode 100644 index 0000000..965951d --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/plugin.py @@ -0,0 +1,394 @@ +"""``pluginctl validate``: port of validate/validate.sh. + +Validates one plugin and writes a markdown report fragment identical to the Bash +output: same ``### Plugin:`` header, description/repo subtext, the +``| Check | Status | Details |`` table (rows hidden when they pass, in the same +order), the optional release/compare links, and the trailing ```` +tab-separated marker consumed by ``report``. Emits ``result``/``is_new``/ +``has_permission`` outputs and exits non-zero on failure. +""" + +from __future__ import annotations + +import json +import os +import re +import urllib.error +import urllib.request +from typing import Optional + +from ..core import actions, gh, git +from ..core.version import is_semver, is_dispatcharr_version, version_greater_than + +METADATA_ONLY_FIELDS = [ + "description", "repo_url", "discord_thread", "min_dispatcharr_version", + "max_dispatcharr_version", "deprecated", "unlisted", "maintainers", +] + +_GH_DOWNLOAD_RE = re.compile(r"^https://github\.com/([^/]+)/([^/]+)/releases/download/([^/]+)/") +_GH_DOWNLOAD_TAG_RE = re.compile(r"^https://github\.com/[^/]+/[^/]+/releases/download/([^/]+)/") + +SPDX_URL = "https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json" + + +# ---- jq-parity scalar helpers ------------------------------------------------ +def jq_r(value, default: Optional[str] = None) -> str: + """Mimic ``jq -r``. Without a default, null/missing renders as ``"null"``. + + With ``default`` (the ``// "x"`` idiom), null/missing renders as ``default``. + """ + if value is None: + return "null" if default is None else default + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +def field_present(raw: dict, key: str) -> bool: + """Reproduce ``jq -e '.\"key\"'``: present unless null or false.""" + val = raw.get(key, None) + return not (val is None or val is False) + + +def tsv(fields: list[str]) -> str: + """Reproduce ``@tsv`` escaping for a single row.""" + def esc(s: str) -> str: + return (s.replace("\\", "\\\\").replace("\t", "\\t") + .replace("\n", "\\n").replace("\r", "\\r")) + return "\t".join(esc(f) for f in fields) + + +# ---- network (injectable for tests) ----------------------------------------- +def http_status(url: str) -> str: + """Follow redirects, return the numeric HTTP status as a string ("000" on error).""" + try: + req = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(req, timeout=15) as resp: + return str(getattr(resp, "status", None) or resp.getcode()) + except urllib.error.HTTPError as exc: # type: ignore[attr-defined] + return str(exc.code) + except Exception: + return "000" + + +def fetch_spdx() -> Optional[dict]: + try: + with urllib.request.urlopen(SPDX_URL, timeout=15) as resp: + return json.loads(resp.read().decode("utf-8")) + except Exception: + return None + + +# ---- helper: source file count (non-external) ------------------------------- +def count_source_files(plugin_dir: str) -> int: + ignore = {"plugin.json", "README.md", "logo.png"} + count = 0 + for root, _dirs, files in os.walk(plugin_dir): + for fn in files: + if fn not in ignore: + count += 1 + return count + + +def run(plugin_name: str, pr_author: str, base_ref: str, output_file: str, + repo: str = "") -> int: + owner, _, name = repo.partition("/") + plugin_dir = f"plugins/{plugin_name}" + plugin_json_path = f"{plugin_dir}/plugin.json" + + failed = False + is_new = False + has_permission = False + rows: list[str] = [] + pre_lines: list[str] = [] # header + desc/repo subtext + post_lines: list[str] = [] # release/compare link + meta_row: Optional[str] = None + + pre_lines.append(f"### Plugin: `{plugin_name}`") + pre_lines.append("") + + raw: Optional[dict] = None + if os.path.isfile(plugin_json_path): + try: + with open(plugin_json_path, encoding="utf-8") as fh: + raw = json.load(fh) + except (OSError, json.JSONDecodeError): + raw = None + if raw is not None: + desc = jq_r(raw.get("description"), "") + repo_url = jq_r(raw.get("repo_url"), "") + if desc: + pre_lines.append(f"_{desc}_") + pre_lines.append("") + if repo_url: + pre_lines.append(f"[Source Repository]({repo_url})") + pre_lines.append("") + + def emit_fragment() -> None: + lines = list(pre_lines) + lines.append("| Check | Status | Details |") + lines.append("|-------|:------:|---------|") + lines.extend(rows) + lines.append("") + lines.extend(post_lines) + if meta_row is not None: + lines.append(meta_row) + text = "\n".join(lines) + "\n" + _write(output_file, text) + + # Folder name format + if not re.match(r"^[a-z0-9]+(-[a-z0-9]+)*$", plugin_name): + rows.append(f"| Folder name | ❌ | Must be lowercase-kebab-case - got " + f"`{plugin_name}`, e.g. `my-plugin-name` |") + failed = True + + # plugin.json existence (early exit) + if not os.path.isfile(plugin_json_path): + rows.append("| `plugin.json` | ❌ | File missing |") + emit_fragment() + actions.set_outputs(result="fail", is_new="false", has_permission="false") + return 0 + + # JSON syntax (early exit) + if raw is None: + rows.append("| JSON syntax | ❌ | Invalid JSON in plugin.json |") + emit_fragment() + actions.set_outputs(result="fail", is_new="false", has_permission="false") + return 0 + + # Required fields + missing = [f"`{key}`" for key in ("name", "version", "description") + if not field_present(raw, key)] + if missing: + failed = True + rows.append(f"| Required fields | ❌ | Missing: {', '.join(missing)} |") + else: + rows.append("| Required fields | ✅ | All required fields present |") + + author = jq_r(raw.get("author"), "") + maintainers = [str(m) for m in (raw.get("maintainers") or []) if m is not None] + maintainers_sp = " ".join(maintainers) + version = jq_r(raw.get("version")) + + # Base branch plugin.json (for version bump + compare link) + base_json_text = git.show(f"origin/{base_ref}:{plugin_json_path}") + base_raw = None + old_version = "" + old_source_url_tmpl = "" + if base_json_text is not None: + try: + base_raw = json.loads(base_json_text) + old_version = jq_r(base_raw.get("version"), "") + old_source_url_tmpl = jq_r(base_raw.get("source_url"), "") + except json.JSONDecodeError: + base_raw = None + + # External source checks + source_type = jq_r(raw.get("source_type"), "local") + release_link = "" + compare_link = "" + gh_tag = "" + old_gh_tag = "" + if source_type != "external": + if count_source_files(plugin_dir) == 0: + rows.append(f"| Plugin files | ❌ | No source files found in " + f"`plugins/{plugin_name}/`. Add your plugin source (e.g. `main.py`) " + f"or set `\"source_type\": \"external\"` in `plugin.json` if your plugin " + f"is hosted elsewhere |") + failed = True + else: + ext_source_url = jq_r(raw.get("source_url"), "") + ext_repo_url = jq_r(raw.get("repo_url"), "") + if not ext_source_url: + rows.append("| `source_url` | ❌ | Required when `source_type` is `external` |") + failed = True + elif not ext_source_url.startswith("https://"): + rows.append("| `source_url` | ❌ | Must be an HTTPS URL |") + failed = True + elif "{version}" not in ext_source_url: + rows.append("| `source_url` | ❌ | Must contain a `{version}` placeholder " + "(e.g. `.../v{version}/plugin.zip`) |") + failed = True + else: + if is_semver(version): + resolved_url = ext_source_url.replace("{version}", version) + code = http_status(resolved_url) + if code == "200": + rows.append("| Release artifact | ✅ | Artifact reachable at resolved URL |") + m = _GH_DOWNLOAD_RE.match(resolved_url) + if m: + gh_owner, gh_repo, gh_tag = m.group(1), m.group(2), m.group(3) + release_link = f"https://github.com/{gh_owner}/{gh_repo}/releases/tag/{gh_tag}" + if old_version and old_source_url_tmpl: + old_resolved = old_source_url_tmpl.replace("{version}", old_version) + om = _GH_DOWNLOAD_TAG_RE.match(old_resolved) + if om: + old_gh_tag = om.group(1) + compare_link = (f"https://github.com/{gh_owner}/{gh_repo}" + f"/compare/{old_gh_tag}...{gh_tag}") + else: + rows.append(f"| Release artifact | ❌ | Could not reach `{resolved_url}` " + f"(HTTP `{code}`) — ensure the release exists |") + failed = True + if not ext_repo_url: + rows.append("| `repo_url` | ❌ | Required for external plugins — set to the upstream " + "source repository URL |") + failed = True + + # Maintainers + if not author and not maintainers_sp: + rows.append("| Maintainers | ❌ | At least one of `author` or `maintainers` must " + "include your GitHub username |") + failed = True + else: + parts = [] + if author: + parts.append(f"`{author}`") + for m in maintainers_sp.split(): + parts.append(f"`{m}`") + rows.append(f"| Maintainers | ✅ | {', '.join(parts)} |") + + # License + license_id = jq_r(raw.get("license"), "") + if not license_id: + rows.append("| License | ❌ | `license` is required - provide an " + "[OSI-approved SPDX identifier](https://spdx.org/licenses/) " + "(e.g. `MIT`, `Apache-2.0`) |") + failed = True + else: + spdx = fetch_spdx() + if spdx is None: + rows.append("| License | ⚠️ | Could not fetch SPDX license list - skipping validation |") + else: + osi = {l["licenseId"]: l.get("name", "") + for l in spdx.get("licenses", []) if l.get("isOsiApproved") is True} + if license_id in osi: + rows.append(f"| License | ✅ | `{license_id}` - {osi[license_id]} |") + else: + rows.append(f"| License | ❌ | `{license_id}` is not an " + "[OSI-approved SPDX identifier](https://spdx.org/licenses/) |") + failed = True + + # Permission + is_repo_maintainer = bool(owner) and gh.has_write_access(owner, name, pr_author) + if is_repo_maintainer: + rows.append("| Permission | ✅ | You have permission to modify this plugin |") + has_permission = True + elif base_raw is not None: + base_author = jq_r(base_raw.get("author"), "") + base_maintainers = [str(m) for m in (base_raw.get("maintainers") or []) if m is not None] + if pr_author == base_author or pr_author in base_maintainers: + rows.append("| Permission | ✅ | You have permission to modify this plugin |") + has_permission = True + else: + rows.append(f"| Permission | ❌ | `{pr_author}` is not listed in `author` or `maintainers` |") + failed = True + else: + if pr_author == author or pr_author in maintainers: + rows.append(f"| Permission | ✅ | New plugin - `{pr_author}` listed in `author`/`maintainers` |") + has_permission = True + else: + rows.append(f"| Permission | ❌ | Add `\"author\": \"{pr_author}\"` to plugin.json |") + failed = True + + # Version format + if is_semver(version): + rows.append(f"| Version | ✅ | `{version}` |") + else: + rows.append(f"| Version | ❌ | `{version}` is not valid semver - expected `X.Y.Z` |") + failed = True + + # Version bump + if base_raw is not None: + if version_greater_than(version, old_version): + rows.append(f"| Version bump | ✅ | `{old_version}` → `{version}` |") + else: + changed_fields = _changed_fields(base_raw, raw) + metadata_only = all(f in METADATA_ONLY_FIELDS for f in changed_fields) + if metadata_only and changed_fields: + rows.append(f"| Version bump | ✅ | `{old_version}` (unchanged - metadata-only update) |") + else: + other_changed = "" + if not changed_fields: + others = git.diff_name_only_range(f"origin/{base_ref}...HEAD", [plugin_dir]) + others = [f for f in others if f != plugin_json_path] + other_changed = others[0] if others else "" + if not changed_fields and not other_changed: + rows.append("| Version bump | ❌ | No changes detected - nothing to publish |") + else: + rows.append(f"| Version bump | ❌ | `{version}` must be greater than current `{old_version}` |") + failed = True + else: + rows.append("| Version bump | ✅ | New plugin |") + is_new = True + + # Dispatcharr version constraints + min_da = jq_r(raw.get("min_dispatcharr_version"), "") + max_da = jq_r(raw.get("max_dispatcharr_version"), "") + if min_da and not is_dispatcharr_version(min_da): + rows.append(f"| `min_dispatcharr_version` | ❌ | `{min_da}` is not valid semver - " + "expected `X.Y.Z` or `vX.Y.Z` |") + failed = True + if max_da and not is_dispatcharr_version(max_da): + rows.append(f"| `max_dispatcharr_version` | ❌ | `{max_da}` is not valid semver - " + "expected `X.Y.Z` or `vX.Y.Z` |") + failed = True + if min_da and max_da and is_dispatcharr_version(max_da) and is_dispatcharr_version(min_da): + _max = max_da[1:] if max_da.startswith("v") else max_da + _min = min_da[1:] if min_da.startswith("v") else min_da + if not version_greater_than(_max, _min) and _max != _min: + rows.append(f"| Version range | ❌ | `max_dispatcharr_version` (`{max_da}`) must be ≥ " + f"`min_dispatcharr_version` (`{min_da}`) |") + failed = True + + # Optional link fields + repo_url = jq_r(raw.get("repo_url"), "") + discord_thread = jq_r(raw.get("discord_thread"), "") + if repo_url and not re.match(r"^https?://", repo_url): + rows.append("| `repo_url` | ❌ | Must start with `http://` or `https://` |") + failed = True + if discord_thread and not re.match(r"^https?://", discord_thread): + rows.append("| `discord_thread` | ❌ | Must start with `http://` or `https://` |") + failed = True + + # Release / compare links (after the table) + if release_link: + link_line = f"[View release {gh_tag} on GitHub]({release_link})" + if compare_link: + link_line += f" · [Compare {old_gh_tag}...{gh_tag}]({compare_link})" + post_lines.append(link_line) + post_lines.append("") + + # Meta row + meta_fields = [ + jq_r(raw.get("name"), ""), + jq_r(raw.get("version"), ""), + jq_r(raw.get("description"), ""), + jq_r(raw.get("author"), ""), + ", ".join(maintainers), + jq_r(raw.get("repo_url"), ""), + jq_r(raw.get("discord_thread"), ""), + ] + meta_row = f"" + + emit_fragment() + actions.set_outputs( + result="pass" if not failed else "fail", + is_new="true" if is_new else "false", + has_permission="true" if has_permission else "false", + ) + return 1 if failed else 0 + + +def _changed_fields(old: dict, new: dict) -> list[str]: + """Keys of NEW whose value differs from OLD (missing old -> null), sorted like jq keys.""" + changed = [k for k in new.keys() if old.get(k) != new[k]] + return sorted(changed) + + +def _write(path: str, text: str) -> None: + if path in ("/dev/stdout", "-"): + print(text, end="") + return + with open(path, "w", encoding="utf-8") as fh: + fh.write(text) diff --git a/.github/pluginctl/src/pluginctl/validate/report.py b/.github/pluginctl/src/pluginctl/validate/report.py new file mode 100644 index 0000000..4c43060 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/report.py @@ -0,0 +1,425 @@ +"""``pluginctl report``: port of validate/report.sh. + +Combines per-plugin fragments, renders the single ```` +comment (same markers / sections / separators / emoji as the Bash), minimizes +prior validation comments as outdated, posts the new one, and closes the PR for +the unauthorized/blacklisted variants. The comment builder is pure so its exact +text is unit-testable; ``run`` performs the IO around it. +""" + +from __future__ import annotations + +import glob +import os +from dataclasses import dataclass, field +from typing import Optional + +from ..core import actions, gh, git + +MARKER = "" + + +@dataclass +class ParsedFragments: + combined_body: str = "" + plugin_links: str = "" + any_failed: bool = False + pr_plugin_names: list[str] = field(default_factory=list) + + +def _read(path: str) -> str: + try: + with open(path, encoding="utf-8") as fh: + return fh.read() + except OSError: + return "" + + +def parse_fragments(fragments_dir: str) -> ParsedFragments: + result = ParsedFragments() + for fragment in sorted(glob.glob(os.path.join(fragments_dir, "*.fragment.md"))): + content = _read(fragment) + if "❌" in content: + result.any_failed = True + # META_ROW extraction + for line in content.splitlines(): + if line.startswith(""): + meta = meta[:-3] + fields = meta.split("\t") + fields += [""] * (7 - len(fields)) + f_name, f_version, f_desc, f_author, f_maint, f_repo, f_discord = fields[:7] + if f_repo or f_discord: + result.plugin_links += f"**`{f_name}`:**\n" + if f_repo: + result.plugin_links += f"- [GitHub Repository]({f_repo})\n" + if f_discord: + result.plugin_links += f"- [Discord Thread]({f_discord})\n" + result.plugin_links += "\n" + break + # visible = fragment minus META_ROW lines, trailing whitespace stripped ($() semantics) + visible_lines = [ln for ln in content.splitlines() if not ln.startswith("\n") + assert "# Plugin Validation Results" in c + assert "## 🎉 All validation checks passed!" in c + assert "This PR modifies **1** plugin(s) and all checks have passed." in c + + +def test_fragment_failure_shows_failed(): + c = report.build_comment(**_base_kwargs(fragment_failed=True)) + assert "## ❌ Validation failed" in c + assert "🎉" not in c + + +def test_unauthorized_close_variant(): + c = report.build_comment(**_base_kwargs(close_pr=True, close_reason="unauthorized")) + assert "## PR Closed: Unauthorized" in c + assert "automatically closed" in c + # main body / success block suppressed + assert "🎉" not in c + + +def test_blacklisted_author_variant(): + c = report.build_comment(**_base_kwargs(close_pr=True, close_reason="author-blacklisted")) + assert "## PR Closed: Account Restricted" in c + + +def test_codeql_high_block_and_separator(): + c = report.build_comment(**_base_kwargs( + codeql_result="failure", codeql_errors="2", + codeql_findings="| Rule | Location | Description |\n")) + assert "❌ **CodeQL found 2 high or critical issue(s)**" in c + assert "\n---\n" in c + assert "## ❌ Validation failed" in c + + +def test_clamav_failure_block(): + c = report.build_comment(**_base_kwargs( + clamav_result="failure", clamav_infected="3", + clamav_findings="| File | Signature |\n")) + assert "❌ **ClamAV detected 3 infected file(s)**." in c + + +def test_title_invalid_block(): + c = report.build_comment(**_base_kwargs( + title_valid="false", title_feedback="Bad prefix.", + title_suggestion="[repo]: x")) + assert "### ❌ PR Title Format" in c + assert "Bad prefix." in c + assert "**Suggested format:** `[repo]: x`" in c + assert "## ❌ Validation failed" in c + + +def test_codeql_skipped_notice_no_separator_when_only_success(): + c = report.build_comment(**_base_kwargs(codeql_result="success")) + # success + no unscanned -> no findings separator, still passes + assert "## 🎉 All validation checks passed!" in c + + +def test_parse_fragments(tmp_path): + f = tmp_path / "demo.fragment.md" + f.write_text("### Plugin: `demo`\n\n| ok |\n\n", + encoding="utf-8") + parsed = report.parse_fragments(str(tmp_path)) + assert parsed.pr_plugin_names == ["demo"] + assert "\n" + ) + assert frag.read_text(encoding="utf-8") == expected + + +def test_missing_plugin_json_early_exit(plugin_repo): + tmp_path, out = plugin_repo + os.remove(tmp_path / "plugins" / "demo-plugin" / "plugin.json") + frag = tmp_path / "frag.md" + rc = validate.run("demo-plugin", "alice", "main", str(frag), repo="org/repo") + assert rc == 0 + assert _outputs(out)["result"] == "fail" + text = frag.read_text(encoding="utf-8") + assert "| `plugin.json` | ❌ | File missing |" in text + assert "//' || true) - if [[ -n "$META_ROW" ]]; then - IFS=$'\t' read -r f_name f_version f_description f_author f_maintainers f_repo_url f_discord_thread <<< "$META_ROW" - TABLE_ROWS+="| $f_name | $f_version | $f_description | $f_author | $f_maintainers |"$'\n' - if [[ -n "$f_repo_url" || -n "$f_discord_thread" ]]; then - PLUGIN_LINKS+="**\`${f_name}\`:**"$'\n' - [[ -n "$f_repo_url" ]] && PLUGIN_LINKS+="- [GitHub Repository](${f_repo_url})"$'\n' - [[ -n "$f_discord_thread" ]] && PLUGIN_LINKS+="- [Discord Thread](${f_discord_thread})"$'\n' - PLUGIN_LINKS+=$'\n' - fi - fi - - # Strip internal marker lines from visible output - VISIBLE=$(grep -v '" - echo "" - echo "# Plugin Validation Results" - echo "" - echo "**Modified plugins:** $PLUGIN_COUNT" - echo "" - - if [[ "${CLOSE_REASON:-}" == "no-valid-plugins" ]]; then - # echo "" - # echo "## Invalid Plugin Folder Name" - echo "" - echo "⚠️ Your PR modifies plugin folder(s) whose names do not meet the naming requirements. Plugin folder names must be **lowercase letters, numbers, and hyphens only** (e.g. \`my-plugin\`). Spaces and other special characters are not allowed." - echo "" - echo "Please rename the folder(s) and update your PR." - if [[ -n "${DISCORD_URL:-}" ]]; then - echo "" - echo "For help: [Dispatcharr Discord]($DISCORD_URL)" - fi - elif [[ "${CLOSE_REASON:-}" == "author-blacklisted" ]]; then - echo "" - echo "## PR Closed: Account Restricted" - echo "" - echo "Your GitHub account (\`$PR_AUTHOR\`) is not permitted to submit plugins to this repository. This PR has been automatically closed." - if [[ -n "${DISCORD_URL:-}" ]]; then - echo "" - echo "If you believe this is an error, please reach out via the [Dispatcharr Discord]($DISCORD_URL)." - fi - elif [[ "${CLOSE_REASON:-}" == "plugin-blacklisted" ]]; then - echo "" - echo "## PR Closed: Plugin Restricted" - echo "" - echo "One or more plugins in this PR are on the restricted list and cannot be submitted to this repository. This PR has been automatically closed." - if [[ -n "${DISCORD_URL:-}" ]]; then - echo "" - echo "If you believe this is an error, please reach out via the [Dispatcharr Discord]($DISCORD_URL)." - fi - elif [[ "$CLOSE_PR" == "true" ]]; then - echo "" - echo "## PR Closed: Unauthorized" - echo "" - echo "Your GitHub username (\`$PR_AUTHOR\`) does not appear in \`author\` or \`maintainers\` for any of the plugin(s) in this PR. This PR has been automatically closed." - echo "If you would like to contribute to this plugin, please consider reaching out to the maintainers of this plugin on Discord, or the plugin's Github repository." - echo "" - echo "If you are submitting a new plugin, add your GitHub username to the \`author\` field in your \`plugin.json\`." - if [[ -n "$PLUGIN_LINKS" ]]; then - echo "" - echo "### Plugin Contact Links" - echo "" - echo "$PLUGIN_LINKS" - fi - if [[ -n "${DISCORD_URL:-}" ]]; then - echo "" - echo "For general help or plugin discussion:" - echo "- [Dispatcharr Discord]($DISCORD_URL)" - fi - else - echo "$COMBINED_BODY" - - if [[ -n "${OUTSIDE_FILES:-}" && "${OUTSIDE_VIOLATION:-}" == "true" ]]; then - OVERALL_FAILED=1 - echo "" - echo "⚠️ This PR modifies files outside of \`plugins/\`, which requires write access to the repository. These changes will block merging." - echo "" - echo "External contributions to repository tooling and scripts are not accepted via PR. If you think something needs fixing, please [open an issue](https://github.com/${GITHUB_REPOSITORY}/issues/new/choose) instead." - echo "" - echo "**Modified files:**" - echo "\`\`\`" - echo "${OUTSIDE_FILES}" - echo "\`\`\`" - echo "" - echo "Remove these changes and resubmit with only modifications inside \`plugins/\`." - if [[ -n "${DISCORD_URL:-}" ]]; then - echo "" - echo "For help: [Dispatcharr Discord]($DISCORD_URL)" - fi - echo "" - fi - - if [[ "${PUB_KEY_CHANGED:-}" == "true" ]]; then - echo "" - echo "---" - echo "" - echo "### ⚠️ Signing Key Change Detected" - echo "" - echo "This PR modifies \`.github/scripts/keys/dispatcharr-plugins.pub\`. This is the public GPG key used by Dispatcharr to verify manifest signatures." - echo "" - echo "**Before merging, confirm:**" - echo "- The corresponding private key and passphrase secrets (\`GPG_PRIVATE_KEY\`, \`GPG_PASSPHRASE\`) have been updated in the repository settings." - echo "- The new public key has been bundled into the Dispatcharr application." - echo "- Existing embedded signatures in \`manifest.json\` files on the \`releases\` branch will be regenerated on next publish." - echo "" - fi - - # insert --- if there are ANY codeql/clamav findings, medium, low, or skip/unscanned notice - if [[ -n "${CODEQL_RESULT:-}" && "${CODEQL_RESULT:-}" != "skipped" && "${CODEQL_RESULT:-}" != "success" ]] || \ - [[ -n "${CODEQL_MEDIUMS:-}" && "${CODEQL_MEDIUMS}" != "0" && "${CODEQL_RESULT:-}" != "skipped" ]] || \ - [[ -n "${CODEQL_LOWS:-}" && "${CODEQL_LOWS}" != "0" && "${CODEQL_RESULT:-}" != "skipped" ]] || \ - [[ "${CODEQL_RESULT:-}" == "skipped" && -n "${CODEQL_UNSCANNED_LANGS:-}" ]] || \ - [[ "${CODEQL_RESULT:-}" != "skipped" && -n "${CODEQL_RESULT:-}" && -n "${CODEQL_UNSCANNED_LANGS:-}" ]] || \ - [[ "${CLAMAV_RESULT:-}" == "failure" ]]; then - echo "" - echo "---" - echo "" - fi - - if [[ "${CLAMAV_RESULT:-}" == "failure" ]]; then - OVERALL_FAILED=1 - INFECTED_LABEL="${CLAMAV_INFECTED:-unknown}" - echo "" - echo "❌ **ClamAV detected $INFECTED_LABEL infected file(s)**." - echo "" - if [[ -f "clamav-findings/clamav-findings.md" ]]; then - cat "clamav-findings/clamav-findings.md" - fi - fi - - if [[ -n "${CODEQL_RESULT:-}" && "${CODEQL_RESULT:-}" != "skipped" && "${CODEQL_RESULT:-}" != "success" ]]; then - # echo "" - # echo "## Code Quality" - echo "" - OVERALL_FAILED=1 - ERROR_LABEL="${CODEQL_ERRORS:-unknown}" - echo "❌ **CodeQL found $ERROR_LABEL high or critical issue(s)** - these must be fixed before merging." - echo "" - if [[ -f "codeql-findings/codeql-findings.md" ]]; then - cat "codeql-findings/codeql-findings.md" - fi - fi - - if [[ -n "${CODEQL_MEDIUMS:-}" && "${CODEQL_MEDIUMS:-}" != "0" && "${CODEQL_RESULT:-}" != "skipped" ]]; then - echo "" - echo "**CodeQL found ${CODEQL_MEDIUMS} medium severity issue(s)**" - echo "These are not blocking, but are included for visibility." - echo "" - if [[ -f "codeql-medium-findings/codeql-medium-findings.md" ]]; then - cat "codeql-medium-findings/codeql-medium-findings.md" - fi - fi - - if [[ -n "${CODEQL_LOWS:-}" && "${CODEQL_LOWS:-}" != "0" && "${CODEQL_RESULT:-}" != "skipped" ]]; then - echo "" - echo "
" - echo "CodeQL found ${CODEQL_LOWS} low severity or informational result(s)" - echo "These are not blocking, but are included for visibility." - echo "" - if [[ -f "codeql-low-findings/codeql-low-findings.md" ]]; then - cat "codeql-low-findings/codeql-low-findings.md" - fi - echo "" - echo "
" - fi - - # CodeQL skipped notice (when no scannable files exist but unscannable types were found) - if [[ "${CODEQL_RESULT:-}" == "skipped" && -n "${CODEQL_UNSCANNED_LANGS:-}" ]]; then - UNSCANNED_DISPLAY=$(echo "${CODEQL_UNSCANNED_LANGS}" | tr ',' ' ') - echo "" - echo "**CodeQL analysis was skipped** - no supported source files were found. The following bundled file type(s) are not covered by CodeQL: \`${UNSCANNED_DISPLAY}\`." - echo "" - elif [[ -n "${CODEQL_UNSCANNED_LANGS:-}" && "${CODEQL_RESULT:-}" != "skipped" && -n "${CODEQL_RESULT:-}" ]]; then - UNSCANNED_DISPLAY=$(echo "${CODEQL_UNSCANNED_LANGS}" | tr ',' ' ') - echo "" - echo "**Note:** The following bundled file type(s) were not scanned by CodeQL (unsupported language): \`${UNSCANNED_DISPLAY}\`." - echo "" - fi - - if [[ -n "${TITLE_VALID:-}" && "${TITLE_VALID}" != "true" ]]; then - echo "" - echo "---" - echo "" - echo "### ❌ PR Title Format" - echo "" - echo "${TITLE_FEEDBACK}" - if [[ -n "${TITLE_SUGGESTION:-}" ]]; then - echo "" - echo "**Suggested format:** \`${TITLE_SUGGESTION}\`" - fi - echo "" - fi - - echo "" - echo "---" - echo "" - if [[ $OVERALL_FAILED -eq 0 ]]; then - echo "## 🎉 All validation checks passed!" - echo "" - echo "This PR modifies **$PLUGIN_COUNT** plugin(s) and all checks have passed." - else - echo "## ❌ Validation failed" - echo "" - echo "Some checks failed. Please review the errors above and update your PR." - fi - - if [[ -n "$OTHER_PLUGINS_SECTION" ]]; then - echo "" - echo "---" - echo "" - echo "$OTHER_PLUGINS_SECTION" - fi - - # if [[ -n "$PLUGIN_LINKS" ]]; then - # echo "" - # echo "---" - # echo "" - # echo "## Plugin Links" - # echo "" - # echo "$PLUGIN_LINKS" - # fi - - # if [[ -n "$TABLE_ROWS" ]]; then - # echo "" - # echo "---" - # echo "" - # echo "## Plugin Metadata" - # echo "" - # echo "$TABLE_HEADER" - # echo "$TABLE_SEP" - # echo "$TABLE_ROWS" - # fi - fi -} > pr_comment.txt - -# Minimize all previous validation comments as outdated before posting the new one -OWNER="${GITHUB_REPOSITORY%%/*}" -REPO="${GITHUB_REPOSITORY##*/}" - -PREV_NODE_IDS=$(gh api graphql -f query=' - query($owner: String!, $repo: String!, $number: Int!) { - repository(owner: $owner, name: $repo) { - pullRequest(number: $number) { - comments(first: 100) { - nodes { id body } - } - } - } - } -' -f owner="$OWNER" -f repo="$REPO" -F number="$PR_NUMBER" \ - --jq '.data.repository.pullRequest.comments.nodes[] - | select(.body | contains("")) - | .id' 2>/dev/null || true) - -if [[ -n "$PREV_NODE_IDS" ]]; then - while IFS= read -r node_id; do - gh api graphql -f query=' - mutation($id: ID!) { - minimizeComment(input: {subjectId: $id, classifier: OUTDATED}) { - minimizedComment { isMinimized } - } - } - ' -f id="$node_id" > /dev/null 2>&1 || true - done <<< "$PREV_NODE_IDS" - echo "Minimized $(echo "$PREV_NODE_IDS" | wc -l | tr -d ' ') previous validation comment(s) as outdated" -fi - -# Post PR comment - script succeeds/fails based on whether the comment posted -gh pr comment "$PR_NUMBER" --body "$(cat pr_comment.txt)" -COMMENT_EXIT=$? - -# Close PR for unauthorized plugin modifications -if [[ "$CLOSE_PR" == "true" && ( "${CLOSE_REASON:-}" == "unauthorized" || "${CLOSE_REASON:-}" == "author-blacklisted" || "${CLOSE_REASON:-}" == "plugin-blacklisted" ) ]]; then - gh pr close "$PR_NUMBER" - echo "PR #$PR_NUMBER closed: unauthorized" - exit $COMMENT_EXIT -fi - -exit $COMMENT_EXIT diff --git a/.github/scripts/validate/validate.sh b/.github/scripts/validate/validate.sh deleted file mode 100755 index 2dae7d4..0000000 --- a/.github/scripts/validate/validate.sh +++ /dev/null @@ -1,425 +0,0 @@ -#!/bin/bash -set -e - -# validate-single-plugin.sh -# Validates one plugin and writes a markdown report fragment to a file. -# -# Usage: validate-single-plugin.sh -# -# Arguments: -# plugin_name - Plugin folder name (e.g. my-plugin) -# pr_author - GitHub username of PR author -# base_ref - Base branch reference (e.g. main) -# output_file - File path to write the markdown report fragment to -# -# Outputs (written to $GITHUB_OUTPUT): -# result - "pass" or "fail" -# is_new - "true" if this is a new plugin (not on base branch) -# has_permission - "true" if pr_author is permitted to modify this plugin -# -# Environment variables required: -# GITHUB_REPOSITORY - Full repository name (owner/repo) -# GH_TOKEN - GitHub token for API access - -PLUGIN_NAME=$1 -PR_AUTHOR=$2 -BASE_REF=$3 -OUTPUT_FILE=${4:-/dev/stdout} - -if [[ -z "$PLUGIN_NAME" || -z "$PR_AUTHOR" || -z "$BASE_REF" ]]; then - echo "Usage: $0 [output_file]" - exit 1 -fi - -REPO_OWNER=$(echo "$GITHUB_REPOSITORY" | cut -d'/' -f1) -REPO_NAME=$(echo "$GITHUB_REPOSITORY" | cut -d'/' -f2) - -PLUGIN_DIR="plugins/$PLUGIN_NAME" -PLUGIN_JSON="$PLUGIN_DIR/plugin.json" -README="$PLUGIN_DIR/README.md" - -check_repo_maintainer() { - local author=$1 - PERMISSION=$(gh api repos/$REPO_OWNER/$REPO_NAME/collaborators/$author/permission --jq .permission 2>/dev/null || echo "none") - if [[ "$PERMISSION" == "admin" || "$PERMISSION" == "maintain" || "$PERMISSION" == "write" ]]; then - echo "1" - else - echo "0" - fi -} - -validate_semver() { - local version=$1 - if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "1"; else echo "0"; fi -} - -validate_dispatcharr_version() { - local version=$1 - if [[ "$version" =~ ^v?[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "1"; else echo "0"; fi -} - -version_greater_than() { - local new_version=$1 - local old_version=$2 - IFS='.' read -r NEW_MAJOR NEW_MINOR NEW_PATCH <<< "$new_version" - IFS='.' read -r OLD_MAJOR OLD_MINOR OLD_PATCH <<< "$old_version" - if (( 10#$NEW_MAJOR > 10#$OLD_MAJOR )); then return 0; fi - if (( 10#$NEW_MAJOR < 10#$OLD_MAJOR )); then return 1; fi - if (( 10#$NEW_MINOR > 10#$OLD_MINOR )); then return 0; fi - if (( 10#$NEW_MINOR < 10#$OLD_MINOR )); then return 1; fi - if (( 10#$NEW_PATCH > 10#$OLD_PATCH )); then return 0; fi - return 1 -} - -failed=0 -is_new="false" -has_permission="false" - -{ - echo "### Plugin: \`$PLUGIN_NAME\`" - echo "" - # Show description and repo link as header subtext if plugin.json is readable - if [[ -f "$PLUGIN_JSON" ]]; then - _desc=$(jq -r '.description // ""' "$PLUGIN_JSON" 2>/dev/null || true) - REPO_URL=$(jq -r '.repo_url // ""' "$PLUGIN_JSON" 2>/dev/null || true) - if [[ -n "$_desc" ]]; then - echo "_${_desc}_" - echo "" - fi - if [[ -n "$REPO_URL" ]]; then - echo "[Source Repository]($REPO_URL)" - echo "" - fi - fi - - TABLE_ROWS=() - - print_table() { - echo "| Check | Status | Details |" - echo "|-------|:------:|---------|" - for row in "${TABLE_ROWS[@]}"; do - echo "$row" - done - echo "" - } - - # ── Structural checks (hidden if pass) ─────────────────────────────────────── - - # Folder name format - if [[ ! "$PLUGIN_NAME" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]; then - TABLE_ROWS+=("| Folder name | ❌ | Must be lowercase-kebab-case - got \`$PLUGIN_NAME\`, e.g. \`my-plugin-name\` |") - failed=1 - fi - - # plugin.json existence (early exit) - if [[ ! -f "$PLUGIN_JSON" ]]; then - TABLE_ROWS+=("| \`plugin.json\` | ❌ | File missing |") - print_table - echo "result=fail" >> "$GITHUB_OUTPUT" - echo "is_new=false" >> "$GITHUB_OUTPUT" - echo "has_permission=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # JSON syntax (early exit) - if ! jq empty "$PLUGIN_JSON" 2>/dev/null; then - TABLE_ROWS+=("| JSON syntax | ❌ | Invalid JSON in plugin.json |") - print_table - echo "result=fail" >> "$GITHUB_OUTPUT" - echo "is_new=false" >> "$GITHUB_OUTPUT" - echo "has_permission=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # ── Required content ────────────────────────────────────────────────────────── - - # Required fields: name, version, description - shown as a combined row - MISSING_FIELDS=() - for key in name version description; do - if ! jq -e ".\"$key\"" "$PLUGIN_JSON" >/dev/null 2>&1; then - MISSING_FIELDS+=("\`$key\`") - failed=1 - fi - done - if [[ ${#MISSING_FIELDS[@]} -gt 0 ]]; then - MISSING_LIST=$(IFS=", "; echo "${MISSING_FIELDS[*]}") - TABLE_ROWS+=("| Required fields | ❌ | Missing: $MISSING_LIST |") - else - TABLE_ROWS+=("| Required fields | ✅ | All required fields present |") - fi - - # Extract metadata - AUTHOR=$(jq -r '.author // ""' "$PLUGIN_JSON") - MAINTAINERS=$(jq -r '[.maintainers[]?] | join(" ")' "$PLUGIN_JSON") - VERSION=$(jq -r '.version' "$PLUGIN_JSON") - - # Pre-fetch base branch plugin.json once - reused for version bump check and compare link - _base_pjson="" - OLD_VERSION="" - OLD_SOURCE_URL_TMPL="" - if git show "origin/${BASE_REF}:${PLUGIN_JSON}" > /dev/null 2>&1; then - _base_pjson=$(git show "origin/${BASE_REF}:${PLUGIN_JSON}") - OLD_VERSION=$(echo "$_base_pjson" | jq -r '.version // ""') - OLD_SOURCE_URL_TMPL=$(echo "$_base_pjson" | jq -r '.source_url // ""') - fi - - # ── External source checks ──────────────────────────────────────────────────── - source_type=$(jq -r '.source_type // "local"' "$PLUGIN_JSON") - release_link="" - compare_link="" - _gh_tag="" - _old_gh_tag="" - if [[ "$source_type" != "external" ]]; then - _src_count=$(find "$PLUGIN_DIR" -type f \ - ! -name 'plugin.json' ! -name 'README.md' ! -name 'logo.png' \ - | wc -l | tr -d ' ') - if [[ "$_src_count" -eq 0 ]]; then - TABLE_ROWS+=("| Plugin files | ❌ | No source files found in \`plugins/$PLUGIN_NAME/\`. Add your plugin source (e.g. \`main.py\`) or set \`\"source_type\": \"external\"\` in \`plugin.json\` if your plugin is hosted elsewhere |") - failed=1 - fi - fi - if [[ "$source_type" == "external" ]]; then - ext_source_url=$(jq -r '.source_url // ""' "$PLUGIN_JSON") - ext_repo_url=$(jq -r '.repo_url // ""' "$PLUGIN_JSON") - - if [[ -z "$ext_source_url" ]]; then - TABLE_ROWS+=("| \`source_url\` | ❌ | Required when \`source_type\` is \`external\` |") - failed=1 - elif [[ ! "$ext_source_url" =~ ^https:// ]]; then - TABLE_ROWS+=("| \`source_url\` | ❌ | Must be an HTTPS URL |") - failed=1 - elif [[ "$ext_source_url" != *"{version}"* ]]; then - TABLE_ROWS+=("| \`source_url\` | ❌ | Must contain a \`{version}\` placeholder (e.g. \`.../v{version}/plugin.zip\`) |") - failed=1 - else - # TABLE_ROWS+=("| \`source_url\` | ✅ | \`${ext_source_url}\` |") - if [[ $(validate_semver "$VERSION") -eq 1 ]]; then - resolved_url="${ext_source_url//\{version\}/$VERSION}" - http_code=$(curl -o /dev/null -s -w "%{http_code}" --max-time 15 -L "$resolved_url" || echo "000") - if [[ "$http_code" == "200" ]]; then - TABLE_ROWS+=("| Release artifact | ✅ | Artifact reachable at resolved URL |") - if [[ "$resolved_url" =~ ^https://github\.com/([^/]+)/([^/]+)/releases/download/([^/]+)/ ]]; then - _gh_owner="${BASH_REMATCH[1]}" - _gh_repo="${BASH_REMATCH[2]}" - _gh_tag="${BASH_REMATCH[3]}" - release_link="https://github.com/${_gh_owner}/${_gh_repo}/releases/tag/${_gh_tag}" - if [[ -n "$OLD_VERSION" && -n "$OLD_SOURCE_URL_TMPL" ]]; then - _old_resolved="${OLD_SOURCE_URL_TMPL//\{version\}/$OLD_VERSION}" - if [[ "$_old_resolved" =~ ^https://github\.com/[^/]+/[^/]+/releases/download/([^/]+)/ ]]; then - _old_gh_tag="${BASH_REMATCH[1]}" - compare_link="https://github.com/${_gh_owner}/${_gh_repo}/compare/${_old_gh_tag}...${_gh_tag}" - fi - fi - fi - else - TABLE_ROWS+=("| Release artifact | ❌ | Could not reach \`$resolved_url\` (HTTP \`$http_code\`) — ensure the release exists |") - failed=1 - fi - fi - fi - - if [[ -z "$ext_repo_url" ]]; then - TABLE_ROWS+=("| \`repo_url\` | ❌ | Required for external plugins — set to the upstream source repository URL |") - failed=1 - fi - fi - - # Maintainers - if [[ -z "$AUTHOR" ]] && [[ -z "$MAINTAINERS" ]]; then - TABLE_ROWS+=("| Maintainers | ❌ | At least one of \`author\` or \`maintainers\` must include your GitHub username |") - failed=1 - else - DISPLAY_PARTS=() - [[ -n "$AUTHOR" ]] && DISPLAY_PARTS+=("\`$AUTHOR\`") - for m in $MAINTAINERS; do DISPLAY_PARTS+=("\`$m\`"); done - DISPLAY=$(printf '%s, ' "${DISPLAY_PARTS[@]}"); DISPLAY="${DISPLAY%, }" - TABLE_ROWS+=("| Maintainers | ✅ | $DISPLAY |") - fi - - # License (required) - LICENSE_ID=$(jq -r '.license // ""' "$PLUGIN_JSON") - if [[ -z "$LICENSE_ID" ]]; then - TABLE_ROWS+=("| License | ❌ | \`license\` is required - provide an [OSI-approved SPDX identifier](https://spdx.org/licenses/) (e.g. \`MIT\`, \`Apache-2.0\`) |") - failed=1 - else - SPDX_JSON=$(curl -fsSL "https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json" 2>/dev/null || echo "") - if [[ -z "$SPDX_JSON" ]]; then - TABLE_ROWS+=("| License | ⚠️ | Could not fetch SPDX license list - skipping validation |") - else - SPDX_VALID=$(echo "$SPDX_JSON" | jq --arg lid "$LICENSE_ID" '[.licenses[] | select(.isOsiApproved == true) | .licenseId] | any(. == $lid)') - if [[ "$SPDX_VALID" == "true" ]]; then - LICENSE_NAME=$(echo "$SPDX_JSON" | jq -r --arg lid "$LICENSE_ID" '.licenses[] | select(.licenseId == $lid) | .name') - TABLE_ROWS+=("| License | ✅ | \`$LICENSE_ID\` - $LICENSE_NAME |") - else - TABLE_ROWS+=("| License | ❌ | \`$LICENSE_ID\` is not an [OSI-approved SPDX identifier](https://spdx.org/licenses/) |") - failed=1 - fi - fi - fi - - # ── Access control ──────────────────────────────────────────────────────────── - - # Permission check - use base branch version to prevent self-granting via the PR - IS_REPO_MAINTAINER=$(check_repo_maintainer "$PR_AUTHOR") - if [[ "$IS_REPO_MAINTAINER" -eq 1 ]]; then - TABLE_ROWS+=("| Permission | ✅ | You have permission to modify this plugin |") - has_permission="true" - elif git show "origin/${BASE_REF}:${PLUGIN_JSON}" > /dev/null 2>&1; then - BASE_JSON=$(git show "origin/${BASE_REF}:${PLUGIN_JSON}") - BASE_AUTHOR=$(echo "$BASE_JSON" | jq -r '.author // ""') - BASE_MAINTAINERS=$(echo "$BASE_JSON" | jq -r '[.maintainers[]?] | join(" ")') - if [[ "$PR_AUTHOR" == "$BASE_AUTHOR" ]] || [[ " $BASE_MAINTAINERS " =~ " $PR_AUTHOR " ]]; then - TABLE_ROWS+=("| Permission | ✅ | You have permission to modify this plugin |") - has_permission="true" - else - TABLE_ROWS+=("| Permission | ❌ | \`$PR_AUTHOR\` is not listed in \`author\` or \`maintainers\` |") - failed=1 - fi - else - # New plugin - PR author must list themselves so future PRs can be authorized - if [[ "$PR_AUTHOR" == "$AUTHOR" ]] || [[ " $MAINTAINERS " =~ " $PR_AUTHOR " ]]; then - TABLE_ROWS+=("| Permission | ✅ | New plugin - \`$PR_AUTHOR\` listed in \`author\`/\`maintainers\` |") - has_permission="true" - else - TABLE_ROWS+=("| Permission | ❌ | Add \`\"author\": \"$PR_AUTHOR\"\` to plugin.json |") - failed=1 - fi - fi - - # ── Version checks ──────────────────────────────────────────────────────────── - - # Version format - if [[ $(validate_semver "$VERSION") -eq 1 ]]; then - TABLE_ROWS+=("| Version | ✅ | \`$VERSION\` |") - else - TABLE_ROWS+=("| Version | ❌ | \`$VERSION\` is not valid semver - expected \`X.Y.Z\` |") - failed=1 - fi - - # Version bump - # These fields may be updated without a version bump - they are metadata only - # and do not affect the packaged ZIP artifact. - METADATA_ONLY_FIELDS=("description" "repo_url" "discord_thread" - "min_dispatcharr_version" "max_dispatcharr_version" "deprecated" "unlisted" "maintainers") - - if [[ -n "$_base_pjson" ]]; then - if version_greater_than "$VERSION" "$OLD_VERSION"; then - TABLE_ROWS+=("| Version bump | ✅ | \`$OLD_VERSION\` → \`$VERSION\` |") - else - # Version unchanged - check if every changed field is in the metadata-only allowlist - OLD_JSON="$_base_pjson" - NEW_JSON=$(cat "$PLUGIN_JSON") - - # Produce a newline-separated list of field names that differ (raw strings, no quotes) - changed_fields=$(jq -rn \ - --argjson old "$OLD_JSON" \ - --argjson new "$NEW_JSON" \ - '[$new | keys[]] | map(select($old[.] != $new[.])) | .[]' 2>/dev/null || true) - - metadata_only_change=true - while IFS= read -r field; do - [[ -z "$field" ]] && continue - allowed=false - for mf in "${METADATA_ONLY_FIELDS[@]}"; do - [[ "$field" == "$mf" ]] && allowed=true && break - done - $allowed || { metadata_only_change=false; break; } - done <<< "$changed_fields" - - if $metadata_only_change && [[ -n "$changed_fields" ]]; then - TABLE_ROWS+=("| Version bump | ✅ | \`$OLD_VERSION\` (unchanged - metadata-only update) |") - else - # Any non-metadata plugin.json change or other file change requires a version bump. - # If plugin.json is identical, check whether other files in the directory changed - # so we can give a more accurate error message. - _other_changed="" - if [[ -z "$changed_fields" ]]; then - _other_changed=$(git diff --name-only "origin/${BASE_REF}...HEAD" -- "$PLUGIN_DIR" \ - | grep -v "^${PLUGIN_JSON}$" | head -1 || true) - fi - if [[ -z "$changed_fields" && -z "$_other_changed" ]]; then - TABLE_ROWS+=("| Version bump | ❌ | No changes detected - nothing to publish |") - else - TABLE_ROWS+=("| Version bump | ❌ | \`$VERSION\` must be greater than current \`$OLD_VERSION\` |") - fi - failed=1 - fi - fi - else - TABLE_ROWS+=("| Version bump | ✅ | New plugin |") - is_new="true" - fi - - # Dispatcharr version constraints (optional, hidden if pass) - MIN_DA_VERSION=$(jq -r '.min_dispatcharr_version // ""' "$PLUGIN_JSON") - MAX_DA_VERSION=$(jq -r '.max_dispatcharr_version // ""' "$PLUGIN_JSON") - - if [[ -n "$MIN_DA_VERSION" ]] && [[ $(validate_dispatcharr_version "$MIN_DA_VERSION") -eq 0 ]]; then - TABLE_ROWS+=("| \`min_dispatcharr_version\` | ❌ | \`$MIN_DA_VERSION\` is not valid semver - expected \`X.Y.Z\` or \`vX.Y.Z\` |") - failed=1 - fi - - if [[ -n "$MAX_DA_VERSION" ]] && [[ $(validate_dispatcharr_version "$MAX_DA_VERSION") -eq 0 ]]; then - TABLE_ROWS+=("| \`max_dispatcharr_version\` | ❌ | \`$MAX_DA_VERSION\` is not valid semver - expected \`X.Y.Z\` or \`vX.Y.Z\` |") - failed=1 - fi - - # min/max range sanity (hidden if pass) - if [[ -n "$MIN_DA_VERSION" && -n "$MAX_DA_VERSION" ]]; then - if [[ $(validate_dispatcharr_version "$MAX_DA_VERSION") -eq 1 && $(validate_dispatcharr_version "$MIN_DA_VERSION") -eq 1 ]]; then - _max="${MAX_DA_VERSION#v}" - _min="${MIN_DA_VERSION#v}" - if ! version_greater_than "$_max" "$_min" && [[ "$_max" != "$_min" ]]; then - TABLE_ROWS+=("| Version range | ❌ | \`max_dispatcharr_version\` (\`$MAX_DA_VERSION\`) must be ≥ \`min_dispatcharr_version\` (\`$MIN_DA_VERSION\`) |") - failed=1 - fi - fi - fi - - # ── Optional link fields (hidden if pass) ──────────────────────────────────── - - DISCORD_THREAD=$(jq -r '.discord_thread // ""' "$PLUGIN_JSON") - - if [[ -n "$REPO_URL" ]] && [[ ! "$REPO_URL" =~ ^https?:// ]]; then - TABLE_ROWS+=("| \`repo_url\` | ❌ | Must start with \`http://\` or \`https://\` |") - failed=1 - fi - - if [[ -n "$DISCORD_THREAD" ]] && [[ ! "$DISCORD_THREAD" =~ ^https?:// ]]; then - TABLE_ROWS+=("| \`discord_thread\` | ❌ | Must start with \`http://\` or \`https://\` |") - failed=1 - fi - - print_table - - if [[ -n "$release_link" ]]; then - _link_line="[View release ${_gh_tag} on GitHub](${release_link})" - [[ -n "$compare_link" ]] && _link_line+=" · [Compare ${_old_gh_tag}...${_gh_tag}](${compare_link})" - echo "$_link_line" - echo "" - fi - - # Metadata row (tab-delimited, consumed by aggregate-report.sh) - echo "" - -} > "$OUTPUT_FILE" - - -# Write job outputs -if [[ $failed -eq 0 ]]; then - echo "result=pass" >> "$GITHUB_OUTPUT" -else - echo "result=fail" >> "$GITHUB_OUTPUT" -fi -echo "is_new=$is_new" >> "$GITHUB_OUTPUT" -echo "has_permission=$has_permission" >> "$GITHUB_OUTPUT" - -exit $failed diff --git a/.github/workflows/_clamav-scan.yml b/.github/workflows/_clamav-scan.yml new file mode 100644 index 0000000..1207b95 --- /dev/null +++ b/.github/workflows/_clamav-scan.yml @@ -0,0 +1,182 @@ +name: ClamAV Scan (reusable) + +# Reusable workflow called by validate-plugin.yml. Virus-scans the changed +# plugins of a PR merge commit. pluginctl runs from the base branch (_trusted), +# never from the untrusted merge tree. + +on: + workflow_call: + inputs: + matrix: + required: true + type: string + pr_number: + required: true + type: string + base_ref: + required: true + type: string + outputs: + clamav_status: + value: ${{ jobs.scan.outputs.clamav_status }} + clamav_infected: + value: ${{ jobs.scan.outputs.clamav_infected }} + +permissions: + contents: read + pull-requests: write + issues: write + +env: + PYTHONPATH: ${{ github.workspace }}/_trusted/.github/pluginctl/src + +jobs: + scan: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + clamav_status: ${{ steps.status.outputs.clamav_status }} + clamav_infected: ${{ steps.status.outputs.clamav_infected }} + steps: + - name: Checkout PR merge commit for scan + uses: actions/checkout@v6 + with: + ref: refs/pull/${{ inputs.pr_number }}/merge + sparse-checkout: .gitignore + sparse-checkout-cone-mode: false + # Merge ref is only virus-scanned by ClamAV; fork code is not executed. + allow-unsafe-pr-checkout: true + + - name: Checkout trusted pluginctl (base) + uses: actions/checkout@v6 + with: + repository: ${{ github.repository }} + ref: ${{ inputs.base_ref }} + path: _trusted + sparse-checkout: .github/pluginctl + sparse-checkout-cone-mode: false + + - name: Limit checkout to changed plugin folders only + run: | + echo '${{ inputs.matrix }}' | jq -r '.[] | "plugins/\(.)"' | git sparse-checkout set --stdin + git checkout + + - name: Get cache keys + id: cache-keys + run: | + echo "week=$(date +%Y-W%V)" >> $GITHUB_OUTPUT + echo "date=$(date +%Y%m%d)" >> $GITHUB_OUTPUT + + - name: Cache ClamAV installation (weekly) + id: cache-clamav-install + uses: actions/cache@v5 + with: + path: /tmp/clamav-apt + key: clamav-install-${{ runner.os }}-${{ steps.cache-keys.outputs.week }} + restore-keys: clamav-install-${{ runner.os }}- + + - name: Cache ClamAV virus definitions (daily) + id: cache-clamav-defs + uses: actions/cache@v5 + with: + path: /tmp/clamav-db + key: clamav-defs-${{ runner.os }}-${{ steps.cache-keys.outputs.date }} + restore-keys: clamav-defs-${{ runner.os }}- + + - name: Install ClamAV + run: | + sudo apt-get update -qq + if [[ "${{ steps.cache-clamav-install.outputs.cache-hit }}" == 'true' ]]; then + echo "Installing ClamAV from cached packages..." + sudo cp /tmp/clamav-apt/*.deb /var/cache/apt/archives/ 2>/dev/null || true + fi + sudo apt-get install -y --no-install-recommends clamav + if [[ "${{ steps.cache-clamav-install.outputs.cache-hit }}" != 'true' ]]; then + echo "Saving ClamAV packages to cache..." + mkdir -p /tmp/clamav-apt + find /var/cache/apt/archives/ \( -name 'clamav*.deb' -o -name 'libclamav*.deb' \) \ + -exec cp {} /tmp/clamav-apt/ \; 2>/dev/null || true + fi + + - name: Update virus definitions + run: | + sudo systemctl stop clamav-freshclam 2>/dev/null || true + sudo mkdir -p /tmp/clamav-db + sudo chown -R clamav:clamav /tmp/clamav-db + if [[ "${{ steps.cache-clamav-defs.outputs.cache-hit }}" != 'true' ]]; then + echo "Downloading fresh ClamAV definitions..." + sudo freshclam --datadir=/tmp/clamav-db + else + echo "Using cached ClamAV definitions" + fi + + - name: Scan changed plugin directories + id: scan + run: | + SCAN_TARGETS=() + EXT_SCAN_DIR="/tmp/external-scan" + mkdir -p "$EXT_SCAN_DIR" + while IFS= read -r plugin_name; do + plugin_json="plugins/$plugin_name/plugin.json" + [[ ! -f "$plugin_json" ]] && continue + source_type=$(jq -r '.source_type // "local"' "$plugin_json") + if [[ "$source_type" == "external" ]]; then + version=$(jq -r '.version' "$plugin_json") + source_url_template=$(jq -r '.source_url // ""' "$plugin_json") + source_url="${source_url_template//\{version\}/$version}" + echo "Downloading external ZIP for ClamAV scan: $source_url" + scan_dir="$EXT_SCAN_DIR/$plugin_name" + mkdir -p "$scan_dir/extracted" + zip_file="$scan_dir/${plugin_name}.zip" + if curl -fsSL "$source_url" -o "$zip_file" --max-time 60; then + unzip -q "$zip_file" -d "$scan_dir/extracted" || true + SCAN_TARGETS+=("$scan_dir/extracted") + else + echo "::warning::Could not download $source_url, scanning plugin.json only for $plugin_name" + SCAN_TARGETS+=("plugins/$plugin_name") + fi + else + SCAN_TARGETS+=("plugins/$plugin_name") + fi + done < <(echo '${{ inputs.matrix }}' | jq -r '.[]') + echo "Scanning: ${SCAN_TARGETS[*]}" + set +e + clamscan --database=/tmp/clamav-db \ + --recursive --infected --no-summary \ + "${SCAN_TARGETS[@]}" > clamav-output.txt 2>&1 + echo "exit_code=$?" >> $GITHUB_OUTPUT + set -e + cat clamav-output.txt + + - name: Parse ClamAV output and set status + id: status + if: always() + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + run: python -m pluginctl clamav-report --output clamav-output.txt --scan-exit "${{ steps.scan.outputs.exit_code }}" + + - name: Upload ClamAV findings for PR comment + if: always() && steps.status.outputs.clamav_status == 'failure' + uses: actions/upload-artifact@v7 + with: + name: clamav-findings + path: clamav-findings.md + if-no-files-found: ignore + + - name: Apply quarantine label + if: always() && steps.status.outputs.clamav_status == 'failure' + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + gh label create "QUARANTINE" \ + --color "FFFF00" \ + --description "ClamAV detected a potential threat in this PR" \ + --repo "$GITHUB_REPOSITORY" 2>/dev/null || true + gh pr edit "${{ inputs.pr_number }}" \ + --add-label "QUARANTINE" \ + --repo "$GITHUB_REPOSITORY" + + - name: Fail job if ClamAV detected threats + if: always() && steps.status.outputs.clamav_status == 'failure' + run: exit 1 diff --git a/.github/workflows/_codeql-scan.yml b/.github/workflows/_codeql-scan.yml new file mode 100644 index 0000000..1b05218 --- /dev/null +++ b/.github/workflows/_codeql-scan.yml @@ -0,0 +1,158 @@ +name: CodeQL Scan (reusable) + +# Reusable workflow called by validate-plugin.yml. Runs CodeQL over the changed +# plugins of a PR merge commit and turns the SARIF into PR-comment inputs. +# pluginctl always runs from the base branch (checked out into _trusted), never +# from the untrusted merge tree. + +on: + workflow_call: + inputs: + matrix: + required: true + type: string + pr_number: + required: true + type: string + base_ref: + required: true + type: string + outputs: + codeql_status: + value: ${{ jobs.analyze.outputs.codeql_status }} + codeql_errors: + value: ${{ jobs.analyze.outputs.codeql_errors }} + codeql_warnings: + value: ${{ jobs.analyze.outputs.codeql_warnings }} + codeql_mediums: + value: ${{ jobs.analyze.outputs.codeql_mediums }} + codeql_lows: + value: ${{ jobs.analyze.outputs.codeql_lows }} + codeql_unscanned_langs: + value: ${{ jobs.analyze.outputs.codeql_unscanned_langs }} + +permissions: + contents: read + security-events: write + +env: + PYTHONPATH: ${{ github.workspace }}/_trusted/.github/pluginctl/src + +jobs: + analyze: + runs-on: ubuntu-latest + timeout-minutes: 15 + outputs: + codeql_status: ${{ steps.status.outputs.codeql_status }} + codeql_errors: ${{ steps.status.outputs.codeql_errors }} + codeql_warnings: ${{ steps.status.outputs.codeql_warnings }} + codeql_mediums: ${{ steps.status.outputs.codeql_mediums }} + codeql_lows: ${{ steps.status.outputs.codeql_lows }} + codeql_unscanned_langs: ${{ steps.status.outputs.codeql_unscanned_langs }} + steps: + - name: Checkout PR merge commit for analysis + uses: actions/checkout@v6 + with: + ref: refs/pull/${{ inputs.pr_number }}/merge + sparse-checkout: .gitignore + sparse-checkout-cone-mode: false + # Merge ref is analyzed statically by CodeQL only; fork code is not executed. + allow-unsafe-pr-checkout: true + + - name: Checkout trusted pluginctl (base) + uses: actions/checkout@v6 + with: + repository: ${{ github.repository }} + ref: ${{ inputs.base_ref }} + path: _trusted + sparse-checkout: .github/pluginctl + sparse-checkout-cone-mode: false + + - name: Limit checkout to changed plugin folders only + run: | + echo '${{ inputs.matrix }}' | jq -r '.[] | "plugins/\(.)"' | git sparse-checkout set --stdin + git checkout + + - name: Populate external plugin source for analysis + run: | + while IFS= read -r plugin_name; do + plugin_json="plugins/$plugin_name/plugin.json" + [[ ! -f "$plugin_json" ]] && continue + source_type=$(jq -r '.source_type // "local"' "$plugin_json") + [[ "$source_type" != "external" ]] && continue + version=$(jq -r '.version' "$plugin_json") + source_url_template=$(jq -r '.source_url // ""' "$plugin_json") + source_url="${source_url_template//\{version\}/$version}" + echo "Fetching release ZIP for CodeQL: $source_url" + curl -fsSL "$source_url" -o "/tmp/${plugin_name}-release.zip" + mkdir -p "/tmp/${plugin_name}-src" + python3 -c "import zipfile,os; z=zipfile.ZipFile('/tmp/${plugin_name}-release.zip'); d='/tmp/${plugin_name}-src'; [z.extract((setattr(m,'filename',m.filename.replace(chr(92),'/')) or m),d) for m in z.infolist()]" + cp -rn "/tmp/${plugin_name}-src/." "plugins/$plugin_name/" + rm -rf "/tmp/${plugin_name}-release.zip" "/tmp/${plugin_name}-src" + done < <(echo '${{ inputs.matrix }}' | jq -r '.[]') + + - name: Detect supported languages + id: detect-langs + run: python -m pluginctl detect-langs + + - name: Initialize CodeQL + if: steps.detect-langs.outputs.found == 'true' + uses: github/codeql-action/init@v4 + with: + languages: ${{ steps.detect-langs.outputs.languages }} + build-mode: none + queries: security-extended + + - name: Perform CodeQL Analysis + if: steps.detect-langs.outputs.found == 'true' + id: analyze + uses: github/codeql-action/analyze@v4 + with: + category: pr-${{ inputs.pr_number }} + output: sarif-results + upload: false + continue-on-error: true + + - name: Parse SARIF and set CodeQL status + id: status + if: always() + env: + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + python -m pluginctl sarif \ + --results-dir sarif-results \ + --repo "${{ github.repository }}" \ + --sha "$(git rev-parse HEAD)" \ + --matrix '${{ inputs.matrix }}' \ + --analyze-outcome "${{ steps.analyze.outcome }}" \ + --found "${{ steps.detect-langs.outputs.found }}" \ + --languages "${{ steps.detect-langs.outputs.languages }}" \ + --unscanned-langs "${{ steps.detect-langs.outputs.unscanned_langs }}" + + - name: Upload findings detail for PR comment + if: always() && steps.status.outputs.codeql_status == 'failure' + uses: actions/upload-artifact@v7 + with: + name: codeql-findings + path: codeql-findings.md + if-no-files-found: ignore + + - name: Upload medium findings detail for PR comment + if: always() + uses: actions/upload-artifact@v7 + with: + name: codeql-medium-findings + path: codeql-medium-findings.md + if-no-files-found: ignore + + - name: Upload low findings detail for PR comment + if: always() + uses: actions/upload-artifact@v7 + with: + name: codeql-low-findings + path: codeql-low-findings.md + if-no-files-found: ignore + + - name: Fail job if CodeQL found high/error/critical issues + if: always() && steps.status.outputs.codeql_status == 'failure' + run: exit 1 diff --git a/.github/workflows/auto-merge-updates.yml b/.github/workflows/auto-merge-updates.yml index 92934ca..51d3841 100644 --- a/.github/workflows/auto-merge-updates.yml +++ b/.github/workflows/auto-merge-updates.yml @@ -1,18 +1,19 @@ name: Auto-merge Plugin Updates # Opt-in via the AUTO_MERGE_UPDATES repo variable. When enabled, squash-merges -# a PR automatically once "Validate Plugin" succeeds - but ONLY for pure -# plugin version-bump updates. Never for new plugins, repo/tooling changes, -# invalid PRs, or PRs that mix a new plugin with an update. +# a PR automatically once "Validate Plugin" succeeds, but ONLY for pure plugin +# version-bump updates. Never for new plugins, repo/tooling changes, invalid +# PRs, or PRs that mix a new plugin with an update. # # Runs the merge with the GitHub App token specifically (never GITHUB_TOKEN): # pushes authored by the App's bot identity correctly cascade into -# publish-plugins.yml's `on: push` trigger, whereas GITHUB_TOKEN-authored -# pushes are exempt from triggering other workflows. If the App isn't -# configured, auto-merge is skipped entirely rather than falling back. +# publish-plugins.yml's `on: push` trigger, whereas GITHUB_TOKEN-authored pushes +# are exempt from triggering other workflows. If the App isn't configured, +# auto-merge is skipped entirely rather than falling back. env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + PYTHONPATH: .github/pluginctl/src on: workflow_run: @@ -33,149 +34,32 @@ jobs: group: auto-merge-updates-${{ github.event.workflow_run.head_sha }} cancel-in-progress: false steps: - - name: Checkout base branch scripts (trusted) + - name: Checkout base branch (trusted) uses: actions/checkout@v6 with: repository: ${{ github.repository }} ref: main fetch-depth: 0 - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - name: Skip - GitHub App not configured - if: steps.config.outputs.use_app != 'true' + if: steps.token.outputs.use_app != 'true' run: | echo "::notice::GH_APP_ID or GH_APP_PRIVATE_KEY not configured. Skipping auto-merge (merging with GITHUB_TOKEN would not trigger publish-plugins.yml)." - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 - with: - client-id: ${{ steps.config.outputs.app_id }} - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - - name: Resolve PR for this workflow run - if: steps.config.outputs.use_app == 'true' - id: resolve-pr + - name: Auto-merge pure plugin update + if: steps.token.outputs.use_app == 'true' env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} + GH_TOKEN: ${{ steps.token.outputs.token }} GH_REPO: ${{ github.repository }} + GITHUB_REPOSITORY: ${{ github.repository }} HEAD_SHA: ${{ github.event.workflow_run.head_sha }} - run: | - # workflow_run.pull_requests is unreliable for pull_request_target runs - # from forks, so match on head SHA against open PRs instead. - MATCHES=$(gh pr list --state open --json number,headRefOid,isDraft \ - | jq -c --arg sha "$HEAD_SHA" '[.[] | select(.headRefOid == $sha)]') - COUNT=$(echo "$MATCHES" | jq 'length') - - if [[ "$COUNT" -ne 1 ]]; then - echo "::notice::Found $COUNT open PR(s) matching head SHA $HEAD_SHA - skipping (expected exactly 1)." - echo "found=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - PR_NUMBER=$(echo "$MATCHES" | jq -r '.[0].number') - IS_DRAFT=$(echo "$MATCHES" | jq -r '.[0].isDraft') - - if [[ "$IS_DRAFT" == "true" ]]; then - echo "::notice::PR #$PR_NUMBER is a draft - skipping." - echo "found=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - echo "found=true" >> "$GITHUB_OUTPUT" - echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT" - - - name: Check labels and mergeable state - if: steps.config.outputs.use_app == 'true' && steps.resolve-pr.outputs.found == 'true' - id: check-labels - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - GH_REPO: ${{ github.repository }} - PR_NUMBER: ${{ steps.resolve-pr.outputs.pr_number }} - run: | - INFO=$(gh pr view "$PR_NUMBER" --json labels,mergeable,mergeStateStatus) - LABELS=$(echo "$INFO" | jq -r '[.labels[].name] | join(",")') - MERGEABLE=$(echo "$INFO" | jq -r '.mergeable') - - echo "Labels: $LABELS" - echo "Mergeable: $MERGEABLE" - - has_label() { - echo ",$LABELS," | grep -qF ",$1," - } - - OK=true - has_label "Plugin Update" || { echo "::notice::Missing 'Plugin Update' label - skipping."; OK=false; } - has_label "New Plugin" && { echo "::notice::Has 'New Plugin' label - skipping."; OK=false; } - has_label "Repo Update" && { echo "::notice::Has 'Repo Update' label - skipping."; OK=false; } - has_label "Invalid" && { echo "::notice::Has 'Invalid' label - skipping."; OK=false; } - has_label "QUARANTINE" && { echo "::notice::Has 'QUARANTINE' label - skipping."; OK=false; } - [[ "$MERGEABLE" == "MERGEABLE" ]] || { echo "::notice::PR not cleanly mergeable (mergeable=$MERGEABLE) - skipping."; OK=false; } - - echo "ok=$OK" >> "$GITHUB_OUTPUT" - - - name: Independently re-verify this is a pure plugin update - if: steps.config.outputs.use_app == 'true' && steps.resolve-pr.outputs.found == 'true' && steps.check-labels.outputs.ok == 'true' - id: reverify - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - GH_REPO: ${{ github.repository }} - PR_NUMBER: ${{ steps.resolve-pr.outputs.pr_number }} - run: | - # Defense in depth: labels are applied by a trusted job, but they are - # ordinary PR labels and could in principle be hand-edited by anyone - # with write access. Re-derive the same signal directly from the diff - # rather than trusting labels alone. - git fetch origin "pull/${PR_NUMBER}/merge:pr-merge-${PR_NUMBER}" 2>/dev/null \ - || { echo "::notice::Could not fetch merge ref for PR #$PR_NUMBER - skipping."; echo "ok=false" >> "$GITHUB_OUTPUT"; exit 0; } - git checkout -q "pr-merge-${PR_NUMBER}" - - MERGE_BASE=$(git merge-base origin/main HEAD) - CHANGED=$(git diff --name-only "$MERGE_BASE" HEAD) - - OK=true - - OUTSIDE=$(echo "$CHANGED" | grep -v '^plugins/' || true) - if [[ -n "$OUTSIDE" ]]; then - echo "::notice::Changes outside plugins/ detected - skipping. Files: $OUTSIDE" - OK=false - fi - - PLUGIN_SLUGS=$(echo "$CHANGED" | grep '^plugins/' | cut -d'/' -f2 | sort -u) - if [[ -z "$PLUGIN_SLUGS" ]]; then - echo "::notice::No plugin changes detected in diff - skipping." - OK=false - fi - - while IFS= read -r slug; do - [[ -z "$slug" ]] && continue - if ! git show "origin/main:plugins/${slug}/plugin.json" > /dev/null 2>&1; then - echo "::notice::Plugin '$slug' does not exist on main - this is a new plugin, not an update. Skipping." - OK=false - fi - done <<< "$PLUGIN_SLUGS" - - echo "ok=$OK" >> "$GITHUB_OUTPUT" - - - name: Squash-merge PR - if: steps.config.outputs.use_app == 'true' && steps.resolve-pr.outputs.found == 'true' && steps.check-labels.outputs.ok == 'true' && steps.reverify.outputs.ok == 'true' - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - GH_REPO: ${{ github.repository }} - PR_NUMBER: ${{ steps.resolve-pr.outputs.pr_number }} - run: | - gh pr merge "$PR_NUMBER" --squash --delete-branch=false - echo "Auto-merged PR #$PR_NUMBER" + WEBHOOK_URL: ${{ vars.WEBHOOK_URL }} + WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} + run: python -m pluginctl automerge --head-sha "$HEAD_SHA" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 0b03718..0000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: CodeQL - -# Pull request analysis is handled by validate-plugin.yml (codeql-analyze job), -# which runs CodeQL as part of the plugin validation flow. -on: - workflow_dispatch: - # push: - # branches: [main] - # schedule: - # - cron: '0 6 * * 1' - -jobs: - analyze: - name: Analyze (python) - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Initialize CodeQL - uses: github/codeql-action/init@v4 - with: - languages: python - queries: security-and-quality - - - name: Autobuild - uses: github/codeql-action/autobuild@v4 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 - with: - category: /language:python \ No newline at end of file diff --git a/.github/workflows/publish-plugins.yml b/.github/workflows/publish-plugins.yml index dce61b2..f1bcb17 100644 --- a/.github/workflows/publish-plugins.yml +++ b/.github/workflows/publish-plugins.yml @@ -5,6 +5,7 @@ permissions: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + PYTHONPATH: .github/pluginctl/src on: push: @@ -13,12 +14,7 @@ on: paths: - 'plugins/**' - '.github/workflows/publish-plugins.yml' - - '.github/scripts/publish/run.sh' - - '.github/scripts/publish/build-zips.sh' - - '.github/scripts/publish/plugin-readmes.sh' - - '.github/scripts/publish/cleanup.sh' - - '.github/scripts/publish/generate-manifest.sh' - - '.github/scripts/publish/releases-readme.sh' + - '.github/pluginctl/**' - '.github/scripts/keys/**' workflow_dispatch: inputs: @@ -28,7 +24,7 @@ on: default: false required: false force_rebuild_plugin: - description: 'Optional: plugin slug to target (e.g. dispatcharr-exporter). Only that plugin is cleared and rebuilt - all others are untouched. Leave blank to wipe and rebuild the entire releases branch (WARNING: purges all version history).' + description: 'Optional: plugin slug to target (e.g. dispatcharr-exporter). Only that plugin is cleared and rebuilt, all others are untouched. Leave blank to wipe and rebuild the entire releases branch (WARNING: purges all version history).' type: string default: '' required: false @@ -52,53 +48,33 @@ jobs: with: fetch-depth: 0 - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - - name: Make scripts executable - run: chmod +x .github/scripts/publish/*.sh - - name: Publish plugins to releases branch id: publish env: - GITHUB_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} - APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + GITHUB_TOKEN: ${{ steps.token.outputs.token }} + APP_SLUG: ${{ steps.token.outputs.app-slug }} GITHUB_REPOSITORY: ${{ github.repository }} GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} FORCE_REBUILD: ${{ inputs.force_rebuild }} FORCE_REBUILD_CONFIRM: ${{ inputs.force_rebuild_confirm }} FORCE_REBUILD_PLUGIN: ${{ inputs.force_rebuild_plugin }} + WEBHOOK_URL: ${{ vars.WEBHOOK_URL }} + WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} run: | set -o pipefail if [[ "${FORCE_REBUILD}" == "true" && "${FORCE_REBUILD_CONFIRM}" != "CONFIRM" ]]; then echo "::error::force_rebuild requires typing CONFIRM in the force_rebuild_confirm input." exit 1 fi - .github/scripts/publish/run.sh ${{ github.ref_name }} 2>&1 | tee publish.log + python -m pluginctl publish ${{ github.ref_name }} 2>&1 | tee publish.log - name: Generate job summary if: always() @@ -107,17 +83,9 @@ jobs: if [ -f publish.log ]; then if grep -q "Successfully published" publish.log; then echo "✅ Publishing completed successfully" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - # Extract plugin info from the temporary repo if available - if [ -d /tmp/tmp.*/repo ]; then - REPO_DIR=$(find /tmp -maxdepth 1 -name "tmp.*" -type d 2>/dev/null | head -1)/repo - if [ -f "$REPO_DIR/manifest.json" ]; then - cat "$REPO_DIR/manifest.json" | jq -r '.manifest.plugins[] | "- **\(.name)** v\(.latest_version)"' >> $GITHUB_STEP_SUMMARY - fi - fi else echo "❌ Publishing failed" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "See logs for details." >> $GITHUB_STEP_SUMMARY fi - fi \ No newline at end of file + fi diff --git a/.github/workflows/update-external-readme.yml b/.github/workflows/update-external-readme.yml index 8341770..7a22aba 100644 --- a/.github/workflows/update-external-readme.yml +++ b/.github/workflows/update-external-readme.yml @@ -3,6 +3,9 @@ name: Publish Plugin Docs PR # Reads the README.md generated by the Publish Plugins workflow from the # releases branch and opens (or updates) a PR in a target repository. +env: + PYTHONPATH: .github/pluginctl/src + on: workflow_run: workflows: ["Publish Plugins"] @@ -17,9 +20,8 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest timeout-minutes: 10 - steps: - - name: Load and validate configuration + - name: Validate configuration id: config env: GH_APP_ID: ${{ vars.GH_APP_ID }} @@ -31,176 +33,37 @@ jobs: echo "::notice::GH_APP_ID, GH_APP_PRIVATE_KEY, EXTERNAL_README_REPO, or EXTERNAL_README_PATH not configured - skipping." echo "skip=true" >> "$GITHUB_OUTPUT" else - echo "skip=false" >> "$GITHUB_OUTPUT" - echo "app_id=$GH_APP_ID" >> "$GITHUB_OUTPUT" - echo "target_repo=$EXTERNAL_README_REPO" >> "$GITHUB_OUTPUT" - echo "target_path=$EXTERNAL_README_PATH" >> "$GITHUB_OUTPUT" + echo "skip=false" >> "$GITHUB_OUTPUT" fi - - name: Generate GitHub App token - if: steps.config.outputs.skip != 'true' - id: app-token - uses: actions/create-github-app-token@v3 - with: - client-id: ${{ steps.config.outputs.app_id }} - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - # Token is org-scoped; the app must be installed on the target repo - owner: ${{ github.repository_owner }} - - - name: Checkout releases branch README + # Keep the base branch checked out: the composite action and pluginctl live + # here. The releases-branch README is read via the origin/releases ref below. + - name: Checkout base branch (trusted) if: steps.config.outputs.skip != 'true' uses: actions/checkout@v6 with: - ref: releases fetch-depth: 1 - clean: false - sparse-checkout: README.md - sparse-checkout-cone-mode: false - - name: Extract release metadata from commit + - name: GitHub App token if: steps.config.outputs.skip != 'true' - id: meta - run: | - COMMIT_MSG=$(git log -1 --format="%B" HEAD) - - # Extract source commit SHA from "Source commit: " line - SOURCE_COMMIT=$(echo "$COMMIT_MSG" | grep -oP '(?<=Source commit: )[0-9a-f]+' || true) - - # Extract plugin list - lines starting with "- " after the source commit line - PLUGIN_LIST=$(echo "$COMMIT_MSG" | grep -E '^\- [a-z0-9].*@' || true) - - RELEASES_COMMIT=$(git rev-parse --short HEAD) - - echo "source_commit=${SOURCE_COMMIT}" >> "$GITHUB_OUTPUT" - echo "releases_commit=${RELEASES_COMMIT}" >> "$GITHUB_OUTPUT" - - # Write plugin list to a file to avoid quoting issues in multi-line output - printf '%s' "$PLUGIN_LIST" > /tmp/plugin_list.txt + id: token + uses: ./.github/actions/gh-app-token + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} - - name: Strip releases-branch preamble from README + - name: Fetch releases branch if: steps.config.outputs.skip != 'true' - env: - SOURCE_REPO: ${{ github.repository }} - run: | - # Remove the intro paragraph and Quick Access section that are only - # relevant on the releases branch itself, keeping everything from - # "## Available Plugins" onward (preserving the "# Plugin Releases" title). - CONTRIB_URL="https://github.com/${SOURCE_REPO}/blob/main/CONTRIBUTING.md" - awk -v contrib="$CONTRIB_URL" ' - /^# Plugin Releases/ { print; print ""; print "Want to get your plugin added to this list? Check out the [plugin repository](" contrib ") to learn how to contribute."; next } - /^## Available Plugins/ { found=1 } - found { print } - ' README.md > README.stripped.md - mv README.stripped.md README.md - - # Rewrite relative links (./zips/...) to absolute GitHub URLs pointing - # at the releases branch. Anchor links (#...) are unaffected because - # they never start with "./". - BASE_URL="https://github.com/${SOURCE_REPO}/tree/releases" - sed -i "s|](\./|](${BASE_URL}/|g" README.md + run: git fetch origin releases - - name: Create or update PR in target repo + - name: Create or update external README PR if: steps.config.outputs.skip != 'true' env: - TARGET_REPO: ${{ steps.config.outputs.target_repo }} - TARGET_PATH: ${{ steps.config.outputs.target_path }} - GH_TOKEN: ${{ steps.app-token.outputs.token }} + GH_TOKEN: ${{ steps.token.outputs.token }} + TARGET_REPO: ${{ vars.EXTERNAL_README_REPO }} + TARGET_PATH: ${{ vars.EXTERNAL_README_PATH }} SOURCE_REPO: ${{ github.repository }} - SOURCE_COMMIT: ${{ steps.meta.outputs.source_commit }} - RELEASES_COMMIT: ${{ steps.meta.outputs.releases_commit }} + GITHUB_REPOSITORY: ${{ github.repository }} PR_REVIEWER: ${{ vars.PR_REVIEWER }} - run: | - if [[ ! -f README.md ]]; then - echo "::error::README.md not found on the releases branch - has the Publish Plugins workflow run yet?" - exit 1 - fi - - PLUGIN_LIST=$(cat /tmp/plugin_list.txt) - - # Build a human-readable change summary block - build_summary() { - echo "**Source commit:** [\`${SOURCE_COMMIT}\`](https://github.com/${SOURCE_REPO}/commit/${SOURCE_COMMIT})" - echo "**Releases commit:** [\`${RELEASES_COMMIT}\`](https://github.com/${SOURCE_REPO}/commit/${RELEASES_COMMIT})" - if [[ -n "$PLUGIN_LIST" ]]; then - echo "" - echo "**Plugins updated:**" - while IFS= read -r line; do - echo "$line" - done <<< "$PLUGIN_LIST" - fi - } - - BRANCH="auto/dispatcharr-plugin-readme" - DEFAULT_BRANCH=$(gh api "repos/$TARGET_REPO" --jq '.default_branch') - - # Create branch if it does not already exist - if ! gh api "repos/$TARGET_REPO/branches/$BRANCH" > /dev/null 2>&1; then - BASE_SHA=$(gh api "repos/$TARGET_REPO/git/refs/heads/$DEFAULT_BRANCH" \ - --jq '.object.sha') - gh api "repos/$TARGET_REPO/git/refs" \ - -X POST \ - -f ref="refs/heads/$BRANCH" \ - -f sha="$BASE_SHA" - echo "Created branch $BRANCH" - fi - - # Get the existing file SHA and content from the branch - EXISTING_RESPONSE=$(gh api "repos/$TARGET_REPO/contents/$TARGET_PATH?ref=$BRANCH" 2>/dev/null || true) - FILE_SHA=$(echo "$EXISTING_RESPONSE" | jq -r '.sha // empty') - - # Skip if the only change is the timestamp line at the bottom of the README - if [[ -n "$FILE_SHA" ]]; then - EXISTING_CONTENT=$(echo "$EXISTING_RESPONSE" | jq -r '.content' | base64 -d 2>/dev/null || true) - NEW_STRIPPED=$(grep -v '^\*Last updated:' README.md || true) - EXISTING_STRIPPED=$(echo "$EXISTING_CONTENT" | grep -v '^\*Last updated:' || true) - if [[ "$NEW_STRIPPED" == "$EXISTING_STRIPPED" ]]; then - echo "README content unchanged (only timestamp differs) - skipping." - echo "::notice::External README not updated: only the timestamp line changed." - exit 0 - fi - fi - - # Upload file (base64-encoded content is required by the GitHub Contents API) - PUT_ARGS=( - -X PUT - -f message="chore: update plugin releases listing (source commit $SOURCE_COMMIT)" - -f content="$(base64 -w 0 README.md)" - -f branch="$BRANCH" - ) - [[ -n "$FILE_SHA" ]] && PUT_ARGS+=(-f sha="$FILE_SHA") - - gh api "repos/$TARGET_REPO/contents/$TARGET_PATH" "${PUT_ARGS[@]}" - - # Re-use an existing open PR for this branch; open a new one if none exists - EXISTING_PR=$(gh pr list \ - --repo "$TARGET_REPO" \ - --state open \ - --head "$BRANCH" \ - --json number \ - | jq -r '.[0].number // empty') - - if [[ -n "$EXISTING_PR" ]]; then - # Add a comment to the existing PR summarising this update - COMMENT=$(printf '### Plugin README update\n\n%s' "$(build_summary)") - gh pr comment "$EXISTING_PR" --repo "$TARGET_REPO" --body "$COMMENT" - echo "Added update comment to existing PR #$EXISTING_PR" - else - PR_BODY=$(printf 'Automated update of the plugin releases listings generated from [`%s`](https://github.com/%s).\n\n%s' \ - "$SOURCE_REPO" "$SOURCE_REPO" "$(build_summary)") - REVIEWER_ARGS=() - if [[ -n "${PR_REVIEWER:-}" ]]; then - IFS=',' read -ra _reviewers <<< "$PR_REVIEWER" - for _r in "${_reviewers[@]}"; do - _r="${_r#@}"; _r="${_r// /}" # strip @ and whitespace - [[ -n "$_r" ]] && REVIEWER_ARGS+=(--reviewer "$_r") - done - fi - gh pr create \ - --repo "$TARGET_REPO" \ - --head "$BRANCH" \ - --base "$DEFAULT_BRANCH" \ - --title "chore: update plugin releases listings" \ - --body "$PR_BODY" \ - "${REVIEWER_ARGS[@]}" - echo "Created new PR" - fi + run: python -m pluginctl external-readme diff --git a/.github/workflows/validate-plugin.yml b/.github/workflows/validate-plugin.yml index b21d602..9b19ddc 100644 --- a/.github/workflows/validate-plugin.yml +++ b/.github/workflows/validate-plugin.yml @@ -8,8 +8,9 @@ permissions: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + PYTHONPATH: .github/pluginctl/src # Comma-separated lists of blocked GitHub usernames and plugin slugs. - # Set these in repo Settings → Variables as AUTHOR_BLACKLIST and PLUGIN_BLACKLIST. + # Set these in repo Settings -> Variables as AUTHOR_BLACKLIST and PLUGIN_BLACKLIST. AUTHOR_BLACKLIST: ${{ vars.AUTHOR_BLACKLIST }} PLUGIN_BLACKLIST: ${{ vars.PLUGIN_BLACKLIST }} @@ -27,7 +28,7 @@ concurrency: jobs: # -------------------------------------------------------------------------- - # Job 1: Post a notice on draft PRs - no validation runs yet + # Job 1: Post a notice on draft PRs, no validation runs yet # -------------------------------------------------------------------------- draft-notice: if: github.event.pull_request.draft == true @@ -39,40 +40,21 @@ jobs: with: ref: ${{ github.event.pull_request.base.ref }} fetch-depth: 1 - sparse-checkout: .github/scripts + sparse-checkout: .github sparse-checkout-cone-mode: false - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - name: Post draft notice env: PR_NUMBER: ${{ github.event.pull_request.number }} - GH_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} - BOT_LOGIN: ${{ steps.config.outputs.use_app == 'true' && format('{0}[bot]', steps.app-token.outputs.app-slug) || 'github-actions[bot]' }} + GH_TOKEN: ${{ steps.token.outputs.token }} + BOT_LOGIN: ${{ steps.token.outputs.bot-login }} GH_REPO: ${{ github.repository }} run: | MARKER="" @@ -86,7 +68,7 @@ jobs: fi # -------------------------------------------------------------------------- - # Job 2: Detect which plugins changed and build the matrix + # Job 2: Detect which plugins changed and build the matrix (+ blacklist) # -------------------------------------------------------------------------- detect-changes: if: github.event.pull_request.draft == false @@ -95,8 +77,8 @@ jobs: outputs: matrix: ${{ steps.detect.outputs.matrix }} plugin_count: ${{ steps.detect.outputs.plugin_count }} - close_pr: ${{ steps.blacklist.outputs.close_pr || steps.detect.outputs.close_pr }} - close_reason: ${{ steps.blacklist.outputs.close_reason || steps.detect.outputs.close_reason }} + close_pr: ${{ steps.detect.outputs.close_pr }} + close_reason: ${{ steps.detect.outputs.close_reason }} outside_files: ${{ steps.detect.outputs.outside_files }} outside_violation: ${{ steps.detect.outputs.outside_violation }} skip_validation: ${{ steps.detect.outputs.skip_validation }} @@ -104,132 +86,31 @@ jobs: has_new_plugin: ${{ steps.detect.outputs.has_new_plugin }} has_updated_plugin: ${{ steps.detect.outputs.has_updated_plugin }} steps: - - name: Checkout base branch scripts (trusted) - uses: actions/checkout@v6 + - name: Trusted checkout + uses: ./.github/actions/trusted-checkout with: + base-ref: ${{ github.event.pull_request.base.ref }} + head-repo: ${{ github.event.pull_request.head.repo.full_name }} + head-sha: ${{ github.event.pull_request.head.sha }} repository: ${{ github.repository }} - ref: ${{ github.event.pull_request.base.ref }} - fetch-depth: 0 - sparse-checkout: .github/scripts - sparse-checkout-cone-mode: false - - - name: Fetch base branch - run: git fetch origin ${{ github.event.pull_request.base.ref }} - - - name: Save trusted scripts before fork checkout - run: cp -r .github/scripts /tmp/trusted-scripts - - - name: Checkout PR plugins (untrusted content only) - uses: actions/checkout@v6 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - sparse-checkout: plugins - sparse-checkout-cone-mode: false - clean: false - # Fork content is only read as data (sparse plugins/), never executed; - # trusted scripts are saved/restored around this step. - allow-unsafe-pr-checkout: true - - name: Restore trusted scripts - run: mkdir -p .github && cp -r /tmp/trusted-scripts .github/scripts - - - name: Re-fetch base branch refs - run: git fetch https://github.com/${{ github.repository }} +${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }} - - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - name: Detect changed plugins id: detect env: - GH_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} + GH_TOKEN: ${{ steps.token.outputs.token }} GITHUB_REPOSITORY: ${{ github.repository }} run: | - chmod +x .github/scripts/validate/*.sh - .github/scripts/validate/detect-changes.sh \ - "${{ github.event.pull_request.user.login }}" \ - "${{ github.event.pull_request.base.ref }}" \ - "${{ github.event.pull_request.head.ref }}" - - - name: Check author and plugin blacklists - id: blacklist - # Only run if detect didn't already decide to close (avoids redundant output) - if: steps.detect.outputs.close_pr != 'true' - env: - PR_AUTHOR: ${{ github.event.pull_request.user.login }} - MATRIX: ${{ steps.detect.outputs.matrix }} - run: | - AUTHOR_BL="${AUTHOR_BLACKLIST:-}" - PLUGIN_BL="${PLUGIN_BLACKLIST:-}" - - # No-op if neither list is configured - if [[ -z "$AUTHOR_BL" && -z "$PLUGIN_BL" ]]; then - exit 0 - fi - - MATCHED=false - REASON="" - - # Check author blacklist (case-insensitive, strips whitespace around commas) - if [[ -n "$AUTHOR_BL" ]]; then - IFS=',' read -ra BLOCKED <<< "$AUTHOR_BL" - for entry in "${BLOCKED[@]}"; do - slug="${entry// /}" - if [[ "${PR_AUTHOR,,}" == "${slug,,}" ]]; then - MATCHED=true - REASON="author-blacklisted" - echo "::warning::PR author '$PR_AUTHOR' is on the author blacklist." - break - fi - done - fi - - # Check plugin blacklist (case-insensitive) - if [[ "$MATCHED" != "true" && -n "$PLUGIN_BL" ]]; then - IFS=',' read -ra BLOCKED <<< "$PLUGIN_BL" - while IFS= read -r plugin; do - [[ -z "$plugin" ]] && continue - for entry in "${BLOCKED[@]}"; do - slug="${entry// /}" - if [[ "${plugin,,}" == "${slug,,}" ]]; then - MATCHED=true - REASON="plugin-blacklisted" - echo "::warning::Plugin '$plugin' is on the plugin blacklist." - break 2 - fi - done - done < <(printf '%s' "$MATRIX" | jq -r '.[]' 2>/dev/null || true) - fi - - if [[ "$MATCHED" == "true" ]]; then - echo "close_pr=true" >> "$GITHUB_OUTPUT" - echo "close_reason=$REASON" >> "$GITHUB_OUTPUT" - fi + python -m pluginctl detect \ + --author "${{ github.event.pull_request.user.login }}" \ + --base-ref "${{ github.event.pull_request.base.ref }}" \ + --head-ref "${{ github.event.pull_request.head.ref }}" # -------------------------------------------------------------------------- # Job 3: Apply PR labels based on change classification @@ -245,75 +126,31 @@ jobs: with: ref: ${{ github.event.pull_request.base.ref }} fetch-depth: 1 - sparse-checkout: .github/scripts + sparse-checkout: .github sparse-checkout-cone-mode: false - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - name: Apply labels env: - GH_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} + GH_TOKEN: ${{ steps.token.outputs.token }} GH_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - HAS_NEW_PLUGIN: ${{ needs.detect-changes.outputs.has_new_plugin }} - HAS_UPDATED_PLUGIN: ${{ needs.detect-changes.outputs.has_updated_plugin }} - OUTSIDE_FILES: ${{ needs.detect-changes.outputs.outside_files }} - OUTSIDE_VIOLATION: ${{ needs.detect-changes.outputs.outside_violation }} - CLOSE_PR: ${{ needs.detect-changes.outputs.close_pr }} run: | - apply_label() { - local label=$1 condition=$2 - if [[ "$condition" == "true" ]]; then - gh pr edit "$PR_NUMBER" --add-label "$label" || true - else - gh pr edit "$PR_NUMBER" --remove-label "$label" 2>/dev/null || true - fi - } - apply_label "New Plugin" "$HAS_NEW_PLUGIN" - apply_label "Plugin Update" "$HAS_UPDATED_PLUGIN" - # Repo Update only when there are outside files AND it's not a violation (authorized) - if [[ -n "$OUTSIDE_FILES" && "$OUTSIDE_VIOLATION" != "true" ]]; then - IS_REPO_UPDATE=true - else - IS_REPO_UPDATE=false - fi - apply_label "Repo Update" "$IS_REPO_UPDATE" - # Invalid when PR has unauthorized outside changes or unauthorized plugin modifications - if [[ "$OUTSIDE_VIOLATION" == "true" || "$CLOSE_PR" == "true" ]]; then - IS_INVALID=true - else - IS_INVALID=false - fi - apply_label "Invalid" "$IS_INVALID" + python -m pluginctl label \ + --pr "${{ github.event.pull_request.number }}" \ + --has-new-plugin "${{ needs.detect-changes.outputs.has_new_plugin }}" \ + --has-updated-plugin "${{ needs.detect-changes.outputs.has_updated_plugin }}" \ + --outside-files "${{ needs.detect-changes.outputs.outside_files }}" \ + --outside-violation "${{ needs.detect-changes.outputs.outside_violation }}" \ + --close-pr "${{ needs.detect-changes.outputs.close_pr }}" # -------------------------------------------------------------------------- # Job 4: Validate PR title format - # Runs on every non-draft, non-closing PR event (including 'edited', which - # fires when the title is renamed). # -------------------------------------------------------------------------- validate-title: needs: [detect-changes] @@ -325,697 +162,46 @@ jobs: title_feedback: ${{ steps.check.outputs.title_feedback }} title_suggestion: ${{ steps.check.outputs.title_suggestion }} steps: + - name: Checkout config + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.base.ref }} + fetch-depth: 1 + sparse-checkout: .github/pluginctl + sparse-checkout-cone-mode: false + - name: Validate PR title format id: check - env: - PR_TITLE: ${{ github.event.pull_request.title }} - PR_AUTHOR: ${{ github.event.pull_request.user.login }} - PLUGIN_COUNT: ${{ needs.detect-changes.outputs.plugin_count }} - MATRIX: ${{ needs.detect-changes.outputs.matrix }} - SKIP_VALIDATION: ${{ needs.detect-changes.outputs.skip_validation }} run: | - TITLE="$PR_TITLE" - AUTHOR="$PR_AUTHOR" - COUNT="${PLUGIN_COUNT:-0}" - SKIP="${SKIP_VALIDATION:-false}" - MATRIX_JSON="${MATRIX:-[]}" - TITLE_VALID=true - TITLE_FEEDBACK="" - TITLE_SUGGESTION="" - - # Build a context-appropriate suggestion for this PR - if [[ "$COUNT" == "0" ]]; then - CONTEXT_SUGGESTION="[repo]: Brief description of changes" - elif [[ "$COUNT" == "1" ]]; then - SLUG=$(printf '%s' "$MATRIX_JSON" | jq -r '.[0] // "plugin-slug"' || echo "plugin-slug") - CONTEXT_SUGGESTION="[$SLUG]: Brief description of changes" - else - CONTEXT_SUGGESTION="[$AUTHOR]: Brief description of changes" - fi - - # Check format and extract prefix using bash =~ to avoid grep/sed ERE inconsistencies. - # [^]] inside bash =~ means "not ]" (] is literal when first after [^). - if [[ "$TITLE" =~ ^\[([^]]+)\]:?[[:space:]]+.+ ]]; then - PREFIX="${BASH_REMATCH[1]}" - - # Validate prefix matches expected context - if [[ "$COUNT" == "0" ]]; then - if [[ "$PREFIX" != "repo" ]]; then - TITLE_VALID=false - TITLE_FEEDBACK="For repo-level or non-plugin changes, the prefix should be \`[repo]\`." - TITLE_SUGGESTION="$CONTEXT_SUGGESTION" - fi - elif [[ "$COUNT" == "1" ]]; then - EXPECTED=$(printf '%s' "$MATRIX_JSON" | jq -r '.[0] // ""' || echo "") - if [[ -n "$EXPECTED" && "$PREFIX" != "$EXPECTED" ]]; then - TITLE_VALID=false - TITLE_FEEDBACK="For a single plugin change, the prefix should match the plugin folder name: \`[$EXPECTED]\`." - TITLE_SUGGESTION="$CONTEXT_SUGGESTION" - fi - else - if [[ "$PREFIX" != "$AUTHOR" ]]; then - TITLE_VALID=false - TITLE_FEEDBACK="For changes to multiple plugins, the prefix should be your GitHub username: \`[$AUTHOR]\`." - TITLE_SUGGESTION="$CONTEXT_SUGGESTION" - fi - fi - else - TITLE_VALID=false - TITLE_FEEDBACK="PR title does not match the required format. Expected: \`[prefix] description\`." - TITLE_SUGGESTION="$CONTEXT_SUGGESTION" - fi - - echo "title_valid=$TITLE_VALID" >> "$GITHUB_OUTPUT" - { - echo "title_feedback<> "$GITHUB_OUTPUT" - { - echo "title_suggestion<> "$GITHUB_OUTPUT" - - if [[ "$TITLE_VALID" != "true" ]]; then - echo "::error::PR title does not match the required format. See CONTRIBUTING.md for details." - exit 1 - fi + python -m pluginctl check-title \ + --title "${{ github.event.pull_request.title }}" \ + --author "${{ github.event.pull_request.user.login }}" \ + --plugin-count "${{ needs.detect-changes.outputs.plugin_count }}" \ + --matrix '${{ needs.detect-changes.outputs.matrix }}' # -------------------------------------------------------------------------- - # Job 5: CodeQL security and quality analysis (runs after validate-plugin) + # Job 5: CodeQL security and quality analysis (reusable workflow) # -------------------------------------------------------------------------- codeql-analyze: needs: [detect-changes, validate-plugin, validate-title] if: needs.validate-plugin.result == 'success' && needs.detect-changes.outputs.skip_validation != 'true' && needs.validate-title.outputs.title_valid == 'true' - runs-on: ubuntu-latest - timeout-minutes: 15 - outputs: - codeql_status: ${{ steps.status.outputs.codeql_status }} - codeql_errors: ${{ steps.status.outputs.codeql_errors }} - codeql_warnings: ${{ steps.status.outputs.codeql_warnings }} - codeql_mediums: ${{ steps.status.outputs.codeql_mediums }} - codeql_lows: ${{ steps.status.outputs.codeql_lows }} - codeql_unscanned_langs: ${{ steps.status.outputs.codeql_unscanned_langs }} - steps: - - name: Checkout PR merge commit for analysis - uses: actions/checkout@v6 - with: - ref: refs/pull/${{ github.event.pull_request.number }}/merge - # Minimal placeholder - overridden immediately by the next step - # (actions/checkout doesn't support dynamic expressions in sparse-checkout) - sparse-checkout: .gitignore - sparse-checkout-cone-mode: false - # Merge ref is analyzed statically by CodeQL only; fork code is not executed. - allow-unsafe-pr-checkout: true - - - name: Limit checkout to changed plugin folders only - run: | - echo '${{ needs.detect-changes.outputs.matrix }}' \ - | jq -r '.[] | "plugins/\(.)"' \ - | git sparse-checkout set --stdin - git checkout - - - name: Populate external plugin source for analysis - run: | - while IFS= read -r plugin_name; do - plugin_json="plugins/$plugin_name/plugin.json" - [[ ! -f "$plugin_json" ]] && continue - - source_type=$(jq -r '.source_type // "local"' "$plugin_json") - [[ "$source_type" != "external" ]] && continue - - version=$(jq -r '.version' "$plugin_json") - source_url_template=$(jq -r '.source_url // ""' "$plugin_json") - source_url="${source_url_template//\{version\}/$version}" - - echo "Fetching release ZIP for CodeQL: $source_url" - curl -fsSL "$source_url" -o "/tmp/${plugin_name}-release.zip" - - mkdir -p "/tmp/${plugin_name}-src" - python3 -c "import zipfile,os; z=zipfile.ZipFile('/tmp/${plugin_name}-release.zip'); d='/tmp/${plugin_name}-src'; [z.extract((setattr(m,'filename',m.filename.replace(chr(92),'/')) or m),d) for m in z.infolist()]" - # Copy extracted files into plugins dir without overwriting the registry plugin.json - cp -rn "/tmp/${plugin_name}-src/." "plugins/$plugin_name/" - rm -rf "/tmp/${plugin_name}-release.zip" "/tmp/${plugin_name}-src" - echo "Release ZIP for $plugin_name extracted into plugins/$plugin_name/" - done < <(echo '${{ needs.detect-changes.outputs.matrix }}' | jq -r '.[]') - - - name: Detect supported languages - id: detect-langs - run: | - LANGS=() - UNSCANNED=() - - # --- CodeQL-supported languages --- - if find plugins -name '*.py' -print -quit | grep -q .; then - LANGS+=(python) - fi - if find plugins \ - \( -path '*/node_modules/*' -o -path '*/dist/*' -o -path '*/build/*' -o -path '*/static/*' \) -prune -o \ - \( -name '*.js' -o -name '*.ts' -o -name '*.jsx' -o -name '*.tsx' \) -print \ - -quit | grep -q .; then - LANGS+=(javascript) - fi - if find plugins -name '*.go' -print -quit | grep -q .; then - LANGS+=(go) - fi - if find plugins -name '*.rb' -print -quit | grep -q .; then - LANGS+=(ruby) - fi - if find plugins \( -name '*.java' -o -name '*.kt' -o -name '*.kts' \) -print -quit | grep -q .; then - LANGS+=("java-kotlin") - fi - if find plugins \( -name '*.c' -o -name '*.cpp' -o -name '*.cc' -o -name '*.h' -o -name '*.hpp' \) -print -quit | grep -q .; then - LANGS+=("c-cpp") - fi - - # --- Files present but not supported by CodeQL --- - if find plugins \( -name '*.sh' -o -name '*.bash' \) -print -quit | grep -q .; then - UNSCANNED+=(shell) - fi - if find plugins -name '*.php' -print -quit | grep -q .; then - UNSCANNED+=(php) - fi - if find plugins -name '*.lua' -print -quit | grep -q .; then - UNSCANNED+=(lua) - fi - if find plugins \( -name '*.pl' -o -name '*.pm' \) -print -quit | grep -q .; then - UNSCANNED+=(perl) - fi - if find plugins -name '*.rs' -print -quit | grep -q .; then - UNSCANNED+=(rust) - fi - - UNSCANNED_CSV="" - [[ ${#UNSCANNED[@]} -gt 0 ]] && UNSCANNED_CSV=$(IFS=,; echo "${UNSCANNED[*]}") - echo "unscanned_langs=$UNSCANNED_CSV" >> "$GITHUB_OUTPUT" - - if [[ ${#LANGS[@]} -gt 0 ]]; then - LANG_CSV=$(IFS=,; echo "${LANGS[*]}") - echo "found=true" >> "$GITHUB_OUTPUT" - echo "languages=$LANG_CSV" >> "$GITHUB_OUTPUT" - echo "Detected languages for CodeQL: $LANG_CSV" - if [[ -n "$UNSCANNED_CSV" ]]; then echo "Unscanned (no CodeQL support): $UNSCANNED_CSV"; fi - else - echo "found=false" >> "$GITHUB_OUTPUT" - echo "languages=" >> "$GITHUB_OUTPUT" - echo "No supported language files found in changed plugins - skipping CodeQL." - if [[ -n "$UNSCANNED_CSV" ]]; then echo "Unscanned file types detected (no CodeQL support): $UNSCANNED_CSV"; fi - fi - - - name: Get merge commit SHA - id: merge - run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Initialize CodeQL - if: steps.detect-langs.outputs.found == 'true' - uses: github/codeql-action/init@v4 - with: - languages: ${{ steps.detect-langs.outputs.languages }} - build-mode: none - # Query suite options (slowest → fastest): - # security-and-quality – security + maintainability/style (quality results are discarded by our SARIF filter anyway) - # security-extended – all security severities, no quality queries (current) - # (omit queries:) – high-confidence security only; drops CVSS 6.0–6.9 medium findings our report surfaces - queries: security-extended - - - name: Perform CodeQL Analysis - if: steps.detect-langs.outputs.found == 'true' - id: analyze - uses: github/codeql-action/analyze@v4 - with: - category: pr-${{ github.event.pull_request.number }} - output: sarif-results - upload: false - continue-on-error: true - - - name: Set CodeQL status output - id: status - if: always() - run: | - # Only block on security findings with CVSS score >= 7.0 (HIGH or CRITICAL). - # CodeQL stores this in rule properties["security-severity"], not in result.level. - JQ_BLOCKING=' - [ .runs[] | - . as $run | - ( - [ (($run.tool.driver.rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}), - ((($run.tool.extensions // [])[].rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}) ] - | from_entries - ) as $secmap | - ($run.results // [])[] | - ((.ruleId // .rule.id // "") | tostring) as $rid | - select((($secmap[$rid] // "0") | tonumber) >= 7.0) - ] | length' - JQ_MEDIUM=' - [ .runs[] | - . as $run | - ( - [ (($run.tool.driver.rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}), - ((($run.tool.extensions // [])[].rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}) ] - | from_entries - ) as $secmap | - ($run.results // [])[] | - ((.ruleId // .rule.id // "") | tostring) as $rid | - (($secmap[$rid] // "0") | tonumber) as $sev | - select($sev >= 6.0 and $sev < 7.0) - ] | length' - JQ_LOW=' - [ .runs[] | - . as $run | - ( - [ (($run.tool.driver.rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}), - ((($run.tool.extensions // [])[].rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}) ] - | from_entries - ) as $secmap | - ($run.results // [])[] | - ((.ruleId // .rule.id // "") | tostring) as $rid | - (($secmap[$rid] // "0") | tonumber) as $sev | - select($sev < 6.0) - ] | length' - RESULT_COUNT=0 - MEDIUM_COUNT=0 - LOW_COUNT=0 - TOTAL_COUNT=0 - if [[ -d "sarif-results" ]]; then - for f in sarif-results/*.sarif sarif-results/*.sarif.gz; do - [[ -f "$f" ]] || continue - if [[ "$f" == *.gz ]]; then - CONTENT=$(gunzip -c "$f") - else - CONTENT=$(cat "$f") - fi - COUNT=$(echo "$CONTENT" | jq "$JQ_BLOCKING") - MED=$(echo "$CONTENT" | jq "$JQ_MEDIUM") - LOW=$(echo "$CONTENT" | jq "$JQ_LOW") - TOT=$(echo "$CONTENT" | jq '[.runs[] | (.results // [])[]] | length') - echo "File $f: ${COUNT:-0} blocking, ${MED:-0} medium, ${LOW:-0} low, $TOT total" - echo "$CONTENT" | jq -r \ - '[ .runs[] | . as $run | - ([ (($run.tool.driver.rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}), - ((($run.tool.extensions // [])[].rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}) ] | from_entries) as $secmap | - ($run.results // [])[] | - ((.ruleId // .rule.id // "") | tostring) as $rid | - select((($secmap[$rid] // "0") | tonumber) >= 7.0) | - " [blocking] \($rid) sec-sev=\($secmap[$rid] // "n/a")" ] | .[]' || true - RESULT_COUNT=$((RESULT_COUNT + ${COUNT:-0})) - MEDIUM_COUNT=$((MEDIUM_COUNT + ${MED:-0})) - LOW_COUNT=$((LOW_COUNT + ${LOW:-0})) - TOTAL_COUNT=$((TOTAL_COUNT + ${TOT:-0})) - done - fi - WARN_COUNT=$(( TOTAL_COUNT > RESULT_COUNT ? TOTAL_COUNT - RESULT_COUNT : 0 )) - echo "Found $RESULT_COUNT high/critical, $MEDIUM_COUNT medium, $LOW_COUNT low, and $WARN_COUNT other CodeQL result(s)" - echo "codeql_errors=$RESULT_COUNT" >> "$GITHUB_OUTPUT" - echo "codeql_warnings=$WARN_COUNT" >> "$GITHUB_OUTPUT" - echo "codeql_mediums=$MEDIUM_COUNT" >> "$GITHUB_OUTPUT" - echo "codeql_lows=$LOW_COUNT" >> "$GITHUB_OUTPUT" - - # Build list of external plugin path prefixes so findings links are suppressed - # for files that came from a downloaded ZIP and don't exist in this repo. - EXTERNAL_PREFIXES='[]' - while IFS= read -r _plugin_name; do - _pjson="plugins/$_plugin_name/plugin.json" - [[ -f "$_pjson" ]] || continue - _stype=$(jq -r '.source_type // "local"' "$_pjson" 2>/dev/null || echo "local") - if [[ "$_stype" == "external" ]]; then - EXTERNAL_PREFIXES=$(printf '%s' "$EXTERNAL_PREFIXES" | jq --arg p "plugins/$_plugin_name/" '. + [$p]') - fi - done < <(echo '${{ needs.detect-changes.outputs.matrix }}' | jq -r '.[]') - - # Generate a findings detail file for inclusion in the PR comment - if [[ "$RESULT_COUNT" -gt 0 ]]; then - MERGE_SHA=$(git rev-parse HEAD) - { - echo "| Rule | Location | Description |" - echo "|------|----------|-------------|" - for f in sarif-results/*.sarif sarif-results/*.sarif.gz; do - [[ -f "$f" ]] || continue - if [[ "$f" == *.gz ]]; then - FC=$(gunzip -c "$f") - else - FC=$(cat "$f") - fi - echo "$FC" | jq -r \ - --arg repo "$GITHUB_REPOSITORY" \ - --arg sha "$MERGE_SHA" \ - --argjson external_prefixes "$EXTERNAL_PREFIXES" \ - '.runs[] | - . as $run | - ( - [ (($run.tool.driver.rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}), - ((($run.tool.extensions // [])[].rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}) ] - | from_entries - ) as $secmap | - ($run.results // [])[] | - . as $result | - (($result.ruleId // $result.rule.id // "") | tostring) as $rid | - select((($secmap[$rid] // "0") | tonumber) >= 7.0) | - (.locations[0].physicalLocation.artifactLocation.uri // "?") as $uri | - ((.locations[0].physicalLocation.region.startLine // "?") | tostring) as $line | - (.message.text // "no description" | gsub("\n"; " ") | gsub("://"; "\u200b://") | gsub("www\\."; "www\u200b.") | gsub("#(?=[0-9])"; "#\u200b") | gsub("\\[(?[^\\]]+)\\][(][0-9]+[)]"; .c) | gsub("\\["; "[") | gsub("\\]"; "]") | if length > 150 then .[0:150] + "\u2026" else . end) as $msg | - ([$external_prefixes[] | . as $p | $uri | startswith($p)] | any) as $is_external | - (if ($uri != "?" and $line != "?" and ($is_external | not)) then "[\($uri):\($line)](https://github.com/\($repo)/blob/\($sha)/\($uri)#L\($line))" else "\($uri):\($line)" end) as $loc | - "| `\($rid)` | \($loc) | \($msg) |"' - done - } > codeql-findings.md - fi - # Generate medium findings detail file for informational display in the PR comment - if [[ "$MEDIUM_COUNT" -gt 0 ]]; then - MERGE_SHA=${MERGE_SHA:-$(git rev-parse HEAD)} - { - echo "| Rule | Location | Description |" - echo "|------|----------|-------------|" - for f in sarif-results/*.sarif sarif-results/*.sarif.gz; do - [[ -f "$f" ]] || continue - if [[ "$f" == *.gz ]]; then - FC=$(gunzip -c "$f") - else - FC=$(cat "$f") - fi - echo "$FC" | jq -r \ - --arg repo "$GITHUB_REPOSITORY" \ - --arg sha "$MERGE_SHA" \ - --argjson external_prefixes "$EXTERNAL_PREFIXES" \ - '.runs[] | - . as $run | - ( - [ (($run.tool.driver.rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}), - ((($run.tool.extensions // [])[].rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}) ] - | from_entries - ) as $secmap | - ($run.results // [])[] | - . as $result | - (($result.ruleId // $result.rule.id // "") | tostring) as $rid | - (($secmap[$rid] // "0") | tonumber) as $sev | - select($sev >= 6.0 and $sev < 7.0) | - (.locations[0].physicalLocation.artifactLocation.uri // "?") as $uri | - ((.locations[0].physicalLocation.region.startLine // "?") | tostring) as $line | - (.message.text // "no description" | gsub("\n"; " ") | gsub("://"; "\u200b://") | gsub("www\\."; "www\u200b.") | gsub("#(?=[0-9])"; "#\u200b") | gsub("\\[(?[^\\]]+)\\][(][0-9]+[)]"; .c) | gsub("\\["; "[") | gsub("\\]"; "]") | if length > 150 then .[0:150] + "\u2026" else . end) as $msg | - ([$external_prefixes[] | . as $p | $uri | startswith($p)] | any) as $is_external | - (if ($uri != "?" and $line != "?" and ($is_external | not)) then "[\($uri):\($line)](https://github.com/\($repo)/blob/\($sha)/\($uri)#L\($line))" else "\($uri):\($line)" end) as $loc | - "| `\($rid)` | \($loc) | \($msg) |"' - done - } > codeql-medium-findings.md - fi - # Generate low findings detail file for informational display in the PR comment (collapsed) - if [[ "$LOW_COUNT" -gt 0 ]]; then - MERGE_SHA=${MERGE_SHA:-$(git rev-parse HEAD)} - { - echo "| Rule | Location | Description |" - echo "|------|----------|-------------|" - for f in sarif-results/*.sarif sarif-results/*.sarif.gz; do - [[ -f "$f" ]] || continue - if [[ "$f" == *.gz ]]; then - FC=$(gunzip -c "$f") - else - FC=$(cat "$f") - fi - echo "$FC" | jq -r \ - --arg repo "$GITHUB_REPOSITORY" \ - --arg sha "$MERGE_SHA" \ - --argjson external_prefixes "$EXTERNAL_PREFIXES" \ - '.runs[] | - . as $run | - ( - [ (($run.tool.driver.rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}), - ((($run.tool.extensions // [])[].rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}) ] - | from_entries - ) as $secmap | - ($run.results // [])[] | - . as $result | - (($result.ruleId // $result.rule.id // "") | tostring) as $rid | - (($secmap[$rid] // "0") | tonumber) as $sev | - select($sev < 6.0) | - (.locations[0].physicalLocation.artifactLocation.uri // "?") as $uri | - ((.locations[0].physicalLocation.region.startLine // "?") | tostring) as $line | - (.message.text // "no description" | gsub("\n"; " ") | gsub("://"; "\u200b://") | gsub("www\\."; "www\u200b.") | gsub("#(?=[0-9])"; "#\u200b") | gsub("\\[(?[^\\]]+)\\][(][0-9]+[)]"; .c) | gsub("\\["; "[") | gsub("\\]"; "]") | if length > 150 then .[0:150] + "\u2026" else . end) as $msg | - ([$external_prefixes[] | . as $p | $uri | startswith($p)] | any) as $is_external | - (if ($uri != "?" and $line != "?" and ($is_external | not)) then "[\($uri):\($line)](https://github.com/\($repo)/blob/\($sha)/\($uri)#L\($line))" else "\($uri):\($line)" end) as $loc | - "| `\($rid)` | \($loc) | \($msg) |"' - done - } > codeql-low-findings.md - fi - ANALYZE_FAILED=false - if [[ "${{ steps.detect-langs.outputs.found }}" == 'true' && "${{ steps.analyze.outcome }}" != "success" && "${{ steps.analyze.outcome }}" != "" ]]; then - ANALYZE_FAILED=true - fi - - UNSCANNED_CSV="${{ steps.detect-langs.outputs.unscanned_langs }}" - CONFIG_ERROR_LANGS="" - - # CodeQL sets CODEQL_ACTION_JOB_STATUS=JOB_STATUS_CONFIGURATION_ERROR when it found - # no indexable source for a requested language (e.g. a committed file that's only - # vendored/minified/generated code, which CodeQL's own extractor refuses to treat as - # source). That's not a security finding or a broken analysis - don't block the PR on - # it, just note which language(s) went unscanned. Any other non-success outcome (a - # real extractor crash, OOM, etc.) still fails the job below. - if [[ "$ANALYZE_FAILED" == 'true' && "${CODEQL_ACTION_JOB_STATUS:-}" == 'JOB_STATUS_CONFIGURATION_ERROR' ]]; then - echo "::warning::CodeQL reported a configuration error (no indexable source found) for language(s): ${{ steps.detect-langs.outputs.languages }}. Treating as skipped instead of failing the PR." - CONFIG_ERROR_LANGS="codeql-config-error($(echo '${{ steps.detect-langs.outputs.languages }}' | tr ',' '+'))" - UNSCANNED_CSV="${UNSCANNED_CSV:+$UNSCANNED_CSV,}${CONFIG_ERROR_LANGS}" - ANALYZE_FAILED=false - fi - - if [[ "$RESULT_COUNT" -gt 0 || "$ANALYZE_FAILED" == 'true' ]]; then - echo "codeql_status=failure" >> "$GITHUB_OUTPUT" - elif [[ "${{ steps.detect-langs.outputs.found }}" == 'false' || -n "$CONFIG_ERROR_LANGS" ]]; then - echo "codeql_status=skipped" >> "$GITHUB_OUTPUT" - else - echo "codeql_status=success" >> "$GITHUB_OUTPUT" - fi - echo "codeql_unscanned_langs=$UNSCANNED_CSV" >> "$GITHUB_OUTPUT" - - - name: Upload findings detail for PR comment - if: always() && steps.status.outputs.codeql_status == 'failure' - uses: actions/upload-artifact@v7 - with: - name: codeql-findings - path: codeql-findings.md - if-no-files-found: ignore - - - name: Upload medium findings detail for PR comment - if: always() - uses: actions/upload-artifact@v7 - with: - name: codeql-medium-findings - path: codeql-medium-findings.md - if-no-files-found: ignore - - - name: Upload low findings detail for PR comment - if: always() - uses: actions/upload-artifact@v7 - with: - name: codeql-low-findings - path: codeql-low-findings.md - if-no-files-found: ignore - - - name: Fail job if CodeQL found high/error/critical issues - if: always() && steps.status.outputs.codeql_status == 'failure' - run: exit 1 + uses: ./.github/workflows/_codeql-scan.yml + with: + matrix: ${{ needs.detect-changes.outputs.matrix }} + pr_number: ${{ github.event.pull_request.number }} + base_ref: ${{ github.event.pull_request.base.ref }} # -------------------------------------------------------------------------- - # Job 6: ClamAV antivirus scan (runs after validate-plugin) + # Job 6: ClamAV antivirus scan (reusable workflow) # -------------------------------------------------------------------------- clamav-scan: needs: [detect-changes, validate-plugin, validate-title] if: needs.validate-plugin.result == 'success' && needs.detect-changes.outputs.skip_validation != 'true' && needs.validate-title.outputs.title_valid == 'true' - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - clamav_status: ${{ steps.status.outputs.clamav_status }} - clamav_infected: ${{ steps.status.outputs.clamav_infected }} - steps: - - name: Checkout PR merge commit for scan - uses: actions/checkout@v6 - with: - ref: refs/pull/${{ github.event.pull_request.number }}/merge - sparse-checkout: .gitignore - sparse-checkout-cone-mode: false - # Merge ref is only virus-scanned by ClamAV; fork code is not executed. - allow-unsafe-pr-checkout: true - - - name: Limit checkout to changed plugin folders only - run: | - echo '${{ needs.detect-changes.outputs.matrix }}' \ - | jq -r '.[] | "plugins/\(.)"' \ - | git sparse-checkout set --stdin - git checkout - - - name: Get cache keys - id: cache-keys - run: | - echo "week=$(date +%Y-W%V)" >> $GITHUB_OUTPUT - echo "date=$(date +%Y%m%d)" >> $GITHUB_OUTPUT - - - name: Cache ClamAV installation (weekly) - id: cache-clamav-install - uses: actions/cache@v5 - with: - path: /tmp/clamav-apt - key: clamav-install-${{ runner.os }}-${{ steps.cache-keys.outputs.week }} - restore-keys: clamav-install-${{ runner.os }}- - - - name: Cache ClamAV virus definitions (daily) - id: cache-clamav-defs - uses: actions/cache@v5 - with: - path: /tmp/clamav-db - key: clamav-defs-${{ runner.os }}-${{ steps.cache-keys.outputs.date }} - restore-keys: clamav-defs-${{ runner.os }}- - - - name: Install ClamAV - run: | - sudo apt-get update -qq - if [[ "${{ steps.cache-clamav-install.outputs.cache-hit }}" == 'true' ]]; then - echo "Installing ClamAV from cached packages..." - sudo cp /tmp/clamav-apt/*.deb /var/cache/apt/archives/ 2>/dev/null || true - fi - sudo apt-get install -y --no-install-recommends clamav - if [[ "${{ steps.cache-clamav-install.outputs.cache-hit }}" != 'true' ]]; then - echo "Saving ClamAV packages to cache..." - mkdir -p /tmp/clamav-apt - find /var/cache/apt/archives/ \( -name 'clamav*.deb' -o -name 'libclamav*.deb' \) \ - -exec cp {} /tmp/clamav-apt/ \; 2>/dev/null || true - fi - - - name: Update virus definitions - run: | - sudo systemctl stop clamav-freshclam 2>/dev/null || true - sudo mkdir -p /tmp/clamav-db - sudo chown -R clamav:clamav /tmp/clamav-db - if [[ "${{ steps.cache-clamav-defs.outputs.cache-hit }}" != 'true' ]]; then - echo "Downloading fresh ClamAV definitions..." - sudo freshclam --datadir=/tmp/clamav-db - else - echo "Using cached ClamAV definitions" - fi - - - name: Scan changed plugin directories - id: scan - run: | - SCAN_TARGETS=() - EXT_SCAN_DIR="/tmp/external-scan" - mkdir -p "$EXT_SCAN_DIR" - - while IFS= read -r plugin_name; do - plugin_json="plugins/$plugin_name/plugin.json" - [[ ! -f "$plugin_json" ]] && continue - - source_type=$(jq -r '.source_type // "local"' "$plugin_json") - - if [[ "$source_type" == "external" ]]; then - version=$(jq -r '.version' "$plugin_json") - source_url_template=$(jq -r '.source_url // ""' "$plugin_json") - source_url="${source_url_template//\{version\}/$version}" - - echo "Downloading external ZIP for ClamAV scan: $source_url" - scan_dir="$EXT_SCAN_DIR/$plugin_name" - mkdir -p "$scan_dir/extracted" - zip_file="$scan_dir/${plugin_name}.zip" - - if curl -fsSL "$source_url" -o "$zip_file" --max-time 60; then - unzip -q "$zip_file" -d "$scan_dir/extracted" || true - SCAN_TARGETS+=("$scan_dir/extracted") - else - echo "::warning::Could not download $source_url — scanning plugin.json only for $plugin_name" - SCAN_TARGETS+=("plugins/$plugin_name") - fi - else - SCAN_TARGETS+=("plugins/$plugin_name") - fi - done < <(echo '${{ needs.detect-changes.outputs.matrix }}' | jq -r '.[]') - - echo "Scanning: ${SCAN_TARGETS[*]}" - set +e - clamscan --database=/tmp/clamav-db \ - --recursive --infected --no-summary \ - "${SCAN_TARGETS[@]}" > clamav-output.txt 2>&1 - echo "exit_code=$?" >> $GITHUB_OUTPUT - set -e - cat clamav-output.txt - - - name: Set ClamAV status outputs - id: status - if: always() - env: - SCAN_EXIT: ${{ steps.scan.outputs.exit_code }} - run: | - INFECTED=0 - if [[ -f "clamav-output.txt" ]]; then - INFECTED=$(grep -c ' FOUND$' clamav-output.txt || true) - fi - echo "clamav_infected=$INFECTED" >> "$GITHUB_OUTPUT" - - if [[ "$INFECTED" -gt 0 ]]; then - { - echo "| File | Signature |" - echo "|------|-----------|" - grep ' FOUND$' clamav-output.txt | while IFS= read -r line; do - FULL_PATH=$(echo "$line" | sed 's/: .* FOUND$//') - FILE=$(echo "$FULL_PATH" | sed "s|^${GITHUB_WORKSPACE}/||") - SIG=$(echo "$line" | sed 's/^[^:]*: //; s/ FOUND$//') - HASH=$(sha256sum "$FULL_PATH" 2>/dev/null | cut -d' ' -f1 || true) - if [[ -n "$HASH" ]]; then - echo "| \`$FILE\` | [\`$SIG\`](https://www.virustotal.com/gui/file/${HASH}) |" - else - echo "| \`$FILE\` | \`$SIG\` |" - fi - done - } > clamav-findings.md - echo "clamav_status=failure" >> "$GITHUB_OUTPUT" - elif [[ "${SCAN_EXIT:-0}" -ge 2 ]]; then - echo "clamav_status=failure" >> "$GITHUB_OUTPUT" - else - echo "clamav_status=success" >> "$GITHUB_OUTPUT" - fi - - - name: Upload ClamAV findings for PR comment - if: always() && steps.status.outputs.clamav_status == 'failure' - uses: actions/upload-artifact@v7 - with: - name: clamav-findings - path: clamav-findings.md - if-no-files-found: ignore - - - name: Load app ID from config - if: always() && steps.status.outputs.clamav_status == 'failure' - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: always() && steps.status.outputs.clamav_status == 'failure' && steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 - with: - client-id: ${{ steps.config.outputs.app_id }} - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - - name: Apply quarantine label - if: always() && steps.status.outputs.clamav_status == 'failure' - env: - GH_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} - run: | - gh label create "QUARANTINE" \ - --color "FFFF00" \ - --description "ClamAV detected a potential threat in this PR" \ - --repo "$GITHUB_REPOSITORY" 2>/dev/null || true - gh pr edit "${{ github.event.pull_request.number }}" \ - --add-label "QUARANTINE" \ - --repo "$GITHUB_REPOSITORY" - - - name: Fail job if ClamAV detected threats - if: always() && steps.status.outputs.clamav_status == 'failure' - run: exit 1 + uses: ./.github/workflows/_clamav-scan.yml + with: + matrix: ${{ needs.detect-changes.outputs.matrix }} + pr_number: ${{ github.event.pull_request.number }} + base_ref: ${{ github.event.pull_request.base.ref }} # -------------------------------------------------------------------------- # Job 7: Validate each plugin in parallel via matrix @@ -1030,79 +216,33 @@ jobs: matrix: plugin: ${{ fromJson(needs.detect-changes.outputs.matrix) }} steps: - - name: Checkout base branch scripts (trusted) - uses: actions/checkout@v6 + - name: Trusted checkout + uses: ./.github/actions/trusted-checkout with: + base-ref: ${{ github.event.pull_request.base.ref }} + head-repo: ${{ github.event.pull_request.head.repo.full_name }} + head-sha: ${{ github.event.pull_request.head.sha }} repository: ${{ github.repository }} - ref: ${{ github.event.pull_request.base.ref }} - fetch-depth: 0 - sparse-checkout: .github/scripts - sparse-checkout-cone-mode: false - - - name: Fetch base branch - run: git fetch origin ${{ github.event.pull_request.base.ref }} - - - name: Save trusted scripts before fork checkout - run: cp -r .github/scripts /tmp/trusted-scripts - - - name: Checkout PR plugins (untrusted content only) - uses: actions/checkout@v6 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - sparse-checkout: plugins - sparse-checkout-cone-mode: false - clean: false - # Fork content is only read as data (sparse plugins/), never executed; - # trusted scripts are saved/restored around this step. - allow-unsafe-pr-checkout: true - - - name: Restore trusted scripts - run: mkdir -p .github && cp -r /tmp/trusted-scripts .github/scripts - - - name: Re-fetch base branch refs - run: git fetch https://github.com/${{ github.repository }} +${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }} - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - name: Validate ${{ matrix.plugin }} id: validate env: - GH_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} + GH_TOKEN: ${{ steps.token.outputs.token }} GITHUB_REPOSITORY: ${{ github.repository }} run: | - chmod +x .github/scripts/validate/*.sh set +e - .github/scripts/validate/validate.sh \ - "${{ matrix.plugin }}" \ - "${{ github.event.pull_request.user.login }}" \ - "${{ github.event.pull_request.base.ref }}" \ - "${{ matrix.plugin }}.fragment.md" + python -m pluginctl validate \ + --plugin "${{ matrix.plugin }}" \ + --author "${{ github.event.pull_request.user.login }}" \ + --base-ref "${{ github.event.pull_request.base.ref }}" \ + --out "${{ matrix.plugin }}.fragment.md" echo "exit_code=$?" >> $GITHUB_OUTPUT continue-on-error: true @@ -1126,40 +266,21 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: Checkout scripts + - name: Checkout config uses: actions/checkout@v6 with: ref: ${{ github.event.pull_request.base.ref }} fetch-depth: 1 - sparse-checkout: .github/scripts + sparse-checkout: .github sparse-checkout-cone-mode: false - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - name: Download all report fragments uses: actions/download-artifact@v8 with: @@ -1199,7 +320,7 @@ jobs: - name: Aggregate and post comment id: report env: - GH_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} + GH_TOKEN: ${{ steps.token.outputs.token }} GITHUB_REPOSITORY: ${{ github.repository }} DISCORD_URL: ${{ vars.DISCORD_URL }} CODEQL_RESULT: ${{ needs.codeql-analyze.outputs.codeql_status }} @@ -1217,15 +338,16 @@ jobs: TITLE_FEEDBACK: ${{ needs.validate-title.outputs.title_feedback }} TITLE_SUGGESTION: ${{ needs.validate-title.outputs.title_suggestion }} BASE_REF: ${{ github.event.pull_request.base.ref }} + WEBHOOK_URL: ${{ vars.WEBHOOK_URL }} + WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} run: | - chmod +x .github/scripts/validate/*.sh set +e - .github/scripts/validate/report.sh \ - "${{ github.event.pull_request.number }}" \ - "${{ github.event.pull_request.user.login }}" \ - "${{ needs.detect-changes.outputs.plugin_count }}" \ - "false" \ - "fragments" + python -m pluginctl report \ + --pr "${{ github.event.pull_request.number }}" \ + --author "${{ github.event.pull_request.user.login }}" \ + --plugin-count "${{ needs.detect-changes.outputs.plugin_count }}" \ + --close-pr false \ + --fragments-dir fragments echo "exit_code=$?" >> $GITHUB_OUTPUT - name: Fail workflow if any plugin failed @@ -1243,60 +365,39 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: Checkout scripts + - name: Checkout config uses: actions/checkout@v6 with: ref: ${{ github.event.pull_request.base.ref }} fetch-depth: 1 - sparse-checkout: .github/scripts + sparse-checkout: .github sparse-checkout-cone-mode: false - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - name: Post close comment and close PR env: - GH_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} + GH_TOKEN: ${{ steps.token.outputs.token }} GITHUB_REPOSITORY: ${{ github.repository }} DISCORD_URL: ${{ vars.DISCORD_URL }} CLOSE_REASON: ${{ needs.detect-changes.outputs.close_reason }} + WEBHOOK_URL: ${{ vars.WEBHOOK_URL }} + WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} run: | - chmod +x .github/scripts/validate/*.sh - .github/scripts/validate/report.sh \ - "${{ github.event.pull_request.number }}" \ - "${{ github.event.pull_request.user.login }}" \ - "${{ needs.detect-changes.outputs.plugin_count }}" \ - "true" \ - "/dev/null" + python -m pluginctl report \ + --pr "${{ github.event.pull_request.number }}" \ + --author "${{ github.event.pull_request.user.login }}" \ + --plugin-count "${{ needs.detect-changes.outputs.plugin_count }}" \ + --close-pr true \ + --fragments-dir /dev/null # -------------------------------------------------------------------------- - # Job 10: Gate - single fixed status check for branch protection rules - # Reference this job by name "Plugin PR Check" in your branch protection. - # Passes only when all jobs succeed. Fails for any error, including - # unauthorized PRs. + # Job 10: Gate, single fixed status check for branch protection rules # -------------------------------------------------------------------------- plugin-pr-check: name: Plugin PR Check @@ -1305,6 +406,14 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 2 steps: + - name: Checkout config + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.base.ref }} + fetch-depth: 1 + sparse-checkout: .github/pluginctl + sparse-checkout-cone-mode: false + - name: Check for quarantine label env: GH_TOKEN: ${{ github.token }} @@ -1330,41 +439,4 @@ jobs: CLAMAV_STATUS: ${{ needs.clamav-scan.outputs.clamav_status }} VALIDATE_RESULT: ${{ needs.validate-plugin.result }} REPORT_RESULT: ${{ needs.report.result }} - run: | - if [[ "$DETECT_RESULT" != "success" ]]; then - echo "::error::Plugin detection failed or no plugin changes found." - exit 1 - fi - if [[ "$SKIP_VALIDATION" == "true" ]]; then - echo "No plugin changes detected and author has write access - passing." - exit 0 - fi - if [[ "$TITLE_RESULT" == "failure" ]]; then - echo "::error::PR title does not match the required format. Rename the PR and re-run." - exit 1 - fi - if [[ "$OUTSIDE_VIOLATION" == "true" ]]; then - echo "::error::PR contains unauthorized changes outside the plugins/ directory." - exit 1 - fi - if [[ "$CLOSE_PR" == "true" ]]; then - echo "::error::PR is unauthorized - no permission to modify these plugins." - exit 1 - fi - if [[ "$CODEQL_RESULT" == "failure" || "$CODEQL_STATUS" == "failure" ]]; then - echo "::error::CodeQL security analysis failed. See the Security tab for details." - exit 1 - fi - if [[ "$CLAMAV_RESULT" == "failure" || "$CLAMAV_STATUS" == "failure" ]]; then - echo "::error::ClamAV antivirus scan detected threats. See PR comment for details." - exit 1 - fi - if [[ "$VALIDATE_RESULT" == "failure" || "$VALIDATE_RESULT" == "cancelled" ]]; then - echo "::error::One or more plugin validations failed. See PR comment for details." - exit 1 - fi - if [[ "$REPORT_RESULT" != "success" ]]; then - echo "::error::Plugin validation failed. See PR comment for details." - exit 1 - fi - echo "All plugins validated successfully." + run: python -m pluginctl gate diff --git a/.github/workflows/yank-plugin-version.yml b/.github/workflows/yank-plugin-version.yml index bac13b2..954313a 100644 --- a/.github/workflows/yank-plugin-version.yml +++ b/.github/workflows/yank-plugin-version.yml @@ -7,6 +7,7 @@ permissions: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + PYTHONPATH: .github/pluginctl/src on: workflow_dispatch: @@ -34,39 +35,17 @@ jobs: with: fetch-depth: 0 - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - - name: Make scripts executable - run: chmod +x .github/scripts/publish/*.sh - - name: Yank plugin version env: - GITHUB_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} - APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + GITHUB_TOKEN: ${{ steps.token.outputs.token }} + APP_SLUG: ${{ steps.token.outputs.app-slug }} GITHUB_REPOSITORY: ${{ github.repository }} GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} @@ -74,9 +53,11 @@ jobs: YANK_VERSION: ${{ inputs.version }} YANK_ISSUE: ${{ inputs.issue_number }} SOURCE_BRANCH: ${{ github.ref_name }} + WEBHOOK_URL: ${{ vars.WEBHOOK_URL }} + WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} run: | set -o pipefail - .github/scripts/publish/yank-version.sh 2>&1 | tee yank.log + python -m pluginctl yank 2>&1 | tee yank.log - name: Generate job summary if: always() diff --git a/docs/webhooks.md b/docs/webhooks.md new file mode 100644 index 0000000..939e7a0 --- /dev/null +++ b/docs/webhooks.md @@ -0,0 +1,82 @@ +# Plugin registry webhooks + +`pluginctl` can emit a signed JSON event after key lifecycle points so an +external consumer (for example a Discord bot) can react to registry activity. + +Emission is **opt-in and best-effort**: + +- Configure two repository settings: `secrets.WEBHOOK_SECRET` and + `vars.WEBHOOK_URL`. +- If either is unset, emission is a silent no-op. +- A delivery failure never fails the pipeline, it logs a `::warning::` and the + workflow continues. + +## Transport + +Each event is a single `POST` to `WEBHOOK_URL` with a compact JSON body and +these headers: + +| Header | Value | +|---|---| +| `Content-Type` | `application/json` | +| `X-PluginCtl-Event` | the event name (e.g. `plugin.published`) | +| `X-PluginCtl-Delivery` | a per-delivery UUIDv4 | +| `X-PluginCtl-Signature` | `sha256=` HMAC-SHA256 of the **raw request body** using `WEBHOOK_SECRET` | + +Verify the signature over the exact bytes received, before parsing JSON. + +## Envelope + +```json +{ + "event": "plugin.published", + "delivered_at": "2026-07-21T12:00:00Z", + "repository": "Dispatcharr/Plugins", + "actor": "octocat", + "data": { } +} +``` + +## Events + +| Event | `data` fields | +|---|---| +| `pr.validated` | `pr`, `author`, `result` (`pass`\|`fail`), `plugins[]`, `checks` (`{codeql,clamav,title}`) | +| `pr.closed_unauthorized` | `pr`, `author`, `reason` | +| `pr.quarantined` | `pr`, `author`, `infected` | +| `pr.auto_merged` | `pr`, `plugins[]` | +| `plugin.published` | `plugin`, `version`, `pr`, `actor` (one per changed plugin) | +| `plugin.yanked` | `plugin`, `version`, `issue`, `rollback_pr` | + +## Reference verifier + +The signature is HMAC-SHA256 of the raw body. Any language works; here is the +Python equivalent of what the consumer must do: + +```python +import hashlib +import hmac + +def verify(secret: str, raw_body: bytes, signature_header: str) -> bool: + """signature_header is the value of X-PluginCtl-Signature, e.g. 'sha256=abcd...'.""" + if not signature_header.startswith("sha256="): + return False + expected = hmac.new(secret.encode("utf-8"), raw_body, hashlib.sha256).hexdigest() + return hmac.compare_digest(expected, signature_header[len("sha256="):]) +``` + +Node.js: + +```js +const crypto = require("crypto"); + +function verify(secret, rawBody, signatureHeader) { + if (!signatureHeader.startsWith("sha256=")) return false; + const expected = crypto.createHmac("sha256", secret).update(rawBody).digest("hex"); + const got = signatureHeader.slice("sha256=".length); + return crypto.timingSafeEqual(Buffer.from(expected), Buffer.from(got)); +} +``` + +Always compare with a constant-time function (`hmac.compare_digest` / +`crypto.timingSafeEqual`). From f0a0e8b1c966347f9ac76a1f9f2faa5b9798dc08 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Wed, 22 Jul 2026 11:28:32 -0400 Subject: [PATCH 2/6] Reimplement manifest base-URL split, icons, and immutable-tag retry (upstream #127) Ports the four behaviors from Dispatcharr/Plugins#127 into the pluginctl publish path. - Split the root manifest's root_url into download_base_url (GitHub Releases, for ZIP paths) and metadata_base_url (GitHub Pages when configured, else raw.githubusercontent.com). Each plugin's manifest_url is now a relative path clients compose with metadata_base_url. - Copy each plugin's source logo (png, svg, jpg, or webp, first match) into metadata// on every publish. - Retry gh release create with a numeric tag suffix on immutable-tag conflicts, skip cleanly when the release already exists, and discard the fresh per-version metadata on a skip so generate-manifest keeps the published asset's checksums. - When a tag carries a retry suffix, keep it in the release directory but use the canonical version for the ZIP asset filename, for versioned urls and latest_url. readmes.py reads download_base_url instead of root_url. 8 new tests. --- .github/pluginctl/src/pluginctl/core/gh.py | 13 +++ .../src/pluginctl/publish/manifest.py | 45 ++++++++-- .../src/pluginctl/publish/readmes.py | 18 ++-- .../pluginctl/src/pluginctl/publish/zips.py | 52 ++++++++++- .github/pluginctl/tests/test_manifest.py | 18 ++-- .github/pluginctl/tests/test_publish_pr127.py | 87 +++++++++++++++++++ .github/pluginctl/tests/test_readmes.py | 2 +- 7 files changed, 210 insertions(+), 25 deletions(-) create mode 100644 .github/pluginctl/tests/test_publish_pr127.py diff --git a/.github/pluginctl/src/pluginctl/core/gh.py b/.github/pluginctl/src/pluginctl/core/gh.py index a3d35c2..8c0d59c 100644 --- a/.github/pluginctl/src/pluginctl/core/gh.py +++ b/.github/pluginctl/src/pluginctl/core/gh.py @@ -132,6 +132,19 @@ def release_create(tag: str, repo: str, title: str, notes: str, asset: str) -> i "--notes", notes, asset]).returncode +def release_create_capture(tag: str, repo: str, title: str, notes: str, + asset: str) -> tuple[int, str]: + """Like release_create but returns (returncode, stderr) for conflict handling.""" + proc = _run(["release", "create", tag, "--repo", repo, "--title", title, + "--notes", notes, asset]) + return proc.returncode, proc.stderr + + +def pages_url(repo: str) -> str: + """GitHub Pages site URL for the repo, or "" when Pages is not configured.""" + return api(f"repos/{repo}/pages", jq=".html_url // empty") or "" + + def release_delete(tag: str, repo: str, cleanup_tag: bool = True) -> int: args = ["release", "delete", tag, "--repo", repo, "--yes"] if cleanup_tag: diff --git a/.github/pluginctl/src/pluginctl/publish/manifest.py b/.github/pluginctl/src/pluginctl/publish/manifest.py index 5a7e8a1..941e354 100644 --- a/.github/pluginctl/src/pluginctl/publish/manifest.py +++ b/.github/pluginctl/src/pluginctl/publish/manifest.py @@ -136,12 +136,21 @@ def build_root_entry(plugin_raw: dict, plugin_name: str, latest_metadata: dict, }) -def build_root_manifest(registry_url: str, registry_name: str, root_url: str, +def build_root_manifest(registry_url: str, registry_name: str, + download_base_url: str, metadata_base_url: str, root_entries: list[dict]) -> dict: + """Root manifest payload. + + ``download_base_url`` is the GitHub Releases base for ZIP paths; + ``metadata_base_url`` is the base for manifest paths (GitHub Pages when + configured, else raw.githubusercontent.com). Clients compose each plugin's + relative ``url`` / ``manifest_url`` with the matching base. + """ return { "registry_url": registry_url, "registry_name": registry_name, - "root_url": root_url, + "download_base_url": download_base_url, + "metadata_base_url": metadata_base_url, "plugins": root_entries, } @@ -253,6 +262,19 @@ def _canonical(version: str) -> str: return re.sub(r"-[0-9]+$", "", version) +ICON_EXTS = ("png", "svg", "jpg", "webp") + + +def _sync_icon(plugin_dir: str, plugin_name: str) -> None: + """Copy the source plugin icon (logo.png|svg|jpg|webp, first match) to metadata//.""" + import shutil + for ext in ICON_EXTS: + src = os.path.join(plugin_dir, f"logo.{ext}") + if os.path.isfile(src): + shutil.copy(src, f"metadata/{plugin_name}/logo.{ext}") + return + + def strip_signatures() -> None: """Remove ``.signature`` from every manifest (no key configured or signing failed).""" import glob as _glob @@ -280,8 +302,11 @@ def generate(source_branch: str, releases_branch: str, repository: str, generated_at = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") registry_url = f"https://github.com/{repository}" registry_name = repository - root_url = f"https://github.com/{repository}/releases/download" + download_base_url = f"https://github.com/{repository}/releases/download" raw_releases_url = f"https://raw.githubusercontent.com/{repository}/{releases_branch}" + # Metadata paths resolve against GitHub Pages when configured, else raw. + pages = gh.pages_url(repository).rstrip("/") + metadata_base_url = pages if pages else raw_releases_url key_id, signing_failed = import_gpg_key(os.environ.get("GPG_PRIVATE_KEY", "")) passphrase = os.environ.get("GPG_PASSPHRASE") or None @@ -318,6 +343,7 @@ def sign_if_needed(path: str, written: bool) -> None: existing_manifest_file = f"metadata/{plugin_name}/manifest.json" os.makedirs(f"metadata/{plugin_name}", exist_ok=True) + _sync_icon(plugin_dir, plugin_name) existing_manifest = None if os.path.isfile(existing_manifest_file): try: @@ -340,8 +366,10 @@ def sign_if_needed(path: str, written: bool) -> None: for release_tag in versioned_tags: zip_version = release_tag[len(prefix):] - zip_url = f"{plugin_name}-{zip_version}/{plugin_name}-{zip_version}.zip" canonical_version = _canonical(zip_version) + # Release dir uses the full tag version (may carry a retry suffix); + # the ZIP asset filename uses the canonical version (how it was built). + zip_url = f"{plugin_name}-{zip_version}/{plugin_name}-{canonical_version}.zip" metadata: dict = {} fresh = os.path.join(build_meta_dir, plugin_key, f"{plugin_key}-{canonical_version}.json") @@ -368,7 +396,8 @@ def sign_if_needed(path: str, written: bool) -> None: latest_url = "" if latest_zip_version: - latest_url = f"{plugin_name}-{latest_zip_version}/{plugin_name}-{latest_zip_version}.zip" + latest_canonical = _canonical(latest_zip_version) + latest_url = f"{plugin_name}-{latest_zip_version}/{plugin_name}-{latest_canonical}.zip" versioned_zips = override_current_min_max(versioned_zips, current_version, min_da, max_da) @@ -381,13 +410,15 @@ def sign_if_needed(path: str, written: bool) -> None: if unlisted: continue - manifest_url = f"{raw_releases_url}/metadata/{plugin_name}/manifest.json" + # Relative: clients compose it with metadata_base_url from the root manifest. + manifest_url = f"metadata/{plugin_name}/manifest.json" root_entries.append(build_root_entry(raw, plugin_name, latest_metadata, latest_size_kb, min_da, max_da, latest_url, manifest_url)) plugin_count += 1 - inner_root = build_root_manifest(registry_url, registry_name, root_url, root_entries) + inner_root = build_root_manifest(registry_url, registry_name, + download_base_url, metadata_base_url, root_entries) written = write_manifest_if_changed("manifest.json", inner_root, generated_at) sign_if_needed("manifest.json", written) diff --git a/.github/pluginctl/src/pluginctl/publish/readmes.py b/.github/pluginctl/src/pluginctl/publish/readmes.py index 9885723..c55e7a7 100644 --- a/.github/pluginctl/src/pluginctl/publish/readmes.py +++ b/.github/pluginctl/src/pluginctl/publish/readmes.py @@ -59,7 +59,7 @@ def _g(data: Optional[dict], key: str, default: str = "") -> str: # Per-plugin README (metadata//README.md) # --------------------------------------------------------------------------- # def render_plugin_readme(plugin_name: str, plugin_raw: dict, manifest: dict, - root_url: str, repository: str, source_branch: str, + download_base_url: str, repository: str, source_branch: str, last_updated: str, has_readme: bool, source_readme: str = "") -> str: name = _g(plugin_raw, "name") @@ -76,7 +76,7 @@ def render_plugin_readme(plugin_name: str, plugin_raw: dict, manifest: dict, manifest_body = manifest.get("manifest", {}) if manifest else {} latest = manifest_body.get("latest") or {} latest_url_path = latest.get("latest_url") or "" - latest_full_url = f"{root_url}/{latest_url_path}" if root_url and latest_url_path else "" + latest_full_url = f"{download_base_url}/{latest_url_path}" if download_base_url and latest_url_path else "" L: list[str] = [] L.append("[Back to All Plugins](../../README.md)") @@ -132,7 +132,7 @@ def render_plugin_readme(plugin_name: str, plugin_raw: dict, manifest: dict, if not v: continue url_path = ver.get("url") or "" - full_url = f"{root_url}/{url_path}" if root_url and url_path else "" + full_url = f"{download_base_url}/{url_path}" if download_base_url and url_path else "" commit_sha = ver.get("commit_sha") or "" commit_short = ver.get("commit_sha_short") or "" build_ts = ver.get("build_timestamp") or "" @@ -234,7 +234,7 @@ def table_row(plugin_raw: dict, deprecated_pass: bool) -> str: def render_plugin_block(*, is_deprecated: bool, plugin_name: str, plugin_raw: dict, manifest: Optional[dict], last_updated: str, commit_sha: str, commit_sha_short: str, version_count, repository: str, - source_branch: str, releases_branch: str, root_url: str, + source_branch: str, releases_branch: str, download_base_url: str, has_source_readme: bool) -> str: name = _g(plugin_raw, "name") version = _g(plugin_raw, "version") @@ -250,7 +250,7 @@ def render_plugin_block(*, is_deprecated: bool, plugin_name: str, plugin_raw: di latest_url_path = "" if manifest: latest_url_path = ((manifest.get("manifest") or {}).get("latest") or {}).get("latest_url") or "" - zip_url = f"{root_url}/{latest_url_path}" if root_url and latest_url_path else "" + zip_url = f"{download_base_url}/{latest_url_path}" if download_base_url and latest_url_path else "" source_url = f"https://github.com/{repository}/tree/{source_branch}/plugins/{plugin_name}" readme_url = f"https://github.com/{repository}/blob/{source_branch}/plugins/{plugin_name}/README.md" @@ -308,7 +308,7 @@ def _plugin_last_updated(source_branch: str, plugin_dir: str) -> str: def generate_plugin_readmes(source_branch: str, repository: str) -> None: """Write metadata//README.md for every plugin (plugin-readmes.sh).""" root = _load_json("manifest.json") or {} - root_url = (root.get("manifest") or {}).get("root_url") or "" + download_base_url = (root.get("manifest") or {}).get("download_base_url") or "" for plugin_dir in sorted(glob.glob("plugins/*/")): plugin_name = os.path.basename(plugin_dir.rstrip("/")) plugin_file = os.path.join(plugin_dir, "plugin.json") @@ -324,7 +324,7 @@ def generate_plugin_readmes(source_branch: str, repository: str) -> None: has_readme = os.path.isfile(source_readme_path) source_readme = _read(source_readme_path) if has_readme else "" content = render_plugin_readme( - plugin_name, plugin_raw, manifest, root_url, repository, source_branch, + plugin_name, plugin_raw, manifest, download_base_url, repository, source_branch, last_updated, has_readme, source_readme) with open(f"metadata/{plugin_name}/README.md", "w", encoding="utf-8") as fh: fh.write(content) @@ -333,7 +333,7 @@ def generate_plugin_readmes(source_branch: str, repository: str) -> None: def generate_releases_readme(source_branch: str, releases_branch: str, repository: str) -> None: """Write the root README.md (releases-readme.sh).""" root = _load_json("manifest.json") or {} - root_url = (root.get("manifest") or {}).get("root_url") or "" + download_base_url = (root.get("manifest") or {}).get("download_base_url") or "" plugin_dirs = sorted(glob.glob("plugins/*/")) raws: dict[str, dict] = {} @@ -390,7 +390,7 @@ def emit_block(pname: str, raw: dict, is_deprecated: bool) -> None: manifest=manifest, last_updated=last_updated, commit_sha=commit_sha, commit_sha_short=commit_short, version_count=version_count, repository=repository, source_branch=source_branch, - releases_branch=releases_branch, root_url=root_url, + releases_branch=releases_branch, download_base_url=download_base_url, has_source_readme=has_source_readme)) # Active detailed sections diff --git a/.github/pluginctl/src/pluginctl/publish/zips.py b/.github/pluginctl/src/pluginctl/publish/zips.py index 584ba73..6e49308 100644 --- a/.github/pluginctl/src/pluginctl/publish/zips.py +++ b/.github/pluginctl/src/pluginctl/publish/zips.py @@ -114,13 +114,24 @@ def run(source_branch: str, repository: str, build_meta_dir: str) -> int: "size_kb": zip_size_kb, }) os.makedirs(os.path.join(build_meta_dir, plugin_key), exist_ok=True) - with open(os.path.join(build_meta_dir, plugin_key, f"{plugin_key}-{version}.json"), - "w", encoding="utf-8") as fh: + meta_file = os.path.join(build_meta_dir, plugin_key, f"{plugin_key}-{version}.json") + with open(meta_file, "w", encoding="utf-8") as fh: fh.write(jsonio.dumps(meta)) notes = _release_notes(repository, plugin_name, commit_sha, commit_sha_short) actions.log(f" {plugin_name} v{version} - uploading to GitHub Releases") - gh.release_create(release_tag, repository, f"{plugin_name} v{version}", notes, zip_path) + result = _upload_with_retry(release_tag, repository, + f"{plugin_name} v{version}", notes, zip_path) + if result is None: # fatal error already reported + os.remove(zip_path) + return 1 + _final_tag, upload_skipped = result + if upload_skipped: + # The release already existed, so the asset on it (not the one we just + # rebuilt) is authoritative. Drop the fresh metadata so generate-manifest + # falls back to the existing manifest's checksums. + if os.path.exists(meta_file): + os.remove(meta_file) os.remove(zip_path) with open("changed_plugins.txt", "w", encoding="utf-8") as fh: @@ -130,6 +141,41 @@ def run(source_branch: str, repository: str, build_meta_dir: str) -> int: return 0 +def classify_release_error(stderr: str) -> str: + """Classify a failed `gh release create`: 'exists', 'immutable', or 'fatal'.""" + if "already exists" in stderr: + return "exists" + if "immutable" in stderr or "Cannot create ref" in stderr: + return "immutable" + return "fatal" + + +def _upload_with_retry(release_tag: str, repository: str, title: str, notes: str, + zip_path: str): + """Create the release, retrying with a numeric suffix on immutable-tag conflicts. + + Returns (final_tag, upload_skipped) on success/skip, or None on a fatal error + (already logged). ``upload_skipped`` is True when the release already existed. + """ + final_tag = release_tag + suffix = 0 + while True: + rc, err = gh.release_create_capture(final_tag, repository, title, notes, zip_path) + if rc == 0: + return final_tag, False + kind = classify_release_error(err) + if kind == "exists": + actions.log(f" {release_tag} - release already exists, skipping upload") + return final_tag, True + if kind == "immutable": + suffix += 1 + final_tag = f"{release_tag}-{suffix}" + actions.log(f" Tag conflict on {release_tag}, retrying as {final_tag}") + continue + actions.eprint(err) + return None + + def _release_notes(repository: str, plugin_name: str, commit_sha: str, commit_sha_short: str) -> str: readme_url = f"https://github.com/{repository}/blob/releases/metadata/{plugin_name}/README.md" diff --git a/.github/pluginctl/tests/test_manifest.py b/.github/pluginctl/tests/test_manifest.py index f26474f..3b7800a 100644 --- a/.github/pluginctl/tests/test_manifest.py +++ b/.github/pluginctl/tests/test_manifest.py @@ -88,16 +88,24 @@ def test_trim_description(): assert len(long) == 200 and long.endswith("...") -def test_root_manifest_compact_matches_jq(tmp_path): +def test_root_manifest_split_base_urls_key_order(tmp_path): import shutil import subprocess entry = m.build_root_entry(PLUGIN_RAW, "demo", META, 10, "", "", - "u", "https://raw/m.json") + "u", "metadata/demo/manifest.json") root = m.build_root_manifest("https://github.com/org/repo", "org/repo", - "https://dl", [entry]) + "https://github.com/org/repo/releases/download", + "https://org.github.io/repo", [entry]) + # root_url split into download_base_url + metadata_base_url, in this order + assert list(root.keys()) == [ + "registry_url", "registry_name", "download_base_url", + "metadata_base_url", "plugins", + ] + assert "root_url" not in root compact = jsonio.dumps(root) if shutil.which("jq"): jq_out = subprocess.run(["jq", "-c", "."], input=compact, capture_output=True, text=True, check=True).stdout.strip() - assert compact == jq_out # stable round-trip through real jq - assert json.loads(compact)["plugins"][0]["slug"] == "demo" + assert compact == jq_out + # manifest_url in each entry is now a relative path + assert json.loads(compact)["plugins"][0]["manifest_url"] == "metadata/demo/manifest.json" diff --git a/.github/pluginctl/tests/test_publish_pr127.py b/.github/pluginctl/tests/test_publish_pr127.py new file mode 100644 index 0000000..c0eb0d9 --- /dev/null +++ b/.github/pluginctl/tests/test_publish_pr127.py @@ -0,0 +1,87 @@ +"""Behavior added by upstream PR #127, reimplemented in the pluginctl publish path.""" + +import json + +from pluginctl.publish import manifest as m +from pluginctl.publish import zips + + +# ---- immutable-tag release retry ---- +def test_classify_release_error(): + assert zips.classify_release_error("a release with the same tag already exists") == "exists" + assert zips.classify_release_error("tag is immutable and cannot be moved") == "immutable" + assert zips.classify_release_error("Cannot create ref, already exists as immutable") == "exists" + assert zips.classify_release_error("Cannot create ref refs/tags/x") == "immutable" + assert zips.classify_release_error("network error 500") == "fatal" + + +def test_upload_retry_suffixes_on_immutable(monkeypatch): + calls = [] + + def fake_create(tag, repo, title, notes, asset): + calls.append(tag) + if tag == "demo-1.0.0": + return 1, "tag is immutable" + return 0, "" + + monkeypatch.setattr(zips.gh, "release_create_capture", fake_create) + final_tag, skipped = zips._upload_with_retry("demo-1.0.0", "org/repo", "t", "n", "z.zip") + assert final_tag == "demo-1.0.0-1" + assert skipped is False + assert calls == ["demo-1.0.0", "demo-1.0.0-1"] + + +def test_upload_skip_when_release_exists(monkeypatch): + monkeypatch.setattr(zips.gh, "release_create_capture", + lambda *a: (1, "release already exists")) + final_tag, skipped = zips._upload_with_retry("demo-1.0.0", "org/repo", "t", "n", "z.zip") + assert final_tag == "demo-1.0.0" and skipped is True + + +def test_upload_fatal_returns_none(monkeypatch): + monkeypatch.setattr(zips.gh, "release_create_capture", lambda *a: (1, "boom")) + assert zips._upload_with_retry("demo-1.0.0", "org/repo", "t", "n", "z.zip") is None + + +# ---- icon sync ---- +def test_sync_icon_prefers_png(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + pdir = tmp_path / "plugins" / "demo" + pdir.mkdir(parents=True) + (pdir / "logo.png").write_bytes(b"png") + (pdir / "logo.svg").write_bytes(b"svg") + (tmp_path / "metadata" / "demo").mkdir(parents=True) + m._sync_icon(str(pdir) + "/", "demo") + assert (tmp_path / "metadata" / "demo" / "logo.png").read_bytes() == b"png" + assert not (tmp_path / "metadata" / "demo" / "logo.svg").exists() + + +def test_sync_icon_falls_back_to_webp(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + pdir = tmp_path / "plugins" / "demo" + pdir.mkdir(parents=True) + (pdir / "logo.webp").write_bytes(b"webp") + (tmp_path / "metadata" / "demo").mkdir(parents=True) + m._sync_icon(str(pdir) + "/", "demo") + assert (tmp_path / "metadata" / "demo" / "logo.webp").read_bytes() == b"webp" + + +def test_sync_icon_no_logo_is_noop(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + pdir = tmp_path / "plugins" / "demo" + pdir.mkdir(parents=True) + (tmp_path / "metadata" / "demo").mkdir(parents=True) + m._sync_icon(str(pdir) + "/", "demo") + assert list((tmp_path / "metadata" / "demo").iterdir()) == [] + + +# ---- canonical ZIP filename in version url (retry suffix in dir, not filename) ---- +def test_version_entry_url_uses_canonical_filename(): + # generate() builds this url; assert the composition rule directly + zip_version = "1.0.0-1" # release tag carried a retry suffix + canonical = m._canonical(zip_version) + assert canonical == "1.0.0" + url = f"demo-{zip_version}/demo-{canonical}.zip" + assert url == "demo-1.0.0-1/demo-1.0.0.zip" + entry = m.build_version_entry({"version": canonical}, url, 5, canonical) + assert entry["url"] == "demo-1.0.0-1/demo-1.0.0.zip" diff --git a/.github/pluginctl/tests/test_readmes.py b/.github/pluginctl/tests/test_readmes.py index 6003d9c..06d181a 100644 --- a/.github/pluginctl/tests/test_readmes.py +++ b/.github/pluginctl/tests/test_readmes.py @@ -50,6 +50,6 @@ def test_render_plugin_block_uses_version_count(): is_deprecated=True, plugin_name="old", plugin_raw=raw, manifest=None, last_updated="2024-01-01T00:00:00Z", commit_sha="abc", commit_sha_short="abc", version_count=3, repository="org/repo", source_branch="main", - releases_branch="releases", root_url="https://dl", has_source_readme=False) + releases_branch="releases", download_base_url="https://dl", has_source_readme=False) assert "### [Old](https://github.com/org/repo/blob/releases/metadata/old/README.md) (deprecated)" in block assert "- [All Versions (3 available)](./metadata/old)" in block From 9a0005db25dadec97c1d809cde9ba192f63ac756 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Wed, 22 Jul 2026 11:49:55 -0400 Subject: [PATCH 3/6] Run pluginctl test suite on tooling changes and gate repo-update PRs on it Adds a tests.yml workflow that runs the pluginctl pytest suite, triggered on push to main for any change outside plugins/, and also exposed via workflow_call so validate-plugin.yml can run it on a PR's merge commit. It runs with a read-only token and no secrets. - validate-plugin.yml gains a test-suite job that calls tests.yml for authorized repo-update PRs (outside_files present, not a violation, not closing). - The result flows through report (a "Tooling test suite" comment section on failure, folded into overall_failed) and through the Plugin PR Check gate, which now fails a repo-update PR with red tests even when validation is otherwise skipped. gate.evaluate and report.build_comment take a test_result; cmd_gate/cmd_report read TEST_RESULT. - Branch protection is unchanged: still one required check (Plugin PR Check). --- .github/pluginctl/src/pluginctl/cli.py | 1 + .../pluginctl/src/pluginctl/validate/gate.py | 7 ++- .../src/pluginctl/validate/report.py | 15 +++++- .github/pluginctl/tests/test_labels_gate.py | 20 ++++++++ .github/pluginctl/tests/test_report.py | 19 +++++++ .github/workflows/tests.yml | 50 +++++++++++++++++++ .github/workflows/validate-plugin.yml | 22 ++++++-- 7 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/tests.yml diff --git a/.github/pluginctl/src/pluginctl/cli.py b/.github/pluginctl/src/pluginctl/cli.py index 1e7a58e..c62e6a9 100644 --- a/.github/pluginctl/src/pluginctl/cli.py +++ b/.github/pluginctl/src/pluginctl/cli.py @@ -87,6 +87,7 @@ def cmd_gate(args) -> int: clamav_status=_env("CLAMAV_STATUS"), validate_result=_env("VALIDATE_RESULT"), report_result=_env("REPORT_RESULT"), + test_result=_env("TEST_RESULT", "skipped"), ) if result.ok: actions.log(result.message) diff --git a/.github/pluginctl/src/pluginctl/validate/gate.py b/.github/pluginctl/src/pluginctl/validate/gate.py index 24506bd..d2f71ec 100644 --- a/.github/pluginctl/src/pluginctl/validate/gate.py +++ b/.github/pluginctl/src/pluginctl/validate/gate.py @@ -20,10 +20,15 @@ def evaluate(detect_result: str, close_pr: str, skip_validation: str, outside_violation: str, title_result: str, codeql_result: str, codeql_status: str, clamav_result: str, clamav_status: str, - validate_result: str, report_result: str) -> GateResult: + validate_result: str, report_result: str, + test_result: str = "skipped") -> GateResult: """Return pass/fail with the exact ::error:: message the Bash ladder emits.""" if detect_result != "success": return GateResult(False, "Plugin detection failed or no plugin changes found.") + # Tooling test suite gates repo-update PRs. Checked before the skip_validation + # early-pass so a pure repo update cannot merge with red tests. + if test_result in ("failure", "cancelled"): + return GateResult(False, "Tooling test suite failed. See the Test Suite job for details.") if skip_validation == "true": return GateResult(True, "No plugin changes detected and author has write access - passing.") if title_result == "failure": diff --git a/.github/pluginctl/src/pluginctl/validate/report.py b/.github/pluginctl/src/pluginctl/validate/report.py index 4c43060..038a98d 100644 --- a/.github/pluginctl/src/pluginctl/validate/report.py +++ b/.github/pluginctl/src/pluginctl/validate/report.py @@ -74,7 +74,7 @@ def build_comment(*, plugin_count: str, close_pr: bool, close_reason: str, codeql_mediums: str, codeql_lows: str, codeql_unscanned_langs: str, clamav_result: str, clamav_infected: str, title_valid: str, title_feedback: str, title_suggestion: str, repository: str, - fragment_failed: bool, + fragment_failed: bool, test_result: str = "skipped", codeql_findings: str = "", codeql_medium_findings: str = "", codeql_low_findings: str = "", clamav_findings: str = "", other_plugins_section: str = "") -> str: @@ -82,6 +82,8 @@ def build_comment(*, plugin_count: str, close_pr: bool, close_reason: str, overall_failed = fragment_failed if title_valid and title_valid != "true": overall_failed = True + if test_result in ("failure", "cancelled"): + overall_failed = True L.append(MARKER) L.append("") @@ -252,6 +254,16 @@ def build_comment(*, plugin_count: str, close_pr: bool, close_reason: str, L.append(f"**Suggested format:** `{title_suggestion}`") L.append("") + if test_result in ("failure", "cancelled"): + L.append("") + L.append("---") + L.append("") + L.append("### ❌ Tooling test suite") + L.append("") + L.append("The automation test suite (`pluginctl`) failed for this change. " + "See the **Test Suite** job in this workflow run for details.") + L.append("") + L.append("") L.append("---") L.append("") @@ -386,6 +398,7 @@ def run(pr_number: str, pr_author: str, plugin_count: str, close_pr: bool, title_suggestion=os.environ.get("TITLE_SUGGESTION", ""), repository=repo, fragment_failed=parsed.any_failed, + test_result=os.environ.get("TEST_RESULT", "skipped"), codeql_findings=_read("codeql-findings/codeql-findings.md"), codeql_medium_findings=_read("codeql-medium-findings/codeql-medium-findings.md"), codeql_low_findings=_read("codeql-low-findings/codeql-low-findings.md"), diff --git a/.github/pluginctl/tests/test_labels_gate.py b/.github/pluginctl/tests/test_labels_gate.py index dcb6ba5..93c0eae 100644 --- a/.github/pluginctl/tests/test_labels_gate.py +++ b/.github/pluginctl/tests/test_labels_gate.py @@ -48,6 +48,26 @@ def test_gate_all_success(): assert r.ok and "validated successfully" in r.message +def test_gate_test_suite_failure_fails_even_when_skip_validation(): + # A pure repo update (skip_validation=true) must not pass with red tests. + r = gate.evaluate("success", "false", "true", "false", "", "", "", "", "", "", "", + test_result="failure") + assert not r.ok and "test suite" in r.message.lower() + + +def test_gate_test_suite_skipped_allows_skip_validation_pass(): + r = gate.evaluate("success", "false", "true", "false", "", "", "", "", "", "", "", + test_result="skipped") + assert r.ok + + +def test_gate_test_suite_success_allows_pass(): + r = gate.evaluate("success", "false", "false", "false", "success", + "success", "success", "success", "success", "success", "success", + test_result="success") + assert r.ok and "validated successfully" in r.message + + def test_gate_ladder_order_outside_violation_before_close(): # outside_violation takes precedence over close_pr in the ladder r = gate.evaluate("success", "true", "false", "true", "success", diff --git a/.github/pluginctl/tests/test_report.py b/.github/pluginctl/tests/test_report.py index 8c1bd9a..93a06a0 100644 --- a/.github/pluginctl/tests/test_report.py +++ b/.github/pluginctl/tests/test_report.py @@ -68,6 +68,25 @@ def test_title_invalid_block(): assert "## ❌ Validation failed" in c +def test_test_suite_failure_section_and_summary(): + c = report.build_comment(**_base_kwargs(test_result="failure")) + assert "### ❌ Tooling test suite" in c + assert "pluginctl" in c + assert "## ❌ Validation failed" in c + assert "🎉" not in c + + +def test_test_suite_success_renders_nothing(): + c = report.build_comment(**_base_kwargs(test_result="success")) + assert "Tooling test suite" not in c + assert "## 🎉 All validation checks passed!" in c + + +def test_test_suite_skipped_renders_nothing(): + c = report.build_comment(**_base_kwargs(test_result="skipped")) + assert "Tooling test suite" not in c + + def test_codeql_skipped_notice_no_separator_when_only_success(): c = report.build_comment(**_base_kwargs(codeql_result="success")) # success + no unscanned -> no findings separator, still passes diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..0b48077 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,50 @@ +name: Tests + +# Runs the pluginctl pytest suite. Dual purpose: +# - on push to main for any change outside plugins/ (tooling CI) +# - via workflow_call from validate-plugin.yml, so a repo-update PR's result +# feeds the Plugin PR Check gate and the PR comment +# Runs with a read-only token and no secrets, so testing PR merge-ref code is +# low-risk. + +on: + push: + branches: [main] + paths-ignore: + - 'plugins/**' + workflow_dispatch: + workflow_call: + inputs: + ref: + description: Git ref to test (empty checks out the default for the event). + type: string + required: false + default: '' + +permissions: + contents: read + +jobs: + pytest: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + sparse-checkout: .github/pluginctl + sparse-checkout-cone-mode: false + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install pytest + run: python -m pip install --quiet pytest + + - name: Run test suite + working-directory: .github/pluginctl + run: python -m pytest -q diff --git a/.github/workflows/validate-plugin.yml b/.github/workflows/validate-plugin.yml index 9b19ddc..68007c1 100644 --- a/.github/workflows/validate-plugin.yml +++ b/.github/workflows/validate-plugin.yml @@ -203,6 +203,20 @@ jobs: pr_number: ${{ github.event.pull_request.number }} base_ref: ${{ github.event.pull_request.base.ref }} + # -------------------------------------------------------------------------- + # Job 6b: Tooling test suite (reusable workflow) for authorized repo updates + # -------------------------------------------------------------------------- + test-suite: + needs: [detect-changes] + if: >- + needs.detect-changes.result == 'success' && + needs.detect-changes.outputs.outside_files != '' && + needs.detect-changes.outputs.outside_violation != 'true' && + needs.detect-changes.outputs.close_pr == 'false' + uses: ./.github/workflows/tests.yml + with: + ref: refs/pull/${{ github.event.pull_request.number }}/merge + # -------------------------------------------------------------------------- # Job 7: Validate each plugin in parallel via matrix # -------------------------------------------------------------------------- @@ -261,8 +275,8 @@ jobs: # Job 8: Aggregate all fragments, post PR comment, set final status # -------------------------------------------------------------------------- report: - needs: [detect-changes, validate-plugin, codeql-analyze, clamav-scan, validate-title] - if: always() && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.close_pr == 'false' && (needs.detect-changes.outputs.skip_validation != 'true' || needs.detect-changes.outputs.pub_key_changed == 'true' || needs.validate-title.outputs.title_valid == 'false') + needs: [detect-changes, validate-plugin, codeql-analyze, clamav-scan, validate-title, test-suite] + if: always() && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.close_pr == 'false' && (needs.detect-changes.outputs.skip_validation != 'true' || needs.detect-changes.outputs.pub_key_changed == 'true' || needs.validate-title.outputs.title_valid == 'false' || needs.test-suite.result == 'failure') runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -337,6 +351,7 @@ jobs: TITLE_VALID: ${{ needs.validate-title.outputs.title_valid }} TITLE_FEEDBACK: ${{ needs.validate-title.outputs.title_feedback }} TITLE_SUGGESTION: ${{ needs.validate-title.outputs.title_suggestion }} + TEST_RESULT: ${{ needs.test-suite.result }} BASE_REF: ${{ github.event.pull_request.base.ref }} WEBHOOK_URL: ${{ vars.WEBHOOK_URL }} WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} @@ -401,7 +416,7 @@ jobs: # -------------------------------------------------------------------------- plugin-pr-check: name: Plugin PR Check - needs: [detect-changes, codeql-analyze, clamav-scan, validate-plugin, validate-title, report] + needs: [detect-changes, codeql-analyze, clamav-scan, validate-plugin, validate-title, report, test-suite] if: always() runs-on: ubuntu-latest timeout-minutes: 2 @@ -439,4 +454,5 @@ jobs: CLAMAV_STATUS: ${{ needs.clamav-scan.outputs.clamav_status }} VALIDATE_RESULT: ${{ needs.validate-plugin.result }} REPORT_RESULT: ${{ needs.report.result }} + TEST_RESULT: ${{ needs.test-suite.result }} run: python -m pluginctl gate From 4999a49378016e34f2e3ea258a3facf93bde00a8 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Sat, 25 Jul 2026 11:55:14 -0400 Subject: [PATCH 4/6] Add regression tests for zero-padded version segments Upstream Plugins#193 fixed a Bash octal-parsing crash in version_greater_than() for versions like 0.2.08. This repo's pluginctl already avoids the bug since Python's int() has no octal ambiguity, but there was no test coverage proving it. --- .github/pluginctl/tests/test_version.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/pluginctl/tests/test_version.py b/.github/pluginctl/tests/test_version.py index ab89365..126c1dc 100644 --- a/.github/pluginctl/tests/test_version.py +++ b/.github/pluginctl/tests/test_version.py @@ -26,6 +26,9 @@ def test_is_dispatcharr_version(s, ok): ("1.0.0", "1.0.0", False), # equal is not greater ("1.0.0", "1.0.1", False), ("0.9.9", "1.0.0", False), + ("0.2.08", "0.2.07", True), # zero-padded segment (upstream Plugins#193 regression) + ("1.09.0", "1.08.0", True), # zero-padded minor segment + ("1.0.08", "1.0.8", False), # equal numeric value across padding, not greater ]) def test_version_greater_than(new, old, gt): assert v.version_greater_than(new, old) is gt From 117e352a3fcda4a4ca5ba83d52cd676d9a2ed38e Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Sat, 25 Jul 2026 12:46:31 -0400 Subject: [PATCH 5/6] Simplify pluginctl: dedupe git/identity/hashing/timestamp helpers, trim dead code Consolidates duplicated git subprocess wrapping and bot-identity setup into core/git.py, adds shared core/hashing.py and core/timeutil.py helpers, drops the unused core/models.Plugin, collapses redundant git-log/API/set_output calls, and removes assorted copy-paste and dead conditionals across validate/ and publish/. Behavior-preserving; pytest suite (130 tests) still passes. --- .github/pluginctl/README.md | 2 +- .github/pluginctl/src/pluginctl/cli.py | 17 ++-- .../pluginctl/src/pluginctl/core/actions.py | 31 +++--- .github/pluginctl/src/pluginctl/core/git.py | 37 ++++--- .../pluginctl/src/pluginctl/core/hashing.py | 17 ++++ .../pluginctl/src/pluginctl/core/models.py | 97 ------------------- .../pluginctl/src/pluginctl/core/timeutil.py | 10 ++ .../pluginctl/src/pluginctl/core/version.py | 9 ++ .../src/pluginctl/integrations/automerge.py | 16 ++- .../pluginctl/integrations/external_readme.py | 14 +-- .../src/pluginctl/integrations/webhooks.py | 5 +- .../src/pluginctl/publish/cleanup.py | 12 +-- .../src/pluginctl/publish/manifest.py | 60 ++++++------ .../src/pluginctl/publish/readmes.py | 93 +++++++++--------- .../pluginctl/src/pluginctl/publish/run.py | 62 ++++++------ .../pluginctl/src/pluginctl/publish/yank.py | 19 +--- .../pluginctl/src/pluginctl/publish/zips.py | 37 +++---- .../src/pluginctl/validate/clamav.py | 8 +- .../src/pluginctl/validate/detect.py | 64 ++++++------ .../src/pluginctl/validate/plugin.py | 26 ++--- .../src/pluginctl/validate/report.py | 11 ++- .../pluginctl/src/pluginctl/validate/sarif.py | 10 +- .github/pluginctl/tests/test_integrations.py | 6 +- 23 files changed, 294 insertions(+), 369 deletions(-) create mode 100644 .github/pluginctl/src/pluginctl/core/hashing.py delete mode 100644 .github/pluginctl/src/pluginctl/core/models.py create mode 100644 .github/pluginctl/src/pluginctl/core/timeutil.py diff --git a/.github/pluginctl/README.md b/.github/pluginctl/README.md index 3410b6f..10fbf6a 100644 --- a/.github/pluginctl/README.md +++ b/.github/pluginctl/README.md @@ -13,7 +13,7 @@ runner; there is no build step. ``` src/pluginctl/ cli.py argparse dispatch (one subcommand per verb) - core/ shared infra: actions, git, gh, jsonio, models, version + core/ shared infra: actions, git, gh, jsonio, hashing, timeutil, version validate/ PR-time pipeline: detect, title, labels, gate, sarif, langs, clamav, plugin (per-plugin checks), report publish/ releases pipeline: manifest, zips, cleanup, readmes, run, yank diff --git a/.github/pluginctl/src/pluginctl/cli.py b/.github/pluginctl/src/pluginctl/cli.py index c62e6a9..f6fb4b5 100644 --- a/.github/pluginctl/src/pluginctl/cli.py +++ b/.github/pluginctl/src/pluginctl/cli.py @@ -32,6 +32,11 @@ def _env(name: str, default: str = "") -> str: return os.environ.get(name, default) +def _repo(args) -> str: + """``--repo``, falling back to the repository the workflow runs in.""" + return args.repo or _env("GITHUB_REPOSITORY") + + # --------------------------------------------------------------------------- # # Handlers # --------------------------------------------------------------------------- # @@ -43,7 +48,7 @@ def cmd_detect(args) -> int: head_ref=args.head_ref or "", author_blacklist=_env("AUTHOR_BLACKLIST"), plugin_blacklist=_env("PLUGIN_BLACKLIST"), - repo=args.repo or _env("GITHUB_REPOSITORY"), + repo=_repo(args), ) @@ -100,7 +105,7 @@ def cmd_sarif(args) -> int: from .validate import sarif return sarif.run( results_dir=args.results_dir, - repo=args.repo or _env("GITHUB_REPOSITORY"), + repo=_repo(args), sha=args.sha, matrix=_matrix(args.matrix), analyze_outcome=args.analyze_outcome, @@ -118,7 +123,7 @@ def cmd_validate(args) -> int: pr_author=args.author, base_ref=args.base_ref, output_file=args.out, - repo=args.repo or _env("GITHUB_REPOSITORY"), + repo=_repo(args), ) @@ -130,7 +135,7 @@ def cmd_report(args) -> int: plugin_count=args.plugin_count, close_pr=_bool(args.close_pr), fragments_dir=args.fragments_dir, - repo=args.repo or _env("GITHUB_REPOSITORY"), + repo=_repo(args), ) @@ -148,8 +153,8 @@ def cmd_clamav_report(args) -> int: def cmd_webhook(args) -> int: from .integrations import webhooks data = json.loads(args.data) if args.data else {} - sent = webhooks.emit(args.event, data) - return 0 if sent or True else 1 # emission failures never fail the pipeline + webhooks.emit(args.event, data) + return 0 # emission failures never fail the pipeline def cmd_publish(args) -> int: diff --git a/.github/pluginctl/src/pluginctl/core/actions.py b/.github/pluginctl/src/pluginctl/core/actions.py index 3ba14f1..6e5d241 100644 --- a/.github/pluginctl/src/pluginctl/core/actions.py +++ b/.github/pluginctl/src/pluginctl/core/actions.py @@ -11,28 +11,35 @@ import uuid -def set_output(name: str, value: str) -> None: - """Append ``name=value`` to ``$GITHUB_OUTPUT`` (multiline-safe). +def _format_output(name: str, value: str) -> str: + """One ``$GITHUB_OUTPUT`` entry; multiline values use GitHub's heredoc form + with a random delimiter to avoid collisions with the payload.""" + if "\n" in value: + delim = f"ghadelim_{uuid.uuid4().hex}" + return f"{name}<<{delim}\n{value}\n{delim}\n" + return f"{name}={value}\n" + - Values containing a newline use the heredoc form GitHub documents, with a - random delimiter to avoid collisions with the payload. - """ +def set_output(name: str, value: str) -> None: + """Append ``name=value`` to ``$GITHUB_OUTPUT`` (multiline-safe).""" path = os.environ.get("GITHUB_OUTPUT") if not path: # Local/dry-run: echo to stdout the same way the shell scripts did. print(f"{name}={value}") return with open(path, "a", encoding="utf-8") as fh: - if "\n" in value: - delim = f"ghadelim_{uuid.uuid4().hex}" - fh.write(f"{name}<<{delim}\n{value}\n{delim}\n") - else: - fh.write(f"{name}={value}\n") + fh.write(_format_output(name, value)) def set_outputs(**kwargs: str) -> None: - for name, value in kwargs.items(): - set_output(name, "" if value is None else str(value)) + values = {name: "" if value is None else str(value) for name, value in kwargs.items()} + path = os.environ.get("GITHUB_OUTPUT") + if not path: + for name, value in values.items(): + print(f"{name}={value}") + return + with open(path, "a", encoding="utf-8") as fh: + fh.write("".join(_format_output(name, value) for name, value in values.items())) def step_summary(markdown: str) -> None: diff --git a/.github/pluginctl/src/pluginctl/core/git.py b/.github/pluginctl/src/pluginctl/core/git.py index d347f58..9c8f233 100644 --- a/.github/pluginctl/src/pluginctl/core/git.py +++ b/.github/pluginctl/src/pluginctl/core/git.py @@ -6,7 +6,8 @@ from typing import Optional -def _run(args: list[str], check: bool = True) -> subprocess.CompletedProcess: +def run(*args: str, check: bool = True) -> subprocess.CompletedProcess: + """``git `` with output captured as text.""" return subprocess.run( ["git", *args], check=check, @@ -15,15 +16,29 @@ def _run(args: list[str], check: bool = True) -> subprocess.CompletedProcess: ) +def configure_identity(app_slug: str) -> None: + """Set the committer identity to the GitHub App bot, else github-actions[bot].""" + if not app_slug: + run("config", "user.name", "github-actions[bot]") + run("config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com") + return + from . import gh + bot_user_id = gh.api(f"/users/{app_slug}%5Bbot%5D", jq=".id") or "" + run("config", "user.name", f"{app_slug}[bot]") + email = (f"{bot_user_id}+{app_slug}[bot]@users.noreply.github.com" if bot_user_id + else f"{app_slug}[bot]@users.noreply.github.com") + run("config", "user.email", email) + + def merge_base(a: str, b: str) -> str: - return _run(["merge-base", a, b]).stdout.strip() + return run("merge-base", a, b).stdout.strip() def diff_name_only(base: str, head: str = "HEAD", paths: Optional[list[str]] = None) -> list[str]: args = ["diff", "--name-only", base, head] if paths: args += ["--", *paths] - out = _run(args).stdout + out = run(*args).stdout return [line for line in out.splitlines() if line] @@ -32,20 +47,20 @@ def diff_name_only_range(range_spec: str, paths: Optional[list[str]] = None) -> args = ["diff", "--name-only", range_spec] if paths: args += ["--", *paths] - out = _run(args).stdout + out = run(*args).stdout return [line for line in out.splitlines() if line] def show(ref_path: str) -> Optional[str]: """``git show :`` returning None when the object does not exist.""" - proc = _run(["show", ref_path], check=False) + proc = run("show", ref_path, check=False) if proc.returncode != 0: return None return proc.stdout def object_exists(ref_path: str) -> bool: - return _run(["show", ref_path], check=False).returncode == 0 + return run("show", ref_path, check=False).returncode == 0 def log_format(fmt: str, ref: str, path: Optional[str] = None) -> Optional[str]: @@ -53,7 +68,7 @@ def log_format(fmt: str, ref: str, path: Optional[str] = None) -> Optional[str]: args = ["log", "-1", f"--format={fmt}", ref] if path is not None: args += ["--", path] - proc = _run(args, check=False) + proc = run(*args, check=False) if proc.returncode != 0: return None out = proc.stdout.strip() @@ -65,16 +80,16 @@ def rev_parse(rev: str, short: bool = False) -> str: if short: args.append("--short") args.append(rev) - return _run(args).stdout.strip() + return run(*args).stdout.strip() def fetch(*args: str) -> None: - _run(["fetch", *args], check=False) + run("fetch", *args, check=False) def sparse_checkout_add(path: str) -> bool: - return _run(["sparse-checkout", "add", path], check=False).returncode == 0 + return run("sparse-checkout", "add", path, check=False).returncode == 0 def checkout(*args: str) -> bool: - return _run(["checkout", *args], check=False).returncode == 0 + return run("checkout", *args, check=False).returncode == 0 diff --git a/.github/pluginctl/src/pluginctl/core/hashing.py b/.github/pluginctl/src/pluginctl/core/hashing.py new file mode 100644 index 0000000..98e6a50 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/hashing.py @@ -0,0 +1,17 @@ +"""Chunked file hashing shared by the ZIP builder and the ClamAV report.""" + +from __future__ import annotations + +import hashlib + +CHUNK_SIZE = 1 << 20 + + +def file_digests(path: str, *algorithms: str) -> list[str]: + """Hex digests of ``path`` for each named algorithm, reading the file once.""" + hashers = [hashlib.new(name) for name in algorithms] + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(CHUNK_SIZE), b""): + for hasher in hashers: + hasher.update(chunk) + return [hasher.hexdigest() for hasher in hashers] diff --git a/.github/pluginctl/src/pluginctl/core/models.py b/.github/pluginctl/src/pluginctl/core/models.py deleted file mode 100644 index d88625b..0000000 --- a/.github/pluginctl/src/pluginctl/core/models.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Typed view over a ``plugin.json`` document. - -Wraps the parsed dict and exposes the handful of accessors the validate/publish -logic needs, with the same ``// ""`` / ``// false`` defaulting the jq programs -used. ``raw`` stays available for the manifest builders that must preserve the -original key set and ordering. -""" - -from __future__ import annotations - -import json -from dataclasses import dataclass, field -from typing import Any, Optional - - -@dataclass -class Plugin: - raw: dict = field(default_factory=dict) - - # ---- constructors ------------------------------------------------------- - @classmethod - def from_json_text(cls, text: str) -> "Plugin": - return cls(raw=json.loads(text)) - - @classmethod - def from_file(cls, path: str) -> "Plugin": - with open(path, encoding="utf-8") as fh: - return cls(raw=json.load(fh)) - - # ---- scalar accessors (jq `// ""` semantics) ---------------------------- - def str_field(self, key: str, default: str = "") -> str: - val = self.raw.get(key) - return default if val is None else str(val) - - @property - def name(self) -> str: - return self.str_field("name") - - @property - def version(self) -> str: - return self.str_field("version") - - @property - def description(self) -> str: - return self.str_field("description") - - @property - def author(self) -> str: - return self.str_field("author") - - @property - def license(self) -> str: - return self.str_field("license") - - @property - def repo_url(self) -> str: - return self.str_field("repo_url") - - @property - def discord_thread(self) -> str: - return self.str_field("discord_thread") - - @property - def source_type(self) -> str: - return self.str_field("source_type", "local") - - @property - def source_url(self) -> str: - return self.str_field("source_url") - - @property - def min_dispatcharr_version(self) -> str: - return self.str_field("min_dispatcharr_version") - - @property - def max_dispatcharr_version(self) -> str: - return self.str_field("max_dispatcharr_version") - - @property - def deprecated(self) -> bool: - return self.raw.get("deprecated") is True - - @property - def unlisted(self) -> bool: - return self.raw.get("unlisted") is True - - @property - def maintainers(self) -> list[str]: - """``[.maintainers[]?]``: always a list, dropping non-list values.""" - val = self.raw.get("maintainers") - return list(val) if isinstance(val, list) else [] - - def maintainers_joined(self, sep: str = " ") -> str: - return sep.join(str(m) for m in self.maintainers) - - def get(self, key: str, default: Any = None) -> Any: - return self.raw.get(key, default) diff --git a/.github/pluginctl/src/pluginctl/core/timeutil.py b/.github/pluginctl/src/pluginctl/core/timeutil.py new file mode 100644 index 0000000..9ea1f1e --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/timeutil.py @@ -0,0 +1,10 @@ +"""Shared timestamp formatting (one definition of the manifest/build ISO format).""" + +from __future__ import annotations + +import datetime + + +def now_iso() -> str: + """Current UTC time as ``YYYY-MM-DDTHH:MM:SSZ`` (the format every artifact uses).""" + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/.github/pluginctl/src/pluginctl/core/version.py b/.github/pluginctl/src/pluginctl/core/version.py index f279b8f..2557027 100644 --- a/.github/pluginctl/src/pluginctl/core/version.py +++ b/.github/pluginctl/src/pluginctl/core/version.py @@ -87,3 +87,12 @@ def _version_cmp(a: str, b: str) -> int: def sort_versions_desc(versions: list[str]) -> list[str]: """Sort bare version strings newest-first, like ``sort -V -r``.""" return sorted(versions, key=cmp_to_key(_version_cmp), reverse=True) + + +def versioned_tags(all_tags: list[str], plugin_name: str) -> list[str]: + """``-`` release tags for one plugin, newest-first, minus ``-latest``.""" + prefix = f"{plugin_name}-" + latest = f"{plugin_name}-latest" + versions = [t[len(prefix):] for t in all_tags + if t.startswith(prefix) and t != latest] + return [f"{prefix}{v}" for v in sort_versions_desc(versions)] diff --git a/.github/pluginctl/src/pluginctl/integrations/automerge.py b/.github/pluginctl/src/pluginctl/integrations/automerge.py index bef2611..5793106 100644 --- a/.github/pluginctl/src/pluginctl/integrations/automerge.py +++ b/.github/pluginctl/src/pluginctl/integrations/automerge.py @@ -12,18 +12,14 @@ import json import os -import subprocess from dataclasses import dataclass from ..core import actions, gh +from ..core.git import run as _git BLOCKING_LABELS = ("New Plugin", "Repo Update", "Invalid", "QUARANTINE") -def _git(*args: str, check: bool = False) -> subprocess.CompletedProcess: - return subprocess.run(["git", *args], check=check, capture_output=True, text=True) - - @dataclass class LabelDecision: ok: bool @@ -45,12 +41,12 @@ def evaluate_labels(labels: list[str], mergeable: str) -> LabelDecision: def reverify_is_pure_update(pr_number: str, base_branch: str = "main") -> bool: """Re-derive from the diff that the PR only updates existing plugins.""" - if _git("fetch", "origin", f"pull/{pr_number}/merge:pr-merge-{pr_number}").returncode != 0: + if _git("fetch", "origin", f"pull/{pr_number}/merge:pr-merge-{pr_number}", check=False).returncode != 0: actions.notice(f"Could not fetch merge ref for PR #{pr_number} - skipping.") return False - _git("checkout", "-q", f"pr-merge-{pr_number}") - merge_base = _git("merge-base", f"origin/{base_branch}", "HEAD").stdout.strip() - changed = _git("diff", "--name-only", merge_base, "HEAD").stdout.splitlines() + _git("checkout", "-q", f"pr-merge-{pr_number}", check=False) + merge_base = _git("merge-base", f"origin/{base_branch}", "HEAD", check=False).stdout.strip() + changed = _git("diff", "--name-only", merge_base, "HEAD", check=False).stdout.splitlines() outside = [f for f in changed if not f.startswith("plugins/")] if outside: @@ -64,7 +60,7 @@ def reverify_is_pure_update(pr_number: str, base_branch: str = "main") -> bool: return False for slug in slugs: - if _git("show", f"origin/{base_branch}:plugins/{slug}/plugin.json").returncode != 0: + if _git("show", f"origin/{base_branch}:plugins/{slug}/plugin.json", check=False).returncode != 0: actions.notice(f"Plugin '{slug}' does not exist on {base_branch} - this is a new " "plugin, not an update. Skipping.") return False diff --git a/.github/pluginctl/src/pluginctl/integrations/external_readme.py b/.github/pluginctl/src/pluginctl/integrations/external_readme.py index eb95ed9..70ee1a6 100644 --- a/.github/pluginctl/src/pluginctl/integrations/external_readme.py +++ b/.github/pluginctl/src/pluginctl/integrations/external_readme.py @@ -15,7 +15,7 @@ import subprocess from typing import Optional -from ..core import actions, gh +from ..core import actions, gh, git BRANCH = "auto/dispatcharr-plugin-readme" _SOURCE_COMMIT_RE = re.compile(r"Source commit: ([0-9a-f]+)") @@ -69,18 +69,14 @@ def run(releases_ref: str = "origin/releases") -> int: source_repo = os.environ.get("SOURCE_REPO") or os.environ.get("GITHUB_REPOSITORY", "") reviewer = os.environ.get("PR_REVIEWER", "") - show = subprocess.run(["git", "show", f"{releases_ref}:README.md"], - capture_output=True, text=True) - if show.returncode != 0: + readme = git.show(f"{releases_ref}:README.md") + if readme is None: actions.error("README.md not found on the releases branch - has the Publish Plugins workflow run yet?") return 1 - readme = show.stdout - commit_msg = subprocess.run(["git", "log", "-1", "--format=%B", releases_ref], - capture_output=True, text=True).stdout + commit_msg = git.log_format("%B", releases_ref) or "" source_commit, plugin_list = extract_metadata(commit_msg) - releases_commit = subprocess.run(["git", "rev-parse", "--short", releases_ref], - capture_output=True, text=True).stdout.strip() + releases_commit = git.rev_parse(releases_ref, short=True) contrib_url = f"https://github.com/{source_repo}/blob/main/CONTRIBUTING.md" readme = strip_preamble(readme, contrib_url) diff --git a/.github/pluginctl/src/pluginctl/integrations/webhooks.py b/.github/pluginctl/src/pluginctl/integrations/webhooks.py index 1bd71e7..9abc4bb 100644 --- a/.github/pluginctl/src/pluginctl/integrations/webhooks.py +++ b/.github/pluginctl/src/pluginctl/integrations/webhooks.py @@ -13,7 +13,6 @@ from __future__ import annotations -import datetime import hashlib import hmac import json @@ -24,6 +23,7 @@ from typing import Any, Optional from ..core import actions +from ..core.timeutil import now_iso USER_AGENT = "pluginctl-webhook/1" @@ -37,8 +37,7 @@ def sign(secret: str, body: bytes) -> str: def build_envelope(event: str, data: dict, repository: str, actor: str) -> dict: return { "event": event, - "delivered_at": datetime.datetime.now(datetime.timezone.utc) - .strftime("%Y-%m-%dT%H:%M:%SZ"), + "delivered_at": now_iso(), "repository": repository, "actor": actor, "data": data, diff --git a/.github/pluginctl/src/pluginctl/publish/cleanup.py b/.github/pluginctl/src/pluginctl/publish/cleanup.py index f090a24..c8587ce 100644 --- a/.github/pluginctl/src/pluginctl/publish/cleanup.py +++ b/.github/pluginctl/src/pluginctl/publish/cleanup.py @@ -10,15 +10,7 @@ import shutil from ..core import actions, gh -from ..core.version import sort_versions_desc - - -def _versioned_tags(all_tags: list[str], plugin_name: str) -> list[str]: - prefix = f"{plugin_name}-" - latest = f"{plugin_name}-latest" - versions = [t[len(prefix):] for t in all_tags - if t.startswith(prefix) and t != latest] - return [f"{prefix}{v}" for v in sort_versions_desc(versions)] +from ..core.version import versioned_tags def run(repository: str, max_versioned_zips: int = 10) -> int: @@ -38,7 +30,7 @@ def run(repository: str, max_versioned_zips: int = 10) -> int: # Prune old versioned releases per plugin (keep the newest max_versioned_zips). for plugin_dir in sorted(glob.glob("plugins/*/")): plugin_name = os.path.basename(plugin_dir.rstrip("/")) - tags = _versioned_tags(all_tags, plugin_name) + tags = versioned_tags(all_tags, plugin_name) if len(tags) <= max_versioned_zips: continue for old_tag in tags[max_versioned_zips:]: diff --git a/.github/pluginctl/src/pluginctl/publish/manifest.py b/.github/pluginctl/src/pluginctl/publish/manifest.py index 941e354..df8e5c1 100644 --- a/.github/pluginctl/src/pluginctl/publish/manifest.py +++ b/.github/pluginctl/src/pluginctl/publish/manifest.py @@ -19,6 +19,27 @@ from ..core import jsonio from ..core.jsonio import drop_none +from ..core.timeutil import now_iso +from ..core.version import versioned_tags + + +def _metadata_fields(metadata: dict, min_da, max_da) -> dict: + """The build-metadata fields shared by ``versions[]`` and ``latest`` entries. + + Key order matters: it is the jq emission order the Bash script produced. + """ + return { + "version": metadata.get("version"), + "commit_sha": metadata.get("commit_sha"), + "commit_sha_short": metadata.get("commit_sha_short"), + "build_timestamp": metadata.get("build_timestamp"), + "last_updated": metadata.get("last_updated"), + "checksum_md5": metadata.get("checksum_md5"), + "checksum_sha256": metadata.get("checksum_sha256"), + "min_dispatcharr_version": min_da, + "max_dispatcharr_version": max_da, + "source_url": metadata.get("source_url"), + } def build_version_entry(metadata: dict, url: str, size, canonical_version: str) -> dict: @@ -30,16 +51,9 @@ def build_version_entry(metadata: dict, url: str, size, canonical_version: str) """ if metadata: return drop_none({ - "version": metadata.get("version"), - "commit_sha": metadata.get("commit_sha"), - "commit_sha_short": metadata.get("commit_sha_short"), - "build_timestamp": metadata.get("build_timestamp"), - "last_updated": metadata.get("last_updated"), - "checksum_md5": metadata.get("checksum_md5"), - "checksum_sha256": metadata.get("checksum_sha256"), - "min_dispatcharr_version": metadata.get("min_dispatcharr_version"), - "max_dispatcharr_version": metadata.get("max_dispatcharr_version"), - "source_url": metadata.get("source_url"), + **_metadata_fields(metadata, + metadata.get("min_dispatcharr_version"), + metadata.get("max_dispatcharr_version")), "url": url, "size": size, }) @@ -72,16 +86,9 @@ def g(key): latest = None if latest_metadata: latest = drop_none({ - "version": latest_metadata.get("version"), - "commit_sha": latest_metadata.get("commit_sha"), - "commit_sha_short": latest_metadata.get("commit_sha_short"), - "build_timestamp": latest_metadata.get("build_timestamp"), - "last_updated": latest_metadata.get("last_updated"), - "checksum_md5": latest_metadata.get("checksum_md5"), - "checksum_sha256": latest_metadata.get("checksum_sha256"), - "min_dispatcharr_version": g("min_dispatcharr_version"), - "max_dispatcharr_version": g("max_dispatcharr_version"), - "source_url": latest_metadata.get("source_url"), + **_metadata_fields(latest_metadata, + g("min_dispatcharr_version"), + g("max_dispatcharr_version")), "latest_url": latest_url, "url": versioned_zips[0]["url"] if versioned_zips else None, "size": latest_size_kb, @@ -235,8 +242,8 @@ def import_gpg_key(private_key: str) -> tuple[str, bool]: Empty ``private_key`` -> ("", False) (signing disabled, not a failure). """ + from ..core import actions if not private_key: - from ..core import actions actions.log("GPG_PRIVATE_KEY not set - signatures will be skipped.") return "", False subprocess.run(["gpg", "--batch", "--import"], input=private_key, @@ -250,7 +257,6 @@ def import_gpg_key(private_key: str) -> tuple[str, bool]: if len(parts) >= 2 and "/" in parts[1]: key_id = parts[1].split("/", 1)[1] break - from ..core import actions if key_id: actions.log(f"GPG signing enabled (key: {key_id})") return key_id, False @@ -295,11 +301,10 @@ def strip_signatures() -> None: def generate(source_branch: str, releases_branch: str, repository: str, build_meta_dir: str) -> int: """Full generate-manifest.sh: per-plugin manifests, root manifest, signing.""" - import datetime import glob from ..core import actions, gh - generated_at = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + generated_at = now_iso() registry_url = f"https://github.com/{repository}" registry_name = repository download_base_url = f"https://github.com/{repository}/releases/download" @@ -352,11 +357,8 @@ def sign_if_needed(path: str, written: bool) -> None: except ValueError: existing_manifest = None - from ..core.version import sort_versions_desc prefix = f"{plugin_name}-" - raw_versions = [t[len(prefix):] for t in all_tags - if t.startswith(prefix) and t != f"{plugin_name}-latest"] - versioned_tags = [f"{prefix}{v}" for v in sort_versions_desc(raw_versions)] + tags = versioned_tags(all_tags, plugin_name) versioned_zips: list[dict] = [] latest_metadata: dict = {} @@ -364,7 +366,7 @@ def sign_if_needed(path: str, written: bool) -> None: latest_size_kb = 0 latest_size_set = False - for release_tag in versioned_tags: + for release_tag in tags: zip_version = release_tag[len(prefix):] canonical_version = _canonical(zip_version) # Release dir uses the full tag version (may carry a retry suffix); diff --git a/.github/pluginctl/src/pluginctl/publish/readmes.py b/.github/pluginctl/src/pluginctl/publish/readmes.py index c55e7a7..690f2d9 100644 --- a/.github/pluginctl/src/pluginctl/publish/readmes.py +++ b/.github/pluginctl/src/pluginctl/publish/readmes.py @@ -13,9 +13,11 @@ import glob import json import os +import re from typing import Optional from ..core import git +from ..core.timeutil import now_iso def fmt_date(value: str) -> str: @@ -167,22 +169,18 @@ def render_plugin_readme(plugin_name: str, plugin_raw: dict, manifest: dict, def _badges(license_id: str, discord_thread: str, repo_url: str) -> str: - badges = "" + badges: list[str] = [] if license_id: - badges = (f"[![License: {license_id}]" - f"(https://img.shields.io/badge/License-{shields_encode(license_id)}-blue?style=flat-square)]" - f"(https://spdx.org/licenses/{license_id}.html)") + badges.append(f"[![License: {license_id}]" + f"(https://img.shields.io/badge/License-{shields_encode(license_id)}-blue?style=flat-square)]" + f"(https://spdx.org/licenses/{license_id}.html)") if discord_thread: - if badges: - badges += " " - badges += ("[![Discord](https://img.shields.io/badge/Discord-Discussion-5865F2?style=flat-square&logo=discord&logoColor=white)]" - f"({discord_thread})") + badges.append("[![Discord](https://img.shields.io/badge/Discord-Discussion-5865F2?style=flat-square&logo=discord&logoColor=white)]" + f"({discord_thread})") if repo_url: - if badges: - badges += " " - badges += ("[![Repository](https://img.shields.io/badge/GitHub-Repository-181717?style=flat-square&logo=github&logoColor=white)]" - f"({repo_url})") - return badges + badges.append("[![Repository](https://img.shields.io/badge/GitHub-Repository-181717?style=flat-square&logo=github&logoColor=white)]" + f"({repo_url})") + return " ".join(badges) def _compat_badges(min_da: str, max_da: str) -> str: @@ -210,14 +208,11 @@ def version_count_from_manifest(manifest_file: str) -> int: # --------------------------------------------------------------------------- # # Root releases README (README.md on the releases branch) # --------------------------------------------------------------------------- # -import re as _re - - def anchor_for(name: str) -> str: """GitHub-style heading anchor: lowercase, non [a-z0-9-] -> -, collapse runs.""" a = name.lower() - a = _re.sub(r"[^a-z0-9-]", "-", a) - a = _re.sub(r"-+", "-", a) + a = re.sub(r"[^a-z0-9-]", "-", a) + a = re.sub(r"-+", "-", a) return a @@ -297,12 +292,21 @@ def render_plugin_block(*, is_deprecated: bool, plugin_name: str, plugin_raw: di # --------------------------------------------------------------------------- # # Orchestrators (IO): called from the releases-branch checkout directory # --------------------------------------------------------------------------- # -def _now_iso() -> str: - return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") +def _plugin_commit(source_branch: str, plugin_dir: str) -> tuple[str, str, str]: + """(last_updated, commit_sha, commit_sha_short) of the plugin's last commit. + + One ``git log`` call for all three fields; empty strings when there is none. + """ + out = git.log_format("%cI%n%H%n%h", f"origin/{source_branch}", plugin_dir) + if not out: + return "", "", "" + fields = out.splitlines() + fields += [""] * (3 - len(fields)) + return fields[0], fields[1], fields[2] def _plugin_last_updated(source_branch: str, plugin_dir: str) -> str: - return git.log_format("%cI", f"origin/{source_branch}", plugin_dir) or _now_iso() + return _plugin_commit(source_branch, plugin_dir)[0] or now_iso() def generate_plugin_readmes(source_branch: str, repository: str) -> None: @@ -335,13 +339,16 @@ def generate_releases_readme(source_branch: str, releases_branch: str, repositor root = _load_json("manifest.json") or {} download_base_url = (root.get("manifest") or {}).get("download_base_url") or "" - plugin_dirs = sorted(glob.glob("plugins/*/")) raws: dict[str, dict] = {} - for pd in plugin_dirs: + has_deprecated = False + for pd in sorted(glob.glob("plugins/*/")): pname = os.path.basename(pd.rstrip("/")) raw = _load_json(os.path.join(pd, "plugin.json")) - if raw is not None: - raws[pname] = raw + if raw is None: + continue + raws[pname] = raw + if raw.get("deprecated") is True: + has_deprecated = True L: list[str] = [] L.append("# Plugin Releases") @@ -358,20 +365,13 @@ def generate_releases_readme(source_branch: str, releases_branch: str, repositor L.append("| Plugin | Version | Author | License | Description |") L.append("|--------|---------|-------|---------|-------------|") - for pass_name in ("active", "deprecated"): - for pd in plugin_dirs: - pname = os.path.basename(pd.rstrip("/")) - raw = raws.get(pname) - if raw is None: - continue - deprecated = raw.get("deprecated") is True + for deprecated_pass in (False, True): + for raw in raws.values(): if raw.get("unlisted") is True: continue - if pass_name == "active" and deprecated: - continue - if pass_name == "deprecated" and not deprecated: + if (raw.get("deprecated") is True) is not deprecated_pass: continue - L.append(table_row(raw, pass_name == "deprecated")) + L.append(table_row(raw, deprecated_pass)) L.append("") L.append("---") @@ -380,9 +380,10 @@ def generate_releases_readme(source_branch: str, releases_branch: str, repositor def emit_block(pname: str, raw: dict, is_deprecated: bool) -> None: pd = f"plugins/{pname}/" manifest = _load_json(f"metadata/{pname}/manifest.json") - last_updated = _plugin_last_updated(source_branch, pd) - commit_sha = git.log_format("%H", f"origin/{source_branch}", pd) or "unknown" - commit_short = git.log_format("%h", f"origin/{source_branch}", pd) or "unknown" + last_updated, commit_sha, commit_short = _plugin_commit(source_branch, pd) + last_updated = last_updated or now_iso() + commit_sha = commit_sha or "unknown" + commit_short = commit_short or "unknown" version_count = version_count_from_manifest(f"metadata/{pname}/manifest.json") has_source_readme = os.path.isfile(f"plugins/{pname}/README.md") L.append(render_plugin_block( @@ -394,25 +395,19 @@ def emit_block(pname: str, raw: dict, is_deprecated: bool) -> None: has_source_readme=has_source_readme)) # Active detailed sections - for pd in plugin_dirs: - pname = os.path.basename(pd.rstrip("/")) - raw = raws.get(pname) - if raw is None or raw.get("deprecated") is True or raw.get("unlisted") is True: + for pname, raw in raws.items(): + if raw.get("deprecated") is True or raw.get("unlisted") is True: continue emit_block(pname, raw, False) - has_deprecated = any( - r.get("deprecated") is True for r in raws.values()) if has_deprecated: L.append("") L.append("## Deprecated Plugins") L.append("") L.append("These plugins are deprecated and may be removed in the future.") L.append("") - for pd in plugin_dirs: - pname = os.path.basename(pd.rstrip("/")) - raw = raws.get(pname) - if raw is None or raw.get("deprecated") is not True or raw.get("unlisted") is True: + for pname, raw in raws.items(): + if raw.get("deprecated") is not True or raw.get("unlisted") is True: continue emit_block(pname, raw, True) diff --git a/.github/pluginctl/src/pluginctl/publish/run.py b/.github/pluginctl/src/pluginctl/publish/run.py index ea002d1..f06498e 100644 --- a/.github/pluginctl/src/pluginctl/publish/run.py +++ b/.github/pluginctl/src/pluginctl/publish/run.py @@ -12,7 +12,8 @@ import subprocess import tempfile -from ..core import actions, git as _git_mod +from ..core import actions +from ..core.git import configure_identity, run as _git from . import cleanup, manifest, readmes, zips RELEASES_BRANCH = "releases" @@ -20,26 +21,6 @@ RELEASES_BRANCH_VERSION = "3" -def _git(*args: str, check: bool = True) -> subprocess.CompletedProcess: - return subprocess.run(["git", *args], check=check, capture_output=True, text=True) - - -def _configure_identity(repository: str, app_slug: str) -> None: - if app_slug: - from ..core import gh - bot_user_id = gh.api(f"/users/{app_slug}%5Bbot%5D", jq=".id") or "" - _git("config", "user.name", f"{app_slug}[bot]") - if bot_user_id: - _git("config", "user.email", - f"{bot_user_id}+{app_slug}[bot]@users.noreply.github.com") - else: - _git("config", "user.email", f"{app_slug}[bot]@users.noreply.github.com") - else: - _git("config", "user.name", "github-actions[bot]") - _git("config", "user.email", - "41898282+github-actions[bot]@users.noreply.github.com") - - def run(source_branch: str) -> int: repository = os.environ["GITHUB_REPOSITORY"] token = os.environ["GITHUB_TOKEN"] @@ -56,7 +37,7 @@ def run(source_branch: str) -> int: try: _git("clone", "--no-checkout", remote, f"{workdir}/repo") os.chdir(f"{workdir}/repo") - _configure_identity(repository, app_slug) + configure_identity(app_slug) _setup_releases_branch(repository, source_branch, token, remote, force_rebuild, force_plugin) @@ -104,8 +85,7 @@ def _setup_releases_branch(repository, source_branch, token, remote, if force_rebuild and force_plugin: if branch_exists: - _git("checkout", RELEASES_BRANCH) - _git("pull", "origin", RELEASES_BRANCH, check=False) + _checkout_existing() else: _orphan_init() actions.log(f"Targeted force rebuild: deleting GitHub Releases for {force_plugin}") @@ -127,21 +107,23 @@ def _setup_releases_branch(repository, source_branch, token, remote, if tag.startswith(f"{pname}-"): gh.release_delete(tag, repository) subprocess.run(["rm", "-rf", "plugins"], check=False) - _git("checkout", "--orphan", RELEASES_BRANCH) - _git("rm", "-rf", ".", check=False) - _git("commit", "--allow-empty", "-m", f"Initialize {RELEASES_BRANCH} branch (force rebuild)") + _orphan_init(f"Initialize {RELEASES_BRANCH} branch (force rebuild)") _git("push", "--force", remote, RELEASES_BRANCH) elif branch_exists: - _git("checkout", RELEASES_BRANCH) - _git("pull", "origin", RELEASES_BRANCH, check=False) + _checkout_existing() else: _orphan_init() -def _orphan_init() -> None: +def _orphan_init(message: str = f"Initialize {RELEASES_BRANCH} branch") -> None: _git("checkout", "--orphan", RELEASES_BRANCH) _git("rm", "-rf", ".", check=False) - _git("commit", "--allow-empty", "-m", f"Initialize {RELEASES_BRANCH} branch") + _git("commit", "--allow-empty", "-m", message) + + +def _checkout_existing() -> None: + _git("checkout", RELEASES_BRANCH) + _git("pull", "origin", RELEASES_BRANCH, check=False) def _commit_and_push(repository, source_branch, remote) -> int: @@ -174,15 +156,27 @@ def _commit_and_push(repository, source_branch, remote) -> int: def _emit_published(repository: str, changed: str) -> None: - """Emit one plugin.published event per changed plugin (best-effort).""" + """Emit one plugin.published event per changed plugin (best-effort, concurrent). + + Deliveries are independent and already non-fatal, so they run in parallel + rather than serializing one 15s timeout per plugin on the publish path. + """ + from concurrent.futures import ThreadPoolExecutor + from ..integrations import webhooks actor = os.environ.get("GITHUB_ACTOR", "") + payloads = [] for line in changed.splitlines(): if "@" not in line: continue plugin_key, _, version = line.partition("@") - webhooks.emit("plugin.published", - webhooks.plugin_published(plugin_key.replace("_", "-"), version, None, actor)) + payloads.append(webhooks.plugin_published(plugin_key.replace("_", "-"), + version, None, actor)) + if not payloads: + return + with ThreadPoolExecutor(max_workers=min(8, len(payloads))) as pool: + for payload in payloads: + pool.submit(webhooks.emit, "plugin.published", payload) def _only_timestamps() -> bool: diff --git a/.github/pluginctl/src/pluginctl/publish/yank.py b/.github/pluginctl/src/pluginctl/publish/yank.py index fdc8963..e001f1a 100644 --- a/.github/pluginctl/src/pluginctl/publish/yank.py +++ b/.github/pluginctl/src/pluginctl/publish/yank.py @@ -16,16 +16,13 @@ import tempfile from ..core import actions, gh +from ..core.git import configure_identity, run as _git from ..core.version import sort_versions_desc from . import manifest, readmes RELEASES_BRANCH = "releases" -def _git(*args: str, check: bool = True) -> subprocess.CompletedProcess: - return subprocess.run(["git", *args], check=check, capture_output=True, text=True) - - def run() -> int: repository = os.environ["GITHUB_REPOSITORY"] token = os.environ["GITHUB_TOKEN"] @@ -54,7 +51,7 @@ def run() -> int: try: _git("clone", "--no-checkout", remote, f"{workdir}/repo") os.chdir(f"{workdir}/repo") - _configure_identity(repository, app_slug) + configure_identity(app_slug) if _git("ls-remote", "--exit-code", "--heads", "origin", RELEASES_BRANCH, check=False).returncode != 0: @@ -146,18 +143,6 @@ def run() -> int: subprocess.run(["rm", "-rf", workdir, build_meta_dir], check=False) -def _configure_identity(repository: str, app_slug: str) -> None: - if app_slug: - bot_user_id = gh.api(f"/users/{app_slug}%5Bbot%5D", jq=".id") or "" - _git("config", "user.name", f"{app_slug}[bot]") - email = (f"{bot_user_id}+{app_slug}[bot]@users.noreply.github.com" if bot_user_id - else f"{app_slug}[bot]@users.noreply.github.com") - _git("config", "user.email", email) - else: - _git("config", "user.name", "github-actions[bot]") - _git("config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com") - - def _commit_releases(remote, plugin, version, issue) -> str: actions.log("\n=== Committing ===") subprocess.run(["rm", "-rf", "plugins"], check=False) diff --git a/.github/pluginctl/src/pluginctl/publish/zips.py b/.github/pluginctl/src/pluginctl/publish/zips.py index 6e49308..f4a2bde 100644 --- a/.github/pluginctl/src/pluginctl/publish/zips.py +++ b/.github/pluginctl/src/pluginctl/publish/zips.py @@ -10,7 +10,6 @@ from __future__ import annotations import glob -import hashlib import json import os import shutil @@ -21,22 +20,9 @@ from typing import Optional from ..core import actions, gh, git, jsonio +from ..core.hashing import file_digests from ..core.jsonio import drop_none - - -def _now() -> str: - import datetime - return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def _checksums(path: str) -> tuple[str, str]: - md5 = hashlib.md5() - sha = hashlib.sha256() - with open(path, "rb") as fh: - for chunk in iter(lambda: fh.read(1 << 20), b""): - md5.update(chunk) - sha.update(chunk) - return md5.hexdigest(), sha.hexdigest() +from ..core.timeutil import now_iso def _download(url: str, dest: str, attempts: int = 3) -> bool: @@ -54,6 +40,16 @@ def _download(url: str, dest: str, attempts: int = 3) -> bool: return False +def _commit_info(source_branch: str, plugin_dir: str) -> tuple[str, str, str]: + """(commit_sha, commit_sha_short, commit_date) of the last commit touching the plugin.""" + out = git.log_format("%H%n%h%n%cI", f"origin/{source_branch}", plugin_dir) + if not out: + return "", "", "" + fields = out.splitlines() + fields += [""] * (3 - len(fields)) + return fields[0], fields[1], fields[2] + + def run(source_branch: str, repository: str, build_meta_dir: str) -> int: changed: list[str] = [] open("changed_plugins.txt", "w", encoding="utf-8").close() @@ -74,10 +70,9 @@ def run(source_branch: str, repository: str, build_meta_dir: str) -> int: continue source_type = raw.get("source_type") or "local" - build_timestamp = _now() - commit_sha = git.log_format("%H", f"origin/{source_branch}", plugin_dir) or "" - commit_sha_short = git.log_format("%h", f"origin/{source_branch}", plugin_dir) or "" - last_updated = git.log_format("%cI", f"origin/{source_branch}", plugin_dir) or _now() + build_timestamp = now_iso() + commit_sha, commit_sha_short, commit_date = _commit_info(source_branch, plugin_dir) + last_updated = commit_date or now_iso() source_url_resolved = "" if source_type == "external": @@ -95,7 +90,7 @@ def run(source_branch: str, repository: str, build_meta_dir: str) -> int: subprocess.run(["zip", "-r", zip_path, plugin_key], cwd=tmpdir, check=True, stdout=subprocess.DEVNULL) - checksum_md5, checksum_sha256 = _checksums(zip_path) + checksum_md5, checksum_sha256 = file_digests(zip_path, "md5", "sha256") min_da = str(raw.get("min_dispatcharr_version") or "") max_da = str(raw.get("max_dispatcharr_version") or "") zip_size_kb = os.path.getsize(zip_path) // 1024 diff --git a/.github/pluginctl/src/pluginctl/validate/clamav.py b/.github/pluginctl/src/pluginctl/validate/clamav.py index 880d439..201d5b2 100644 --- a/.github/pluginctl/src/pluginctl/validate/clamav.py +++ b/.github/pluginctl/src/pluginctl/validate/clamav.py @@ -6,20 +6,16 @@ from __future__ import annotations -import hashlib import os from typing import Callable, Optional from ..core import actions +from ..core.hashing import file_digests def _sha256_file(path: str) -> str: try: - h = hashlib.sha256() - with open(path, "rb") as fh: - for chunk in iter(lambda: fh.read(1 << 20), b""): - h.update(chunk) - return h.hexdigest() + return file_digests(path, "sha256")[0] except OSError: return "" diff --git a/.github/pluginctl/src/pluginctl/validate/detect.py b/.github/pluginctl/src/pluginctl/validate/detect.py index 63c50c4..69f23ce 100644 --- a/.github/pluginctl/src/pluginctl/validate/detect.py +++ b/.github/pluginctl/src/pluginctl/validate/detect.py @@ -23,6 +23,13 @@ def is_safe_name(name: str) -> bool: return bool(SAFE_NAME_RE.match(name)) +def author_in_plugin_json(pr_author: str, data: dict) -> bool: + """True when pr_author is the plugin's ``author`` or one of its ``maintainers``.""" + author = data.get("author") or "" + maintainers = [str(m) for m in (data.get("maintainers") or []) if m is not None] + return pr_author == author or pr_author in maintainers + + def author_has_plugin_permission(pr_author: str, base_json_text: Optional[str]) -> bool: """True when pr_author is the base-branch author or in base maintainers. @@ -34,9 +41,7 @@ def author_has_plugin_permission(pr_author: str, base_json_text: Optional[str]) data = json.loads(base_json_text) except json.JSONDecodeError: return False - author = data.get("author") or "" - maintainers = [str(m) for m in (data.get("maintainers") or []) if m is not None] - return pr_author == author or pr_author in maintainers + return author_in_plugin_json(pr_author, data) @dataclass @@ -50,37 +55,37 @@ def check_blacklists(pr_author: str, plugins: list[str], """Case-insensitive author/plugin blacklist check (whitespace-trimmed entries).""" author_bl = (author_blacklist or "").strip() plugin_bl = (plugin_blacklist or "").strip() - if not author_bl and not plugin_bl: - return BlacklistResult() - if author_bl: - for entry in author_bl.split(","): - slug = entry.replace(" ", "") - if pr_author.lower() == slug.lower(): - actions.warning(f"PR author '{pr_author}' is on the author blacklist.") - return BlacklistResult(True, "author-blacklisted") + if author_bl and _csv_contains(pr_author, author_bl): + actions.warning(f"PR author '{pr_author}' is on the author blacklist.") + return BlacklistResult(True, "author-blacklisted") if plugin_bl: - blocked = plugin_bl.split(",") for plugin in plugins: - if not plugin: - continue - for entry in blocked: - slug = entry.replace(" ", "") - if plugin.lower() == slug.lower(): - actions.warning(f"Plugin '{plugin}' is on the plugin blacklist.") - return BlacklistResult(True, "plugin-blacklisted") + if plugin and _csv_contains(plugin, plugin_bl): + actions.warning(f"Plugin '{plugin}' is on the plugin blacklist.") + return BlacklistResult(True, "plugin-blacklisted") return BlacklistResult() +def _csv_contains(value: str, csv: str) -> bool: + """Case-insensitive membership in a comma-separated list (spaces stripped).""" + return any(value.lower() == entry.replace(" ", "").lower() for entry in csv.split(",")) + + def run(pr_author: str, base_ref: str, head_ref: str = "", author_blacklist: str = "", plugin_blacklist: str = "", repo: str = "") -> int: """Execute detection; write outputs; return process exit code (0 ok, 1 hard block).""" owner, _, name = repo.partition("/") + cached_write_access: Optional[bool] = None def write_access() -> bool: - return gh.has_write_access(owner, name, pr_author) + """``gh api`` permission lookup, resolved at most once per run.""" + nonlocal cached_write_access + if cached_write_access is None: + cached_write_access = gh.has_write_access(owner, name, pr_author) + return cached_write_access # Bot-authored yank rollback PRs bypass plugin validation entirely. if pr_author.endswith("[bot]") and head_ref.startswith("yank/"): @@ -147,20 +152,19 @@ def write_access() -> bool: close_pr = True close_reason = bl.reason - matrix_json = json.dumps(plugin_list, separators=(",", ":")) - actions.set_output("matrix", matrix_json) - actions.set_output("plugin_count", str(plugin_count)) - actions.set_output("close_pr", "true" if close_pr else "false") - actions.set_output("skip_validation", "false") - actions.set_output("outside_violation", "true" if has_outside_violation else "false") - actions.set_output("close_reason", close_reason) + actions.set_outputs(matrix=json.dumps(plugin_list, separators=(",", ":")), + plugin_count=str(plugin_count), + close_pr="true" if close_pr else "false", + skip_validation="false", + outside_violation="true" if has_outside_violation else "false", + close_reason=close_reason) if outside_changes: actions.set_output("outside_files", "\n".join(outside_changes)) pub_key_changed = is_maintainer and PUB_KEY_PATH in outside_changes - actions.set_output("pub_key_changed", "true" if pub_key_changed else "false") - actions.set_output("has_new_plugin", "true" if has_new_plugin else "false") - actions.set_output("has_updated_plugin", "true" if has_updated_plugin else "false") + actions.set_outputs(pub_key_changed="true" if pub_key_changed else "false", + has_new_plugin="true" if has_new_plugin else "false", + has_updated_plugin="true" if has_updated_plugin else "false") actions.log(f"Detected {plugin_count} plugin(s): {' '.join(plugin_list)}") actions.log(f"close_pr={'true' if close_pr else 'false'}") diff --git a/.github/pluginctl/src/pluginctl/validate/plugin.py b/.github/pluginctl/src/pluginctl/validate/plugin.py index 965951d..bfd32de 100644 --- a/.github/pluginctl/src/pluginctl/validate/plugin.py +++ b/.github/pluginctl/src/pluginctl/validate/plugin.py @@ -19,6 +19,7 @@ from ..core import actions, gh, git from ..core.version import is_semver, is_dispatcharr_version, version_greater_than +from .detect import author_in_plugin_json, is_safe_name METADATA_ONLY_FIELDS = [ "description", "repo_url", "discord_thread", "min_dispatcharr_version", @@ -26,7 +27,6 @@ ] _GH_DOWNLOAD_RE = re.compile(r"^https://github\.com/([^/]+)/([^/]+)/releases/download/([^/]+)/") -_GH_DOWNLOAD_TAG_RE = re.compile(r"^https://github\.com/[^/]+/[^/]+/releases/download/([^/]+)/") SPDX_URL = "https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json" @@ -137,7 +137,7 @@ def emit_fragment() -> None: _write(output_file, text) # Folder name format - if not re.match(r"^[a-z0-9]+(-[a-z0-9]+)*$", plugin_name): + if not is_safe_name(plugin_name): rows.append(f"| Folder name | ❌ | Must be lowercase-kebab-case - got " f"`{plugin_name}`, e.g. `my-plugin-name` |") failed = True @@ -221,9 +221,9 @@ def emit_fragment() -> None: release_link = f"https://github.com/{gh_owner}/{gh_repo}/releases/tag/{gh_tag}" if old_version and old_source_url_tmpl: old_resolved = old_source_url_tmpl.replace("{version}", old_version) - om = _GH_DOWNLOAD_TAG_RE.match(old_resolved) + om = _GH_DOWNLOAD_RE.match(old_resolved) if om: - old_gh_tag = om.group(1) + old_gh_tag = om.group(3) compare_link = (f"https://github.com/{gh_owner}/{gh_repo}" f"/compare/{old_gh_tag}...{gh_tag}") else: @@ -275,9 +275,7 @@ def emit_fragment() -> None: rows.append("| Permission | ✅ | You have permission to modify this plugin |") has_permission = True elif base_raw is not None: - base_author = jq_r(base_raw.get("author"), "") - base_maintainers = [str(m) for m in (base_raw.get("maintainers") or []) if m is not None] - if pr_author == base_author or pr_author in base_maintainers: + if author_in_plugin_json(pr_author, base_raw): rows.append("| Permission | ✅ | You have permission to modify this plugin |") has_permission = True else: @@ -325,15 +323,17 @@ def emit_fragment() -> None: # Dispatcharr version constraints min_da = jq_r(raw.get("min_dispatcharr_version"), "") max_da = jq_r(raw.get("max_dispatcharr_version"), "") - if min_da and not is_dispatcharr_version(min_da): + min_ok = is_dispatcharr_version(min_da) + max_ok = is_dispatcharr_version(max_da) + if min_da and not min_ok: rows.append(f"| `min_dispatcharr_version` | ❌ | `{min_da}` is not valid semver - " "expected `X.Y.Z` or `vX.Y.Z` |") failed = True - if max_da and not is_dispatcharr_version(max_da): + if max_da and not max_ok: rows.append(f"| `max_dispatcharr_version` | ❌ | `{max_da}` is not valid semver - " "expected `X.Y.Z` or `vX.Y.Z` |") failed = True - if min_da and max_da and is_dispatcharr_version(max_da) and is_dispatcharr_version(min_da): + if min_da and max_da and max_ok and min_ok: _max = max_da[1:] if max_da.startswith("v") else max_da _min = min_da[1:] if min_da.startswith("v") else min_da if not version_greater_than(_max, _min) and _max != _min: @@ -364,10 +364,10 @@ def emit_fragment() -> None: jq_r(raw.get("name"), ""), jq_r(raw.get("version"), ""), jq_r(raw.get("description"), ""), - jq_r(raw.get("author"), ""), + author, ", ".join(maintainers), - jq_r(raw.get("repo_url"), ""), - jq_r(raw.get("discord_thread"), ""), + repo_url, + discord_thread, ] meta_row = f"" diff --git a/.github/pluginctl/src/pluginctl/validate/report.py b/.github/pluginctl/src/pluginctl/validate/report.py index 038a98d..f72a187 100644 --- a/.github/pluginctl/src/pluginctl/validate/report.py +++ b/.github/pluginctl/src/pluginctl/validate/report.py @@ -18,6 +18,13 @@ MARKER = "" +# close_reason values for which the PR itself is closed (vs. merely reported on). +CLOSING_REASONS = ("unauthorized", "author-blacklisted", "plugin-blacklisted") + + +def closes_pr(close_pr: bool, close_reason: str) -> bool: + return close_pr and close_reason in CLOSING_REASONS + @dataclass class ParsedFragments: @@ -414,7 +421,7 @@ def run(pr_number: str, pr_author: str, plugin_count: str, close_pr: bool, _emit_webhook(pr_number, pr_author, close_pr, close_reason, parsed) - if close_pr and close_reason in ("unauthorized", "author-blacklisted", "plugin-blacklisted"): + if closes_pr(close_pr, close_reason): gh.pr_close(pr_number) actions.log(f"PR #{pr_number} closed: unauthorized") return comment_exit @@ -422,7 +429,7 @@ def run(pr_number: str, pr_author: str, plugin_count: str, close_pr: bool, def _emit_webhook(pr_number, pr_author, close_pr, close_reason, parsed) -> None: from ..integrations import webhooks - if close_pr and close_reason in ("unauthorized", "author-blacklisted", "plugin-blacklisted"): + if closes_pr(close_pr, close_reason): webhooks.emit("pr.closed_unauthorized", webhooks.pr_closed_unauthorized(pr_number, pr_author, close_reason)) return diff --git a/.github/pluginctl/src/pluginctl/validate/sarif.py b/.github/pluginctl/src/pluginctl/validate/sarif.py index 04c7fad..6c9b7ff 100644 --- a/.github/pluginctl/src/pluginctl/validate/sarif.py +++ b/.github/pluginctl/src/pluginctl/validate/sarif.py @@ -102,9 +102,9 @@ def classify(sarif_objs: list[dict]) -> Counts: for result in (run.get("results") or []): counts.total += 1 sev = severity_of(result, secmap) - if sev >= 7.0: + if is_blocking(sev): counts.blocking += 1 - elif 6.0 <= sev < 7.0: + elif is_medium(sev): counts.medium += 1 else: counts.low += 1 @@ -223,8 +223,8 @@ def run(results_dir: str, repo: str, sha: str, matrix: list[str], actions.set_output("codeql_mediums", str(counts.medium)) actions.set_output("codeql_lows", str(counts.low)) - if counts.blocking > 0: - _write(results_dir and "codeql-findings.md", + if counts.blocking > 0 and results_dir: + _write("codeql-findings.md", findings_table(objs, is_blocking, repo, sha, external_prefixes)) if counts.medium > 0: _write("codeql-medium-findings.md", @@ -253,7 +253,5 @@ def run(results_dir: str, repo: str, sha: str, matrix: list[str], def _write(path: str, content: str) -> None: - if not path: - return with open(path, "w", encoding="utf-8") as fh: fh.write(content) diff --git a/.github/pluginctl/tests/test_integrations.py b/.github/pluginctl/tests/test_integrations.py index 05455a0..5df9d0c 100644 --- a/.github/pluginctl/tests/test_integrations.py +++ b/.github/pluginctl/tests/test_integrations.py @@ -1,5 +1,5 @@ +from pluginctl.core import version from pluginctl.integrations import automerge, external_readme -from pluginctl.publish import cleanup # ---- automerge label gate ---- @@ -54,7 +54,7 @@ def test_rewrite_links(): assert "[anchor](#section)" in out # anchors untouched -# ---- cleanup tag selection ---- +# ---- release tag selection ---- def test_versioned_tags_sorted_desc_excludes_latest(): tags = ["demo-1.0.0", "demo-1.10.0", "demo-1.2.0", "demo-latest", "other-1.0.0"] - assert cleanup._versioned_tags(tags, "demo") == ["demo-1.10.0", "demo-1.2.0", "demo-1.0.0"] + assert version.versioned_tags(tags, "demo") == ["demo-1.10.0", "demo-1.2.0", "demo-1.0.0"] From 1cfe183c347c7ab3930fb09b37463fd6ac3c835f Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Sat, 25 Jul 2026 13:02:21 -0400 Subject: [PATCH 6/6] Close two PR-validation trust-boundary gaps found in security review An unsafe-named plugins/ folder was silently dropped from the scan matrix instead of blocking the PR, letting it ride along unscanned to merge and publish alongside a legitimate change. detect.py now closes the PR (unsafe-plugin-name) instead, and zips.py adds the same allowlist check as defense in depth before publishing. Separately, a plugin.json description containing an embedded newline could smuggle in a forged line ahead of the genuine one, letting report.parse_fragments render attacker-chosen repo/discord links into the bot's PR comment. parse_fragments now anchors on the last marker line, and plugin.py collapses description to a single line before embedding it. --- .../pluginctl/src/pluginctl/publish/zips.py | 4 ++ .../src/pluginctl/validate/detect.py | 17 ++++++++- .../src/pluginctl/validate/plugin.py | 12 +++++- .../src/pluginctl/validate/report.py | 37 +++++++++++-------- .github/pluginctl/tests/test_detect.py | 37 +++++++++++++++++++ .github/pluginctl/tests/test_report.py | 20 ++++++++++ .github/pluginctl/tests/test_validate.py | 23 ++++++++++++ .../pluginctl/tests/test_zips_safe_name.py | 31 ++++++++++++++++ 8 files changed, 163 insertions(+), 18 deletions(-) create mode 100644 .github/pluginctl/tests/test_zips_safe_name.py diff --git a/.github/pluginctl/src/pluginctl/publish/zips.py b/.github/pluginctl/src/pluginctl/publish/zips.py index f4a2bde..2803a48 100644 --- a/.github/pluginctl/src/pluginctl/publish/zips.py +++ b/.github/pluginctl/src/pluginctl/publish/zips.py @@ -23,6 +23,7 @@ from ..core.hashing import file_digests from ..core.jsonio import drop_none from ..core.timeutil import now_iso +from ..validate.detect import is_safe_name def _download(url: str, dest: str, attempts: int = 3) -> bool: @@ -56,6 +57,9 @@ def run(source_branch: str, repository: str, build_meta_dir: str) -> int: for plugin_dir in sorted(glob.glob("plugins/*/")): plugin_name = os.path.basename(plugin_dir.rstrip("/")) + if not is_safe_name(plugin_name): + actions.warning(f"Skipping publish for unsafe plugin folder name: '{plugin_name}'") + continue plugin_key = plugin_name.replace("-", "_") with open(os.path.join(plugin_dir, "plugin.json"), encoding="utf-8") as fh: raw = json.load(fh) diff --git a/.github/pluginctl/src/pluginctl/validate/detect.py b/.github/pluginctl/src/pluginctl/validate/detect.py index 69f23ce..9572b7f 100644 --- a/.github/pluginctl/src/pluginctl/validate/detect.py +++ b/.github/pluginctl/src/pluginctl/validate/detect.py @@ -107,15 +107,28 @@ def write_access() -> bool: if not plugin_list: return _no_plugins(pr_author, outside_changes, has_outside_violation, write_access) - # Allowlist: only safe kebab-case names enter the matrix. + # Allowlist: only safe kebab-case names enter the matrix. A folder with an + # unsafe name is never scanned (ClamAV/CodeQL/publish all iterate the safe + # matrix), so its presence must block the whole PR rather than being + # silently dropped - otherwise it can ride along with a legitimate, + # passing plugin change straight to `main` and publish unscanned. safe_list: list[str] = [] + unsafe_list: list[str] = [] for plugin in plugin_list: if is_safe_name(plugin): safe_list.append(plugin) else: - actions.warning(f"Skipping plugin with unsafe folder name: '{plugin}'") + unsafe_list.append(plugin) + actions.warning(f"Unsafe plugin folder name: '{plugin}'") plugin_list = safe_list + if unsafe_list: + actions.error(f"Unsafe plugin folder name(s) detected: {', '.join(unsafe_list)}") + actions.set_outputs(close_pr="true", close_reason="unsafe-plugin-name", + plugin_count="0", matrix="[]", + has_new_plugin="false", has_updated_plugin="false") + return 0 + if not plugin_list: actions.error("No valid plugin changes detected in this PR.") actions.set_outputs(close_pr="true", close_reason="no-valid-plugins", diff --git a/.github/pluginctl/src/pluginctl/validate/plugin.py b/.github/pluginctl/src/pluginctl/validate/plugin.py index bfd32de..328716a 100644 --- a/.github/pluginctl/src/pluginctl/validate/plugin.py +++ b/.github/pluginctl/src/pluginctl/validate/plugin.py @@ -50,6 +50,12 @@ def field_present(raw: dict, key: str) -> bool: return not (val is None or val is False) +def _single_line(s: str) -> str: + """Collapse newlines/carriage returns so a free-text field can't inject + extra fragment lines (e.g. a forged ```` marker).""" + return s.replace("\r\n", " ").replace("\n", " ").replace("\r", " ") + + def tsv(fields: list[str]) -> str: """Reproduce ``@tsv`` escaping for a single row.""" def esc(s: str) -> str: @@ -115,7 +121,11 @@ def run(plugin_name: str, pr_author: str, base_ref: str, output_file: str, except (OSError, json.JSONDecodeError): raw = None if raw is not None: - desc = jq_r(raw.get("description"), "") + # Free-text fields are attacker-controlled. Collapse newlines before + # embedding them in the fragment so a value can never introduce a + # forged "" marker line ahead of the real one + # that `report.parse_fragments` looks for. + desc = _single_line(jq_r(raw.get("description"), "")) repo_url = jq_r(raw.get("repo_url"), "") if desc: pre_lines.append(f"_{desc}_") diff --git a/.github/pluginctl/src/pluginctl/validate/report.py b/.github/pluginctl/src/pluginctl/validate/report.py index f72a187..83490ae 100644 --- a/.github/pluginctl/src/pluginctl/validate/report.py +++ b/.github/pluginctl/src/pluginctl/validate/report.py @@ -48,23 +48,30 @@ def parse_fragments(fragments_dir: str) -> ParsedFragments: content = _read(fragment) if "❌" in content: result.any_failed = True - # META_ROW extraction - for line in content.splitlines(): + # META_ROW extraction. plugin.py always appends the genuine marker as + # the fragment's last line, so anchor here rather than taking the + # first match - a free-text field (e.g. description) earlier in the + # fragment must never be able to forge this by embedding its own + # ""): - meta = meta[:-3] - fields = meta.split("\t") - fields += [""] * (7 - len(fields)) - f_name, f_version, f_desc, f_author, f_maint, f_repo, f_discord = fields[:7] - if f_repo or f_discord: - result.plugin_links += f"**`{f_name}`:**\n" - if f_repo: - result.plugin_links += f"- [GitHub Repository]({f_repo})\n" - if f_discord: - result.plugin_links += f"- [Discord Thread]({f_discord})\n" - result.plugin_links += "\n" + meta_line = line break + if meta_line is not None: + meta = meta_line[len(""): + meta = meta[:-3] + fields = meta.split("\t") + fields += [""] * (7 - len(fields)) + f_name, f_version, f_desc, f_author, f_maint, f_repo, f_discord = fields[:7] + if f_repo or f_discord: + result.plugin_links += f"**`{f_name}`:**\n" + if f_repo: + result.plugin_links += f"- [GitHub Repository]({f_repo})\n" + if f_discord: + result.plugin_links += f"- [Discord Thread]({f_discord})\n" + result.plugin_links += "\n" # visible = fragment minus META_ROW lines, trailing whitespace stripped ($() semantics) visible_lines = [ln for ln in content.splitlines() if not ln.startswith(""-prefixed line must not be able to spoof the + Plugin Contact Links - only the genuine, always-last marker counts.""" + f = tmp_path / "demo.fragment.md" + f.write_text( + "### Plugin: `demo`\n\n" + "_Cool plugin_\n" + "\n" + "\n| ok |\n" + "\n", + encoding="utf-8", + ) + parsed = report.parse_fragments(str(tmp_path)) + assert "evil.example" not in parsed.plugin_links + assert "discord.gg/evil" not in parsed.plugin_links + assert "**`Demo`:**" in parsed.plugin_links + assert "- [GitHub Repository](https://real.example)" in parsed.plugin_links assert parsed.any_failed is False diff --git a/.github/pluginctl/tests/test_validate.py b/.github/pluginctl/tests/test_validate.py index 1ae6ec4..6a0bba7 100644 --- a/.github/pluginctl/tests/test_validate.py +++ b/.github/pluginctl/tests/test_validate.py @@ -107,3 +107,26 @@ def test_unauthorized_author_new_plugin(plugin_repo): text = frag.read_text(encoding="utf-8") assert '| Permission | ❌ | Add `"author": "mallory"` to plugin.json |' in text assert _outputs(out)["has_permission"] == "false" + + +def test_description_newline_cannot_forge_meta_row(plugin_repo): + """A description embedding its own ""-prefixed line + must not produce that line in the fragment - only the genuine, always-last + marker plugin.py itself appends should ever start with that prefix.""" + tmp_path, out = plugin_repo + pj = tmp_path / "plugins" / "demo-plugin" / "plugin.json" + data = json.loads(pj.read_text()) + data["description"] = ("Cool plugin\n") + pj.write_text(json.dumps(data), encoding="utf-8") + frag = tmp_path / "frag.md" + validate.run("demo-plugin", "alice", "main", str(frag), repo="org/repo") + lines = frag.read_text(encoding="utf-8").splitlines() + meta_lines = [ln for ln in lines if ln.startswith("")].split("\t") + f_repo, f_discord = fields[5], fields[6] + assert "evil.example" not in f_repo + assert "discord.gg/evil" not in f_discord diff --git a/.github/pluginctl/tests/test_zips_safe_name.py b/.github/pluginctl/tests/test_zips_safe_name.py new file mode 100644 index 0000000..fc55ea9 --- /dev/null +++ b/.github/pluginctl/tests/test_zips_safe_name.py @@ -0,0 +1,31 @@ +import json + +from pluginctl.core import gh +from pluginctl.publish import zips + + +def test_run_skips_unsafe_named_plugin_directory(tmp_path, monkeypatch): + """publish/zips.run() must never build/upload a plugin folder whose name + failed the validate-time safe-name allowlist, even if it somehow reached + `main` (defense in depth alongside the detect.py gate).""" + monkeypatch.chdir(tmp_path) + + safe_dir = tmp_path / "plugins" / "good-plugin" + safe_dir.mkdir(parents=True) + (safe_dir / "plugin.json").write_text(json.dumps({ + "name": "Good Plugin", "version": "1.0.0", + }), encoding="utf-8") + + # Unsafe-named folder with no plugin.json - if run() ever tried to process + # it (instead of skipping via the safe-name check), this would raise + # FileNotFoundError and fail the test. + unsafe_dir = tmp_path / "plugins" / "Bad_Plugin" + unsafe_dir.mkdir(parents=True) + + monkeypatch.setattr(gh, "release_exists", lambda *a, **k: True) + + rc = zips.run("main", "org/repo", str(tmp_path / "build_meta")) + + assert rc == 0 + changed = (tmp_path / "changed_plugins.txt").read_text(encoding="utf-8") + assert "Bad_Plugin" not in changed