diff --git a/.github/workflows/preflight.yml b/.github/workflows/preflight.yml index f8b853f9..558b23c6 100644 --- a/.github/workflows/preflight.yml +++ b/.github/workflows/preflight.yml @@ -74,6 +74,11 @@ jobs: src: - added|modified: 'alws/**/*.py' - added|modified: 'scripts/**/*.py' + code: + - added|modified: '**' + - '!**/*.yml' + - '!**/*.yaml' + - '!**/*.md' - name: Prepare GPG key run: | @@ -120,6 +125,7 @@ jobs: # Run pytest ignoring tests/test_oval due to inability to clone oval-processor - name: Run pytest + if: ${{ steps.changed-files.outputs.code == 'true' }} run: | docker compose run --rm web_server_tests bash -o pipefail -c " pytest -v --ignore tests/test_oval --cov \ @@ -128,6 +134,7 @@ jobs: --cov-report=term | tee $REPORTS_DIR/pytest-output.txt" - name: Check migrations + if: ${{ steps.changed-files.outputs.code == 'true' }} run: docker compose run --rm web_server_tests alembic --config tests/test-alembic.ini upgrade head - name: Stop services @@ -175,3 +182,45 @@ jobs: run: | cat $REPORTS_DIR/{coverage,pytest,pylint,black,isort,bandit}-report.md \ > $GITHUB_STEP_SUMMARY 2>/dev/null || true + + check-reference-repos: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + + - name: Check out repository + uses: actions/checkout@v4 + + - name: Detect changes + uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + relevant: + - 'reference_data/**' + - 'scripts/check_repos_existence.py' + - '.github/workflows/preflight.yml' + + - name: Set up Python + if: steps.changes.outputs.relevant == 'true' + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + if: steps.changes.outputs.relevant == 'true' + run: pip install requests PyYAML + + - name: Check remote_url reachability + if: steps.changes.outputs.relevant == 'true' + run: | + status=0 + shopt -s nullglob + for f in reference_data/*.yml reference_data/*.yaml; do + if grep -q '^\s*remote_url:' "$f"; then + echo "::group::$f" + python scripts/check_repos_existence.py "$f" || status=1 + echo "::endgroup::" + fi + done + exit $status diff --git a/scripts/check_repos_existence.py b/scripts/check_repos_existence.py new file mode 100644 index 00000000..5a6e9954 --- /dev/null +++ b/scripts/check_repos_existence.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Check that every remote_url in a reference_data YAML is reachable.""" +import argparse +import sys +from concurrent.futures import ThreadPoolExecutor, as_completed + +import requests +import yaml + + +def collect_repos(data): + repos = [] + for platform in data: + for repo in platform.get('repositories', []) or []: + url = repo.get('remote_url') + if url: + repos.append((repo.get('name', '?'), repo.get('arch', '?'), url)) + return repos + + +def check(url, timeout=15): + try: + r = requests.head(url, allow_redirects=True, timeout=timeout) + if r.status_code in (405, 403): + r = requests.get(url, stream=True, timeout=timeout) + return r.status_code + except requests.RequestException as e: + return f'ERR: {e.__class__.__name__}' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('yaml_file') + parser.add_argument('--workers', type=int, default=16) + args = parser.parse_args() + + with open(args.yaml_file) as f: + data = yaml.safe_load(f) + + repos = collect_repos(data) + print(f'Checking {len(repos)} repositories...\n') + + missing = [] + with ThreadPoolExecutor(max_workers=args.workers) as pool: + futures = {pool.submit(check, url): (name, arch, url) for name, arch, url in repos} + for fut in as_completed(futures): + name, arch, url = futures[fut] + status = fut.result() + ok = status == 200 + marker = 'OK ' if ok else 'BAD' + print(f'[{marker}] {status} {arch:10s} {name:40s} {url}') + if not ok: + missing.append((name, arch, url, status)) + + print(f'\n{len(missing)} unreachable repos:') + for name, arch, url, status in missing: + print(f' {status} {arch} {name} {url}') + + sys.exit(1 if missing else 0) + + +if __name__ == '__main__': + main()