Add a check for verifying repositories existence

.github/workflows/preflight.yml

@@ -74,6 +74,11 @@ jobs:
             src:
               - added|modified: 'alws/**/*.py'
               - added|modified: 'scripts/**/*.py'
+            code:
+              - added|modified: '**'
+              - '!**/*.yml'
+              - '!**/*.yaml'
+              - '!**/*.md'
 
       - name: Prepare GPG key
         run: |
@@ -120,6 +125,7 @@ jobs:
 
       # Run pytest ignoring tests/test_oval due to inability to clone oval-processor
       - name: Run pytest
+        if: ${{ steps.changed-files.outputs.code == 'true' }}
         run: |
           docker compose run --rm web_server_tests bash -o pipefail -c "
             pytest -v --ignore tests/test_oval --cov \
@@ -128,6 +134,7 @@ jobs:
               --cov-report=term | tee $REPORTS_DIR/pytest-output.txt"
 
       - name: Check migrations
+        if: ${{ steps.changed-files.outputs.code == 'true' }}
         run: docker compose run --rm web_server_tests alembic --config tests/test-alembic.ini upgrade head
 
       - name: Stop services
@@ -175,3 +182,45 @@ jobs:
         run: |
           cat $REPORTS_DIR/{coverage,pytest,pylint,black,isort,bandit}-report.md \
             > $GITHUB_STEP_SUMMARY 2>/dev/null || true
+
+  check-reference-repos:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Detect changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          filters: |
+            relevant:
+              - 'reference_data/**'
+              - 'scripts/check_repos_existence.py'
+              - '.github/workflows/preflight.yml'
+
+      - name: Set up Python
+        if: steps.changes.outputs.relevant == 'true'
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        if: steps.changes.outputs.relevant == 'true'
+        run: pip install requests PyYAML
+
+      - name: Check remote_url reachability
+        if: steps.changes.outputs.relevant == 'true'
+        run: |
+          status=0
+          shopt -s nullglob
+          for f in reference_data/*.yml reference_data/*.yaml; do
+            if grep -q '^\s*remote_url:' "$f"; then
+              echo "::group::$f"
+              python scripts/check_repos_existence.py "$f" || status=1
+              echo "::endgroup::"
+            fi
+          done
+          exit $status

scripts/check_repos_existence.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Check that every remote_url in a reference_data YAML is reachable."""
+import argparse
+import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import requests
+import yaml
+
+
+def collect_repos(data):
+    repos = []
+    for platform in data:
+        for repo in platform.get('repositories', []) or []:
+            url = repo.get('remote_url')
+            if url:
+                repos.append((repo.get('name', '?'), repo.get('arch', '?'), url))
+    return repos
+
+
+def check(url, timeout=15):
+    try:
+        r = requests.head(url, allow_redirects=True, timeout=timeout)
+        if r.status_code in (405, 403):
+            r = requests.get(url, stream=True, timeout=timeout)
+        return r.status_code
+    except requests.RequestException as e:
+        return f'ERR: {e.__class__.__name__}'
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('yaml_file')
+    parser.add_argument('--workers', type=int, default=16)
+    args = parser.parse_args()
+
+    with open(args.yaml_file) as f:
+        data = yaml.safe_load(f)
+
+    repos = collect_repos(data)
+    print(f'Checking {len(repos)} repositories...\n')
+
+    missing = []
+    with ThreadPoolExecutor(max_workers=args.workers) as pool:
+        futures = {pool.submit(check, url): (name, arch, url) for name, arch, url in repos}
+        for fut in as_completed(futures):
+            name, arch, url = futures[fut]
+            status = fut.result()
+            ok = status == 200
+            marker = 'OK ' if ok else 'BAD'
+            print(f'[{marker}] {status} {arch:10s} {name:40s} {url}')
+            if not ok:
+                missing.append((name, arch, url, status))
+
+    print(f'\n{len(missing)} unreachable repos:')
+    for name, arch, url, status in missing:
+        print(f'  {status} {arch} {name} {url}')
+
+    sys.exit(1 if missing else 0)
+
+
+if __name__ == '__main__':
+    main()