feat: add monitoring to cronjobs

MarceloRobert · MarceloRobert · commit dcbc86a4e008 · 2026-04-07T16:00:27.000-03:00
Closes #1792
diff --git a/.env.example b/.env.example
@@ -65,6 +65,18 @@ PROMETHEUS_METRICS_ENABLED=False
 # This maps external port 2001 to the internal Prometheus metrics port
 EXTERNAL_PROM_METRICS_PORT=2001
 
+# -----------------------------------------------------------------------------
+# Notifications monitoring (optional)
+# -----------------------------------------------------------------------------
+# Healthcheck.io private tokens/UUIDs used by cronjob monitoring.
+# The public base URL is defined in code and not stored in env.
+HEALTHCHECK_ID_DELETE_UNUSED_HARDWARE_STATUS=
+HEALTHCHECK_ID_NOTIFICATIONS_HARDWARE_SUMMARY=
+HEALTHCHECK_ID_NOTIFICATIONS_METRICS_SUMMARY=
+HEALTHCHECK_ID_NOTIFICATIONS_NEW_ISSUES=
+HEALTHCHECK_ID_NOTIFICATIONS_SUMMARY_MICROSOFT=
+HEALTHCHECK_ID_NOTIFICATIONS_SUMMARY_MAESTRO=
+
 # -----------------------------------------------------------------------------
 # Email / Notifications (optional)
 # -----------------------------------------------------------------------------
diff --git a/backend/kernelCI/settings.py b/backend/kernelCI/settings.py
@@ -124,6 +124,29 @@ def get_json_env_var(name, default):
 # To run cronjobs locally, execute
 # poetry run ./manage.py crontab arg
 # where "arg" is add, remove or show
+
+HEALTHCHECK_MONITORING_PATH_MAP: dict[str, str] = {
+    "delete_unused_hardware_status": os.environ.get(
+        "HEALTHCHECK_ID_DELETE_UNUSED_HARDWARE_STATUS", ""
+    ),
+    "notifications_hardware_summary": os.environ.get(
+        "HEALTHCHECK_ID_NOTIFICATIONS_HARDWARE_SUMMARY", ""
+    ),
+    "notifications_metrics_summary": os.environ.get(
+        "HEALTHCHECK_ID_NOTIFICATIONS_METRICS_SUMMARY", ""
+    ),
+    "notifications_new_issues": os.environ.get(
+        "HEALTHCHECK_ID_NOTIFICATIONS_NEW_ISSUES", ""
+    ),
+    "notifications_summary_microsoft": os.environ.get(
+        "HEALTHCHECK_ID_NOTIFICATIONS_SUMMARY_MICROSOFT", ""
+    ),
+    "notifications_summary_maestro": os.environ.get(
+        "HEALTHCHECK_ID_NOTIFICATIONS_SUMMARY_MAESTRO", ""
+    ),
+}
+"""Maps monitoring_id to the relative_path that will be appended to the base healthcheck URL."""
+
 SKIP_CRONJOBS = is_boolean_or_string_true(os.environ.get("SKIP_CRONJOBS", False))
 if SKIP_CRONJOBS:
     CRONJOBS = []
@@ -133,12 +156,15 @@ def get_json_env_var(name, default):
         "CRONTAB_COMMAND_SUFFIX", ">> /proc/1/fd/1 2>&1"
     )
     CRONJOBS = [
+        # not using a monitoring_id in the first task since it should
+        # be removed once the denormalization is set in stone
         ("0 * * * *", "kernelCI_app.tasks.update_checkout_cache"),
         (
             "59 * * * *",
             "django.core.management.call_command",
             [
                 "notifications",
+                "--monitoring-id=notifications_new_issues",
                 "--action=new_issues",
                 "--to=kernelci-results@groups.io",
                 "--cc=gus@collabora.com",
@@ -151,6 +177,7 @@ def get_json_env_var(name, default):
             "django.core.management.call_command",
             [
                 "notifications",
+                "--monitoring-id=notifications_summary_microsoft",
                 "--action=summary",
                 "--to=kernelcialerts@microsoft.com",
                 "--cc=kernelci-results@groups.io",
@@ -165,6 +192,7 @@ def get_json_env_var(name, default):
             "django.core.management.call_command",
             [
                 "notifications",
+                "--monitoring-id=notifications_summary_maestro",
                 "--action=summary",
                 "--add-mailing-lists",
                 "--send",
@@ -177,6 +205,7 @@ def get_json_env_var(name, default):
             "django.core.management.call_command",
             [
                 "notifications",
+                "--monitoring-id=notifications_hardware_summary",
                 "--action=hardware_summary",
                 "--cc=kernelci-results@groups.io",
                 "--send",
@@ -188,13 +217,15 @@ def get_json_env_var(name, default):
             "django.core.management.call_command",
             [
                 "delete_unused_hardware_status",
+                "--monitoring-id=delete_unused_hardware_status",
             ],
         ),
         (
             "0 0 * * 6",
             "django.core.management.call_command",
             [
                 "notifications",
+                "--monitoring-id=notifications_metrics_summary",
                 "--action=metrics_summary",
                 "--to=kernelci@lists.linux.dev",
                 "--cc=kernelci-results@groups.io",
diff --git a/backend/kernelCI_app/management/commands/delete_unused_hardware_status.py b/backend/kernelCI_app/management/commands/delete_unused_hardware_status.py
@@ -7,6 +7,10 @@
 import logging
 from django.core.management.base import BaseCommand
 from django.db import transaction
+from kernelCI_app.management.commands.helpers.healthcheck import (
+    MONITORING_ID_PARAM_HELP_TEXT,
+    run_with_healthcheck_monitoring,
+)
 from kernelCI_app.models import HardwareStatus, LatestCheckout, ProcessedListingItems
 
 logger = logging.getLogger(__name__)
@@ -30,8 +34,21 @@ def add_arguments(self, parser):
             default=10000,
             help="Number of records to delete per batch (default: 10000)",
         )
+        parser.add_argument(
+            "--monitoring-id",
+            type=str,
+            default=None,
+            help=MONITORING_ID_PARAM_HELP_TEXT,
+        )
 
     def handle(self, *args, **options):
+        monitoring_id = options.get("monitoring_id")
+        return run_with_healthcheck_monitoring(
+            monitoring_id=monitoring_id,
+            action=lambda: self._run_action(options),
+        )
+
+    def _run_action(self, options):
         dry_run = options["dry_run"]
         batch_size = options["batch_size"]
 
diff --git a/backend/kernelCI_app/management/commands/helpers/healthcheck.py b/backend/kernelCI_app/management/commands/helpers/healthcheck.py
@@ -0,0 +1,71 @@
+from collections.abc import Callable
+from typing import Any
+
+from django.conf import settings
+
+import requests
+
+from kernelCI_app.helpers.logger import out
+
+MONITORING_ID_PARAM_HELP_TEXT = (
+    "Monitoring ID configured in settings for healthcheck.io pings "
+    "(optional, used only for monitoring the command execution over time)"
+)
+HEALTHCHECK_BASE_URL = "https://hc-ping.com"
+
+
+def _resolve_monitoring_url(monitoring_id: str) -> str | None:
+    monitoring_path_map: dict[str, str] = settings.HEALTHCHECK_MONITORING_PATH_MAP
+    monitoring_path = monitoring_path_map.get(monitoring_id)
+
+    if not monitoring_path:
+        return None
+
+    return f"{HEALTHCHECK_BASE_URL.rstrip('/')}/{monitoring_path.lstrip('/')}"
+
+
+def _ping_healthcheck(monitoring_id: str, status: str) -> None:
+    monitoring_url = _resolve_monitoring_url(monitoring_id)
+    if not monitoring_url:
+        out(
+            "No healthcheck URL configured for monitoring_id='%s', skipping %s ping."
+            % (monitoring_id, status)
+        )
+        return
+
+    # Success just needs to ping base healthcheck.io url + uuid, no subpath
+    monitoring_status_url = (
+        f"{monitoring_url.rstrip('/')}/{status.lstrip('/')}"
+        if status != "success"
+        else monitoring_url
+    )
+    try:
+        response = requests.get(monitoring_status_url, timeout=10)
+        response.raise_for_status()
+        out(
+            "Success at pinging healthcheck '%s' with monitoring_id '%s'"
+            % (monitoring_status_url, monitoring_id)
+        )
+    except requests.RequestException as e:
+        out(
+            "ERROR: failed to ping healthcheck for monitoring_id='%s' and status='%s': %s"
+            % (monitoring_id, status, e)
+        )
+
+
+def run_with_healthcheck_monitoring(
+    *, monitoring_id: str | None, action: Callable[[], Any]
+) -> Any:
+    if not monitoring_id:
+        return action()
+
+    _ping_healthcheck(monitoring_id, "start")
+
+    try:
+        result = action()
+    except Exception:
+        _ping_healthcheck(monitoring_id, "fail")
+        raise
+
+    _ping_healthcheck(monitoring_id, "success")
+    return result
diff --git a/backend/kernelCI_app/management/commands/notifications.py b/backend/kernelCI_app/management/commands/notifications.py
@@ -18,6 +18,10 @@
     setup_jinja_template,
     send_email_report,
 )
+from kernelCI_app.management.commands.helpers.healthcheck import (
+    MONITORING_ID_PARAM_HELP_TEXT,
+    run_with_healthcheck_monitoring,
+)
 
 from kernelCI_app.management.commands.helpers.summary import (
     SIGNUP_FOLDER,
@@ -892,6 +896,12 @@ def add_arguments(self, parser):
             action="store_true",
             help="Ignore recipients.yaml file (optional for all actions)",
         )
+        parser.add_argument(
+            "--monitoring-id",
+            type=str,
+            default=None,
+            help=MONITORING_ID_PARAM_HELP_TEXT,
+        )
 
         # Action argument (replaces subparsers)
         actions = [
@@ -972,6 +982,13 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, **options):
+        monitoring_id = options.get("monitoring_id")
+        return run_with_healthcheck_monitoring(
+            monitoring_id=monitoring_id,
+            action=lambda: self._run_action(options),
+        )
+
+    def _run_action(self, options):
         # Setup connections
         service = smtp_setup_connection()
 
diff --git a/backend/kernelCI_app/tests/unitTests/commands/healthcheck_helper_test.py b/backend/kernelCI_app/tests/unitTests/commands/healthcheck_helper_test.py
@@ -0,0 +1,80 @@
+from django.test import SimpleTestCase, override_settings
+from unittest.mock import Mock, patch
+
+from kernelCI_app.management.commands.helpers.healthcheck import (
+    _resolve_monitoring_url,
+    run_with_healthcheck_monitoring,
+)
+
+TEST_BASE_URL = "https://example.com"
+
+
+@override_settings(
+    HEALTHCHECK_MONITORING_PATH_MAP={
+        "job-1": "private-token",
+        "job-2": "something/with/slashes",
+    }
+)
+class TestRunWithHealthcheckMonitoring(SimpleTestCase):
+    @patch(
+        "kernelCI_app.management.commands.helpers.healthcheck.HEALTHCHECK_BASE_URL",
+        TEST_BASE_URL,
+    )
+    def test_resolve_monitoring_url_success(self):
+        result = _resolve_monitoring_url("job-1")
+        self.assertEqual(result, f"{TEST_BASE_URL}/private-token")
+
+    @patch("kernelCI_app.management.commands.helpers.healthcheck.requests.get")
+    @patch(
+        "kernelCI_app.management.commands.helpers.healthcheck.HEALTHCHECK_BASE_URL",
+        TEST_BASE_URL,
+    )
+    def test_success_path_pings_start_and_success(self, mock_get):
+        response = Mock()
+        response.raise_for_status.return_value = None
+        mock_get.return_value = response
+
+        result = run_with_healthcheck_monitoring(
+            monitoring_id="job-1", action=lambda: "ok"
+        )
+
+        assert result == "ok"
+        assert mock_get.call_count == 2
+        mock_get.assert_any_call(f"{TEST_BASE_URL}/private-token/start", timeout=10)
+        mock_get.assert_any_call(f"{TEST_BASE_URL}/private-token", timeout=10)
+
+    @patch("kernelCI_app.management.commands.helpers.healthcheck.requests.get")
+    @patch(
+        "kernelCI_app.management.commands.helpers.healthcheck.HEALTHCHECK_BASE_URL",
+        TEST_BASE_URL,
+    )
+    def test_failure_path_pings_start_and_fail(self, mock_get):
+        response = Mock()
+        response.raise_for_status.return_value = None
+        mock_get.return_value = response
+
+        with self.assertRaisesRegex(RuntimeError, "boom"):
+            run_with_healthcheck_monitoring(
+                monitoring_id="job-1",
+                action=lambda: (_ for _ in ()).throw(RuntimeError("boom")),
+            )
+
+        assert mock_get.call_count == 2
+        mock_get.assert_any_call(f"{TEST_BASE_URL}/private-token/start", timeout=10)
+        mock_get.assert_any_call(f"{TEST_BASE_URL}/private-token/fail", timeout=10)
+
+    @patch("kernelCI_app.management.commands.helpers.healthcheck.requests.get")
+    def test_no_monitoring_id_skips_pings(self, mock_get):
+        result = run_with_healthcheck_monitoring(monitoring_id=None, action=lambda: 42)
+
+        assert result == 42
+        mock_get.assert_not_called()
+
+    @patch("kernelCI_app.management.commands.helpers.healthcheck.requests.get")
+    def test_unknown_monitoring_id_skips_network_and_runs_action(self, mock_get):
+        result = run_with_healthcheck_monitoring(
+            monitoring_id="missing-id", action=lambda: "ran"
+        )
+
+        assert result == "ran"
+        mock_get.assert_not_called()
diff --git a/docs/monitoring.md b/docs/monitoring.md
@@ -97,6 +97,23 @@ The monitoring system supports multi-worker Gunicorn deployments using Prometheu
 - `PROMETHEUS_METRICS_PORT`: Port for the metrics aggregator (default: `8001`)
 - `PROMETHEUS_MULTIPROC_DIR`: Directory for multiprocess metric files (default: `/tmp/prometheus_multiproc_dir`)
 
+### Cronjob Healthchecks
+
+The backend can ping healthcheck.io for cronjobs that run Django management commands.
+
+- The public base URL is defined in code as `HEALTHCHECK_BASE_URL`.
+- Private monitor tokens stay in environment variables and are mapped in Django settings.
+- Each monitored cron run sends pings to `/start`, `/success`, and `/fail`.
+
+Configure these variables in `.env.backend`:
+
+- `HEALTHCHECK_MONITORING_NOTIFICATIONS_NEW_ISSUES_HOURLY`
+- `HEALTHCHECK_MONITORING_NOTIFICATIONS_SUMMARY_MICROSOFT_DAILY`
+- `HEALTHCHECK_MONITORING_NOTIFICATIONS_SUMMARY_MAESTRO_DAILY`
+- `HEALTHCHECK_MONITORING_NOTIFICATIONS_HARDWARE_SUMMARY_WEEKLY`
+- `HEALTHCHECK_MONITORING_DELETE_UNUSED_HARDWARE_STATUS_WEEKLY`
+- `HEALTHCHECK_MONITORING_NOTIFICATIONS_METRICS_SUMMARY_WEEKLY`
+
 ## `prometheus.yml`
 - **Target**: `host.docker.internal:8001` (backend running locally)
 - **Metrics Path**: `/metrics/`
