Prometheus metrics for pipeline tracking

datastore/db/management/commands/set_status.py

@@ -1,6 +1,14 @@
+from datetime import datetime, timezone
+
 from django.core.management.base import BaseCommand, CommandError
 
 from db.models import Status, Statuses
+from prometheus.views import (
+    DURATION_OF_LAST_RUN_FOR_DATAGETTER,
+    DURATION_OF_LAST_RUN_FOR_DATASTORE_LOAD,
+    DURATION_OF_LAST_RUN_FOR_GRANTNAV_DATA_PACKAGE_BUILD,
+    DURATION_OF_LAST_RUN_FOR_MONITORING_SNAPSHOT,
+)
 
 
 class Command(BaseCommand):
@@ -50,7 +58,31 @@ def handle(self, *args, **options):
                 item.status = Statuses.__dict__.get(options["status"])
             except KeyError:
                 CommandError("Unknown status use --list-options to list statuses")
-            item.save()
 
+            if Statuses.__dict__.get(options["status"]) in (
+                Statuses.IDLE,
+                Statuses.READY,
+            ):
+                if options.get("what") == "datagetter":
+                    DURATION_OF_LAST_RUN_FOR_DATAGETTER.set(
+                        (datetime.now(timezone.utc) - item.when).total_seconds()
+                    )
+
+                elif options.get("what") == "datastore":
+                    DURATION_OF_LAST_RUN_FOR_DATASTORE_LOAD.set(
+                        (datetime.now(timezone.utc) - item.when).total_seconds()
+                    )
+
+                elif options.get("what") == "grantnav_data_package":
+                    DURATION_OF_LAST_RUN_FOR_GRANTNAV_DATA_PACKAGE_BUILD.set(
+                        (datetime.now(timezone.utc) - item.when).total_seconds()
+                    )
+
+                elif options.get("what") == "monitoring_snapshot":
+                    DURATION_OF_LAST_RUN_FOR_MONITORING_SNAPSHOT.set(
+                        (datetime.now(timezone.utc) - item.when).total_seconds()
+                    )
+
+            item.save()
         else:
             raise CommandError("Not enough parameters supplied to set status")

datastore/prometheus/views.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timezone
 import re
 import logging
 import django.db
@@ -56,6 +57,31 @@
     "The number of grants which explicitly specify a beneficiary location geocode, but for which we were unable to lookup the geocode. This suggests our geocode lookup information may be out of date, or that publishers have used an invalid geocode.",
 )
 
+DURATION_OF_LAST_RUN_FOR_DATAGETTER = Gauge(
+    "datastore_last_run_datagetter_duration_seconds",
+    "The duration that the datagetter part of the last data run took.",
+)
+
+DURATION_OF_LAST_RUN_FOR_DATASTORE_LOAD = Gauge(
+    "datastore_last_run_datastore_load_duration_seconds",
+    "The duration that the datastore load part of the last data run took.",
+)
+
+DURATION_OF_LAST_RUN_FOR_GRANTNAV_DATA_PACKAGE_BUILD = Gauge(
+    "datastore_last_run_grantnav_package_build_duration_seconds",
+    "The duration that the grantnav latest data package build part of the last data run took.",
+)
+
+DURATION_OF_LAST_RUN_FOR_MONITORING_SNAPSHOT = Gauge(
+    "datastore_last_run_monitoring_snapshot_duration_seconds",
+    "The duration that the monitoring snapshot generation part of the last data run took.",
+)
+
+TIME_SINCE_LAST_GRANTNAV_DATA_PACKAGE_BUILD = Gauge(
+    "datastore_time_since_last_grantnav_data_package_build_seconds",
+    "Length of time since the last grantnav latest data package was built.",
+)
+
 
 class ServiceMetrics(View):
     def _num_errors_log(self):
@@ -169,14 +195,23 @@ class GreaterThan(Func):
                 exc_info=e,
             )
 
+    def _time_since_last_grantnav_data_package_build(self):
+        item, c = db.Status.objects.get_or_create(what="grantnav_data_package")
+        if item.status == db.Statuses.READY:
+            TIME_SINCE_LAST_GRANTNAV_DATA_PACKAGE_BUILD.set(
+                (datetime.now(timezone.utc) - item.when).total_seconds()
+            )
+
     def get(self, *args, **kwargs):
+        logger.info("Fetching metrics")
         # Update gauges unless we're in the middle of processing/loading
         if db.Status.all_idle_and_ready():
             self._num_errors_log()
             self._total_latest_grants()
             self._total_datagetter_grants()
             self._total_num_sources_in_last_run()
             self._num_current_grants_with_beneficiary_location_geocode_without_lookup()
+            self._time_since_last_grantnav_data_package_build()
 
         # Generate latest uses default of the global registry
         return HttpResponse(generate_latest(), content_type="text/plain")

datastore/tests/test_metrics.py

@@ -1,8 +1,11 @@
 import tempfile
+import time
 
+from django.core import management
 from django.test import TestCase
 
 import prometheus.views
+from db.models import Status, Statuses
 
 
 class TestMetrics(TestCase):
@@ -18,3 +21,70 @@ def test_num_of_errors(self):
         suffix, labels, value = prometheus.views.NUM_ERRORS_LOGGED._samples()[0]
 
         self.assertEqual(value, 2.0, "unexpected number of errors in metrics")
+
+    def test_status_durations(self):
+        view = prometheus.views.ServiceMetrics()
+
+        Status.objects.create(what="datagetter", status=Statuses.IDLE)
+        Status.objects.create(what="datastore", status=Statuses.IDLE)
+        Status.objects.create(what="grantnav_data_package", status=Statuses.READY)
+        Status.objects.create(what="monitoring_snapshot", status=Statuses.READY)
+
+        management.call_command("set_status", what="datagetter", status="IN_PROGRESS")
+        time.sleep(2)
+        management.call_command("set_status", what="datagetter", status="IDLE")
+        (
+            suffix,
+            labels,
+            value,
+        ) = prometheus.views.DURATION_OF_LAST_RUN_FOR_DATAGETTER._samples()[0]
+        self.assertAlmostEqual(value, 2, places=1)
+
+        management.call_command("set_status", what="datastore", status="IN_PROGRESS")
+        time.sleep(2)
+        management.call_command("set_status", what="datastore", status="IDLE")
+        (
+            suffix,
+            labels,
+            value,
+        ) = prometheus.views.DURATION_OF_LAST_RUN_FOR_DATASTORE_LOAD._samples()[0]
+        self.assertAlmostEqual(value, 2, places=1)
+
+        management.call_command(
+            "set_status", what="grantnav_data_package", status="IN_PROGRESS"
+        )
+        time.sleep(2)
+        management.call_command(
+            "set_status", what="grantnav_data_package", status="READY"
+        )
+        (
+            suffix,
+            labels,
+            value,
+        ) = prometheus.views.DURATION_OF_LAST_RUN_FOR_GRANTNAV_DATA_PACKAGE_BUILD._samples()[
+            0
+        ]
+        self.assertAlmostEqual(value, 2, places=1)
+
+        time.sleep(2)
+        view._time_since_last_grantnav_data_package_build()
+        (
+            suffix,
+            labels,
+            value,
+        ) = prometheus.views.TIME_SINCE_LAST_GRANTNAV_DATA_PACKAGE_BUILD._samples()[0]
+        self.assertAlmostEqual(value, 2, places=1)
+
+        management.call_command(
+            "set_status", what="monitoring_snapshot", status="IN_PROGRESS"
+        )
+        time.sleep(2)
+        management.call_command(
+            "set_status", what="monitoring_snapshot", status="READY"
+        )
+        (
+            suffix,
+            labels,
+            value,
+        ) = prometheus.views.DURATION_OF_LAST_RUN_FOR_MONITORING_SNAPSHOT._samples()[0]
+        self.assertAlmostEqual(value, 2, places=1)