diff --git a/datastore/db/management/commands/set_status.py b/datastore/db/management/commands/set_status.py index 06047929..fd3276b7 100644 --- a/datastore/db/management/commands/set_status.py +++ b/datastore/db/management/commands/set_status.py @@ -1,6 +1,14 @@ +from datetime import datetime, timezone + from django.core.management.base import BaseCommand, CommandError from db.models import Status, Statuses +from prometheus.views import ( + DURATION_OF_LAST_RUN_FOR_DATAGETTER, + DURATION_OF_LAST_RUN_FOR_DATASTORE_LOAD, + DURATION_OF_LAST_RUN_FOR_GRANTNAV_DATA_PACKAGE_BUILD, + DURATION_OF_LAST_RUN_FOR_MONITORING_SNAPSHOT, +) class Command(BaseCommand): @@ -50,7 +58,31 @@ def handle(self, *args, **options): item.status = Statuses.__dict__.get(options["status"]) except KeyError: CommandError("Unknown status use --list-options to list statuses") - item.save() + if Statuses.__dict__.get(options["status"]) in ( + Statuses.IDLE, + Statuses.READY, + ): + if options.get("what") == "datagetter": + DURATION_OF_LAST_RUN_FOR_DATAGETTER.set( + (datetime.now(timezone.utc) - item.when).total_seconds() + ) + + elif options.get("what") == "datastore": + DURATION_OF_LAST_RUN_FOR_DATASTORE_LOAD.set( + (datetime.now(timezone.utc) - item.when).total_seconds() + ) + + elif options.get("what") == "grantnav_data_package": + DURATION_OF_LAST_RUN_FOR_GRANTNAV_DATA_PACKAGE_BUILD.set( + (datetime.now(timezone.utc) - item.when).total_seconds() + ) + + elif options.get("what") == "monitoring_snapshot": + DURATION_OF_LAST_RUN_FOR_MONITORING_SNAPSHOT.set( + (datetime.now(timezone.utc) - item.when).total_seconds() + ) + + item.save() else: raise CommandError("Not enough parameters supplied to set status") diff --git a/datastore/prometheus/views.py b/datastore/prometheus/views.py index 705daacd..61f309db 100644 --- a/datastore/prometheus/views.py +++ b/datastore/prometheus/views.py @@ -1,3 +1,4 @@ +from datetime import datetime, timezone import re import logging import django.db @@ -56,6 +57,31 @@ "The number of grants which explicitly specify a beneficiary location geocode, but for which we were unable to lookup the geocode. This suggests our geocode lookup information may be out of date, or that publishers have used an invalid geocode.", ) +DURATION_OF_LAST_RUN_FOR_DATAGETTER = Gauge( + "datastore_last_run_datagetter_duration_seconds", + "The duration that the datagetter part of the last data run took.", +) + +DURATION_OF_LAST_RUN_FOR_DATASTORE_LOAD = Gauge( + "datastore_last_run_datastore_load_duration_seconds", + "The duration that the datastore load part of the last data run took.", +) + +DURATION_OF_LAST_RUN_FOR_GRANTNAV_DATA_PACKAGE_BUILD = Gauge( + "datastore_last_run_grantnav_package_build_duration_seconds", + "The duration that the grantnav latest data package build part of the last data run took.", +) + +DURATION_OF_LAST_RUN_FOR_MONITORING_SNAPSHOT = Gauge( + "datastore_last_run_monitoring_snapshot_duration_seconds", + "The duration that the monitoring snapshot generation part of the last data run took.", +) + +TIME_SINCE_LAST_GRANTNAV_DATA_PACKAGE_BUILD = Gauge( + "datastore_time_since_last_grantnav_data_package_build_seconds", + "Length of time since the last grantnav latest data package was built.", +) + class ServiceMetrics(View): def _num_errors_log(self): @@ -169,7 +195,15 @@ class GreaterThan(Func): exc_info=e, ) + def _time_since_last_grantnav_data_package_build(self): + item, c = db.Status.objects.get_or_create(what="grantnav_data_package") + if item.status == db.Statuses.READY: + TIME_SINCE_LAST_GRANTNAV_DATA_PACKAGE_BUILD.set( + (datetime.now(timezone.utc) - item.when).total_seconds() + ) + def get(self, *args, **kwargs): + logger.info("Fetching metrics") # Update gauges unless we're in the middle of processing/loading if db.Status.all_idle_and_ready(): self._num_errors_log() @@ -177,6 +211,7 @@ def get(self, *args, **kwargs): self._total_datagetter_grants() self._total_num_sources_in_last_run() self._num_current_grants_with_beneficiary_location_geocode_without_lookup() + self._time_since_last_grantnav_data_package_build() # Generate latest uses default of the global registry return HttpResponse(generate_latest(), content_type="text/plain") diff --git a/datastore/tests/test_metrics.py b/datastore/tests/test_metrics.py index c3ba40b3..52a3e256 100644 --- a/datastore/tests/test_metrics.py +++ b/datastore/tests/test_metrics.py @@ -1,8 +1,11 @@ import tempfile +import time +from django.core import management from django.test import TestCase import prometheus.views +from db.models import Status, Statuses class TestMetrics(TestCase): @@ -18,3 +21,70 @@ def test_num_of_errors(self): suffix, labels, value = prometheus.views.NUM_ERRORS_LOGGED._samples()[0] self.assertEqual(value, 2.0, "unexpected number of errors in metrics") + + def test_status_durations(self): + view = prometheus.views.ServiceMetrics() + + Status.objects.create(what="datagetter", status=Statuses.IDLE) + Status.objects.create(what="datastore", status=Statuses.IDLE) + Status.objects.create(what="grantnav_data_package", status=Statuses.READY) + Status.objects.create(what="monitoring_snapshot", status=Statuses.READY) + + management.call_command("set_status", what="datagetter", status="IN_PROGRESS") + time.sleep(2) + management.call_command("set_status", what="datagetter", status="IDLE") + ( + suffix, + labels, + value, + ) = prometheus.views.DURATION_OF_LAST_RUN_FOR_DATAGETTER._samples()[0] + self.assertAlmostEqual(value, 2, places=1) + + management.call_command("set_status", what="datastore", status="IN_PROGRESS") + time.sleep(2) + management.call_command("set_status", what="datastore", status="IDLE") + ( + suffix, + labels, + value, + ) = prometheus.views.DURATION_OF_LAST_RUN_FOR_DATASTORE_LOAD._samples()[0] + self.assertAlmostEqual(value, 2, places=1) + + management.call_command( + "set_status", what="grantnav_data_package", status="IN_PROGRESS" + ) + time.sleep(2) + management.call_command( + "set_status", what="grantnav_data_package", status="READY" + ) + ( + suffix, + labels, + value, + ) = prometheus.views.DURATION_OF_LAST_RUN_FOR_GRANTNAV_DATA_PACKAGE_BUILD._samples()[ + 0 + ] + self.assertAlmostEqual(value, 2, places=1) + + time.sleep(2) + view._time_since_last_grantnav_data_package_build() + ( + suffix, + labels, + value, + ) = prometheus.views.TIME_SINCE_LAST_GRANTNAV_DATA_PACKAGE_BUILD._samples()[0] + self.assertAlmostEqual(value, 2, places=1) + + management.call_command( + "set_status", what="monitoring_snapshot", status="IN_PROGRESS" + ) + time.sleep(2) + management.call_command( + "set_status", what="monitoring_snapshot", status="READY" + ) + ( + suffix, + labels, + value, + ) = prometheus.views.DURATION_OF_LAST_RUN_FOR_MONITORING_SNAPSHOT._samples()[0] + self.assertAlmostEqual(value, 2, places=1)