FIX: Add microseconds to the webpoda queries (#292)

* FIX: Add microseconds to the webpoda queries

When doing <= queries, we need to make sure we go up to the end of
the time interval. If a packet had a timestamp of 23:59:59.2 then that
wouldn't be caught by a query of <= 23:59:59 because there is an
implicit floor in the backend web API.

* MNT: Change to an exclusive right-edge for querying webpoda

Rather than trying to add maximum values to times, we can switch
over to using an exclusive right edge with a less-than comparison.
This helps avoid issues with microsecond precision in the database
timestamps when we were only querying with seconds resolution before.

Author: greglucas

imap_data_access/webpoda.py

@@ -186,8 +186,8 @@ def get_packet_times_ert(
     # are handled correctly, so pass them as params
     params = (
         # Query the ERT field between start and end date
-        f"ert>={start_time.strftime('%Y-%m-%dT%H:%M:%S')}"
-        f"&ert<={end_time.strftime('%Y-%m-%dT%H:%M:%S')}"
+        f"ert>={start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
+        f"&ert<{end_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
         # only get the time (packet time)
         # Represent all times in yyyy-MM-dd'T'HH:mm:ss format
         "&project(time)&formatTime(\"yyyy-MM-dd'T'HH:mm:ss\")"
@@ -245,8 +245,8 @@ def get_packet_binary_data_sctime(
     query_range = f"{WEBPODA_APID_URL}/{SYSTEM_ID}/apid_{apid}.bin"
     params = (
         # Query the SCT field between start and end date
-        f"time>={start_time.strftime('%Y-%m-%dT%H:%M:%S')}"
-        f"&time<={end_time.strftime('%Y-%m-%dT%H:%M:%S')}"
+        f"time>={start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
+        f"&time<{end_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
         # only the raw packet data
         "&project(packet)"
     )
@@ -303,7 +303,7 @@ def download_daily_data(
     logger.info(f"Unique spacecraft dates with packets: {unique_dates}")
 
     # Iterate over the packet dates to make a query for each individual spacecraft day
-    # packet_date 00:00:00 -> packet_date 23:59:59
+    # packet_date 00:00:00 -> packet_date+1 00:00:00
     for date in unique_dates:
         path = _get_latest_version_file_path(
             instrument=instrument,
@@ -314,7 +314,7 @@ def download_daily_data(
             continue
 
         daily_start_time = datetime.datetime.combine(date, datetime.time.min)
-        daily_end_time = datetime.datetime.combine(date, datetime.time.max)
+        daily_end_time = daily_start_time + datetime.timedelta(days=1)
 
         # Some instruments request a buffer of packets on either side of the midnight
         # boundary to ensure their packet groupings work together.
@@ -439,13 +439,12 @@ def download_repointing_data(
                 f"{packet_times[-1]}, skipping"
             )
             continue
-        # NOTE: All queries are <= / >= following this, so we need to make sure we
-        #       are not double grabbing packets into the pointings.
-        #       The times included are [repointing_start, repointing_end), exclusive
-        #       on the right edge
+        # NOTE: We need to make sure we are not double grabbing packets into the
+        #       pointings. The times included are [repointing_start, repointing_end),
+        #       exclusive on the right edge
         pointing_end = datetime.datetime.strptime(
             repointings[i + 1]["repoint_end_utc"], "%Y-%m-%d %H:%M:%S.%f"
-        ) - datetime.timedelta(seconds=1)
+        )
         if pointing_end < packet_times[0]:
             # This pointing is before the first packet time, so skip it
             logger.debug(

tests/test_webpoda.py

@@ -42,8 +42,8 @@ def test_get_packet_times_ert(mock_send_request, mock_request):
         f"https://lasp.colorado.edu/ops/imap/poda/dap2/apids/SID1/apid_{apid}.txt",
         headers={"Authorization": "Basic test_token"},
         params=(
-            f"ert>={start_time.strftime('%Y-%m-%dT%H:%M:%S')}"
-            f"&ert<={end_time.strftime('%Y-%m-%dT%H:%M:%S')}"
+            f"ert>={start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
+            f"&ert<{end_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
             "&project(time)&formatTime(\"yyyy-MM-dd'T'HH:mm:ss\")"
         ),
     )
@@ -60,7 +60,7 @@ def test_get_packet_binary_data_sctime(mock_send_request, mock_request):
     mock_send_request.return_value = mock_response
 
     start_time = datetime.datetime(2024, 12, 1, 0, 0, 0)
-    end_time = datetime.datetime(2024, 12, 1, 23, 59, 59)
+    end_time = datetime.datetime(2024, 12, 1, 23, 59, 59, 999999)
     apid = 1136
 
     result = get_packet_binary_data_sctime(apid, start_time, end_time)
@@ -71,8 +71,8 @@ def test_get_packet_binary_data_sctime(mock_send_request, mock_request):
         f"https://lasp.colorado.edu/ops/imap/poda/dap2/apids/SID1/apid_{apid}.bin",
         headers={"Authorization": "Basic test_token"},
         params=(
-            f"time>={start_time.strftime('%Y-%m-%dT%H:%M:%S')}"
-            f"&time<={end_time.strftime('%Y-%m-%dT%H:%M:%S')}"
+            f"time>={start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
+            f"&time<{end_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
             "&project(packet)"
         ),
     )
@@ -113,8 +113,8 @@ def test_download_daily_data(
     assert call == (
         1184,
         start_time - datetime.timedelta(minutes=1),
-        datetime.datetime.combine(start_time, datetime.time.max)
-        + datetime.timedelta(minutes=1),
+        # end time + 1 day, then buffer of 1 minute
+        start_time + datetime.timedelta(days=1) + datetime.timedelta(minutes=1),
     )
 
     # We expect two daily files to be created because we have packets