I-ALiRT - cdf debug (IMAP-Science-Operations-Center#2692)

Author: laspsandoval

imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml

@@ -209,10 +209,10 @@ codice_hi_energy_plus:
   dtype: float32
   LABLAXIS: Energy upper delta
 
-codice_hi_elevation:
-  CATDESC: Elevation Angle
-  FIELDNAM: Elevation Angle
-  LABLAXIS: Elevation Angle
+codice_hi_polar:
+  CATDESC: Polar Angle
+  FIELDNAM: Polar Angle
+  LABLAXIS: Polar Angle
   SCALETYP: linear
   UNITS: degrees
   VALIDMIN: 0.0
@@ -222,9 +222,9 @@ codice_hi_elevation:
   FORMAT: F12.6
   dtype: float32
 
-codice_hi_elevation_labels:
-  CATDESC: CoDICE-Hi elevation labels
-  FIELDNAM: CoDICE-Hi elevation labels
+codice_hi_polar_labels:
+  CATDESC: CoDICE-Hi polar labels
+  FIELDNAM: CoDICE-Hi polar labels
   FORMAT: A12
   VAR_TYPE: metadata
   UNITS: " "
@@ -262,7 +262,7 @@ swe_electron_energy:
 # Variables
 codice_hi_h:
   <<: *default_float32
-  CATDESC: Proton flux from CoDICE-Hi in 15 energy bins between 0.02 to 3.62 MeV over 4 spin sectors and 4 azimuth bins
+  CATDESC: Proton flux from CoDICE-Hi in 15 energy bins between 0.02 to 3.62 MeV over 4 spin sectors and 4 polar angle bins
   FIELDNAM: H+ intensity
   LABLAXIS: H+ intensity
   DEPEND_0: codice_hi_epoch
@@ -271,9 +271,9 @@ codice_hi_h:
   VALIDMAX: 100000000.0
   DEPEND_1: codice_hi_energy_center
   DEPEND_2: codice_hi_spin_sector
-  DEPEND_3: codice_hi_elevation
+  DEPEND_3: codice_hi_polar
   DISPLAY_TYPE: no_plot
-  FORMAT: F6.1
+  FORMAT: F11.1
   VAR_NOTES: Energy ranges (MeV) are 0.0200 to 0.0283, 0.0283 to 0.0400, 0.0400 to 0.0566, 0.0566 to 0.0800, 0.0800 to 0.113, 0.113 to 0.160, 0.160 to 0.226, 0.226 to 0.320, 0.320 to 0.453, 0.453 to 0.640, 0.640 to 0.905, 0.905 to 1.28, 1.28 to 1.81, 1.81 to 2.56, 2.56 to 3.62.
 
 codice_lo_c_over_o_abundance:
@@ -362,6 +362,7 @@ hit_e_a_side_med_en:
   UNITS: counts per second
   VALIDMIN: 0
   VALIDMAX: 1000000000.0
+  FORMAT: F17.6
   VAR_NOTES: A-side indicates the HIT aperture that is nominally looking in the anti-sunward direction. B-side indicates the aperture looking nominally in the sunward direction.
 
 hit_e_a_side_high_en:
@@ -395,6 +396,7 @@ hit_e_b_side_med_en:
   UNITS: counts per second
   VALIDMIN: 0
   VALIDMAX: 1000000000.0
+  FORMAT: F17.6
   VAR_NOTES: A-side indicates the HIT aperture that is nominally looking in the anti-sunward direction. B-side indicates the aperture looking nominally in the sunward direction.
 
 hit_e_b_side_high_en:
@@ -417,6 +419,7 @@ hit_h_omni_low_en:
   UNITS: counts per second
   VALIDMIN: 0
   VALIDMAX: 1000000000.0
+  FORMAT: F17.6
   VAR_NOTES: Omni indicates the sum of the number of particles divided by the area of the full sky.
 
 hit_h_omni_med_en:
@@ -428,6 +431,7 @@ hit_h_omni_med_en:
   UNITS: counts per second
   VALIDMIN: 0
   VALIDMAX: 1000000000.0
+  FORMAT: F17.6
   VAR_NOTES: Omni indicates the sum of the number of particles divided by the area of the full sky.
 
 hit_h_a_side_high_en:
@@ -461,6 +465,7 @@ hit_he_omni_low_en:
   UNITS: counts per second
   VALIDMIN: 0
   VALIDMAX: 1000000000.0
+  FORMAT: F17.6
   VAR_NOTES: Omni indicates the sum of the number of particles divided by the area of the full sky.
 
 hit_he_omni_high_en:
@@ -472,6 +477,7 @@ hit_he_omni_high_en:
   UNITS: counts per second
   VALIDMIN: 0
   VALIDMAX: 1000000000.0
+  FORMAT: F17.6
   VAR_NOTES: Omni indicates the sum of the number of particles divided by the area of the full sky.
 
 mag_B_magnitude:

imap_processing/ialirt/calculate_ingest.py

@@ -107,9 +107,9 @@ def format_ingest_data(last_filename: str, log_lines: list) -> dict:
     last_timestamp_str = last_timestamp_str.replace("_", ":")
     end_of_time = datetime.strptime(last_timestamp_str, "%Y-%jT%H:%M:%S")
 
-    # File is created every 5 minutes.
+    # File creation time of last file minus 48 hrs.
     start_of_time = datetime.strptime(last_timestamp_str, "%Y-%jT%H:%M:%S") - timedelta(
-        minutes=5
+        hours=48
     )
 
     # Parse file.

imap_processing/ialirt/utils/constants.py

@@ -12,7 +12,7 @@
         "codice_hi_epoch",
         "codice_hi_energy_center",
         "codice_hi_spin_sector",
-        "codice_hi_elevation",
+        "codice_hi_polar",
     ],
     # C/O abundance ratio
     "codice_lo_c_over_o_abundance": ["codice_lo_epoch"],

imap_processing/ialirt/utils/create_xarray.py

@@ -1,6 +1,7 @@
 """Creates xarray based on structure of queried DynamoDB."""
 
 from collections import defaultdict
+from datetime import datetime, timedelta
 
 import numpy as np
 import xarray as xr
@@ -18,6 +19,7 @@
     hit_restricted_fields,
     swe_energy,
 )
+from imap_processing.spice.time import et_to_ttj2000ns, str_to_et
 
 
 def create_xarray_from_records(records: list[dict]) -> xr.Dataset:  # noqa: PLR0912
@@ -44,6 +46,19 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset:  # noqa: PLR0
     epochs: dict[str, list[int]] = {inst: [] for inst in (one_epoch | multi_epoch)}
     by_inst: dict[str, list[dict]] = defaultdict(list)
 
+    # Get the start and end ttj2000ns.
+    date = records[0]["time_utc"]  # e.g. "2025-06-20T08:00:00Z"
+
+    # Parse as UTC
+    dt = datetime.fromisoformat(date)
+
+    # Start and end of that UTC day
+    start_str = dt.date().isoformat() + "T00:00:00Z"
+    end_str = (dt.date() + timedelta(days=1)).isoformat() + "T00:00:00Z"
+
+    start_ttj2000 = et_to_ttj2000ns(str_to_et(start_str))
+    end_ttj2000 = et_to_ttj2000ns(str_to_et(end_str))
+
     for record in records:
         inst = record.get("instrument")
         by_inst[record["instrument"]].append(record)
@@ -162,21 +177,21 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset:  # noqa: PLR0
         ),
     )
 
-    elevation = xr.DataArray(
+    polar = xr.DataArray(
         HI_IALIRT_ELEVATION_ANGLE,
-        name="codice_hi_elevation",
-        dims=["codice_hi_elevation"],
+        name="codice_hi_polar",
+        dims=["codice_hi_polar"],
         attrs=cdf_manager.get_variable_attributes(
-            "codice_hi_elevation", check_schema=False
+            "codice_hi_polar", check_schema=False
         ),
     )
 
-    elevation_labels = xr.DataArray(
-        [f"{float(v):.1f}deg" for v in elevation.values],
-        name="codice_hi_elevation_labels",
-        dims=["codice_hi_elevation"],
+    polar_labels = xr.DataArray(
+        [f"{float(v):.1f}deg" for v in polar.values],
+        name="codice_hi_polar_labels",
+        dims=["codice_hi_polar"],
         attrs=cdf_manager.get_variable_attributes(
-            "codice_hi_elevation_labels", check_schema=False
+            "codice_hi_polar_labels", check_schema=False
         ),
     )
 
@@ -221,8 +236,8 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset:  # noqa: PLR0
         "codice_hi_energy_center": codice_hi_energy_centers,
         "codice_hi_energy_minus": codice_energy_minus,
         "codice_hi_energy_plus": codice_energy_plus,
-        "codice_hi_elevation": elevation,
-        "codice_hi_elevation_labels": elevation_labels,
+        "codice_hi_polar": polar,
+        "codice_hi_polar_labels": polar_labels,
         "codice_hi_spin_sector": spin_sector,
         "codice_hi_spin_sector_labels": spin_sector_labels,
         "swe_electron_energy": swe_electron_energy,
@@ -305,4 +320,16 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset:  # noqa: PLR0
             if key.startswith("sc_"):
                 dataset[key].data[i, :] = np.asarray(record[key], dtype=np.float32)
 
+    # Trim data that does not fit within the UTC day.
+    for inst in epochs.keys():
+        if inst == "spacecraft":
+            dim = "ephemeris_epoch"
+        else:
+            dim = f"{inst}_epoch"
+
+        if dim in dataset.coords:
+            dataset = dataset.sel(
+                {dim: (dataset[dim] >= start_ttj2000) & (dataset[dim] < end_ttj2000)}
+            )
+
     return dataset

imap_processing/tests/ialirt/unit/test_create_xarray.py

@@ -16,8 +16,8 @@ def test_create_dataset():
         {
             "instrument": "mag",
             "time_utc": "2025-06-20T08:00:00",
-            "ttj2000ns": 123456789000001,
-            "mag_epoch": 123456789000001,
+            "ttj2000ns": 803692869184000000,
+            "mag_epoch": 803692869184000000,
             "mag_B_GSE": [Decimal("5.0"), Decimal("-3.2"), Decimal("1.1")],
             "mag_B_GSM": [Decimal("4.8"), Decimal("-3.0"), Decimal("1.0")],
             "mag_B_RTN": [Decimal("5.1"), Decimal("-3.3"), Decimal("1.2")],
@@ -30,12 +30,12 @@ def test_create_dataset():
         {
             "instrument": "codice_hi",
             "time_utc": "2025-06-20T08:00:00",
-            "ttj2000ns": 123456789000000,
+            "ttj2000ns": 803692869184000000,
             "codice_hi_epoch": [
-                123456789000000,
-                123456789000000,
-                123456789000000,
-                123456789000000,
+                803692869184000000,
+                803692869184000000,
+                803692869184000000,
+                803692869184000000,
             ],
             "codice_hi_h": [
                 [
@@ -48,8 +48,8 @@ def test_create_dataset():
         {
             "instrument": "codice_lo",
             "time_utc": "2025-06-20T08:00:00",
-            "ttj2000ns": 123456789000000,
-            "codice_lo_epoch": 123456789000000,
+            "ttj2000ns": 803692869184000000,
+            "codice_lo_epoch": 803692869184000000,
             "codice_lo_c_over_o_abundance": Decimal("0.5"),
             "codice_lo_mg_over_o_abundance": Decimal("0.3"),
             "codice_lo_fe_over_o_abundance": Decimal("0.2"),
@@ -60,8 +60,8 @@ def test_create_dataset():
         {
             "instrument": "hit",
             "time_utc": "2025-06-20T08:00:00",
-            "ttj2000ns": 123456789000002,
-            "hit_epoch": 123456789000002,
+            "ttj2000ns": 803692869184000000,
+            "hit_epoch": 803692869184000000,
             "hit_e_a_side_low_en": Decimal("0.0"),
             "hit_e_a_side_med_en": Decimal("0.0"),
             "hit_e_a_side_high_en": Decimal("0.0"),
@@ -78,17 +78,17 @@ def test_create_dataset():
         {
             "instrument": "swapi",
             "time_utc": "2025-06-20T08:00:00",
-            "ttj2000ns": 123456789000002,
-            "swapi_epoch": 123456789000002,
+            "ttj2000ns": 803692869184000000,
+            "swapi_epoch": 803692869184000000,
             "swapi_pseudo_proton_speed": Decimal("400.0"),
             "swapi_pseudo_proton_density": Decimal("5.0"),
             "swapi_pseudo_proton_temperature": Decimal("100000.0"),
         },
         {
             "instrument": "swe",
             "time_utc": "2025-06-20T08:00:00",
-            "ttj2000ns": 123456789000002,
-            "swe_epoch": 123456789000002,
+            "ttj2000ns": 803692869184000000,
+            "swe_epoch": 803692869184000000,
             "swe_normalized_counts": [Decimal("0.0") for _ in range(8)],
             "swe_counterstreaming_electrons": Decimal("1.0"),
         },
@@ -140,7 +140,7 @@ def test_create_dataset():
         np.zeros(8, dtype=np.uint32),
     )
     np.testing.assert_allclose(
-        dataset["mag_B_GSE"].sel(mag_epoch=123456789000001).values,
+        dataset["mag_B_GSE"].sel(mag_epoch=803692869184000000).values,
         [5.0, -3.2, 1.1],
     )
 
@@ -155,7 +155,7 @@ def test_create_dataset():
         "codice_hi_epoch",
         "codice_hi_energy_center",
         "codice_hi_spin_sector",
-        "codice_hi_elevation",
+        "codice_hi_polar",
     )
 
     # Tests that you can write to a cdf.