2301 hi l2 early look spacecraft frame map (IMAP-Science-Operations-Center#2312)

* Add more informative message when naming frame has unexpected value

* Fix some issues in L1C processing found when using flight data

* Modify L1C test based on recent change

* Fix tests
add descriptive comment

* Descriptive comment

* Update imap_processing/hi/hi_l1c.py

Co-authored-by: Greg Lucas <greg.m.lucas@gmail.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: Greg Lucas <greg.m.lucas@gmail.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: 3 people

imap_processing/ena_maps/utils/naming.py

@@ -342,7 +342,9 @@ def get_map_coord_frame(frame_str: _coord_frame_str_types) -> SpiceFrame:
         elif frame_str == "gcs":
             return SpiceFrame.IMAP_GCS
         else:
-            raise NotImplementedError("Coordinate frame is not yet implemented.")
+            raise NotImplementedError(
+                f"Coordinate frame {frame_str} is not yet implemented."
+            )
 
     def to_empty_map(
         self,

imap_processing/hi/hi_l1c.py

@@ -102,7 +102,7 @@ def generate_pset_dataset(
 
     pset_dataset = empty_pset_dataset(
         de_dataset.epoch.data[0],
-        de_dataset.esa_energy_step.data,
+        de_dataset.esa_energy_step,
         config_df.cal_prod_config.number_of_products,
         logical_source_parts["sensor"],
     )
@@ -121,7 +121,7 @@ def generate_pset_dataset(
 
 
 def empty_pset_dataset(
-    epoch_val: int, l1b_energy_steps: np.ndarray, n_cal_prods: int, sensor_str: str
+    epoch_val: int, l1b_energy_steps: xr.DataArray, n_cal_prods: int, sensor_str: str
 ) -> xr.Dataset:
     """
     Allocate an empty xarray.Dataset with appropriate pset coordinates.
@@ -130,7 +130,7 @@ def empty_pset_dataset(
     ----------
     epoch_val : int
         The starting epoch in J2000 TT nanoseconds for data in the PSET.
-    l1b_energy_steps : np.ndarray
+    l1b_energy_steps : xarray.DataArray
         The array of esa_energy_step data from the L1B DE product.
     n_cal_prods : int
         Number of calibration products to allocate.
@@ -164,8 +164,12 @@ def empty_pset_dataset(
         "hi_pset_esa_energy_step", check_schema=False
     ).copy()
     dtype = attrs.pop("dtype")
-    # Find the unique, non-zero esa_energy_steps from the L1B data
-    esa_energy_steps = np.array(sorted(set(l1b_energy_steps) - {0}), dtype=dtype)
+    # Find the unique esa_energy_steps from the L1B data
+    # Exclude 0 and FILLVAL
+    esa_energy_steps = np.array(
+        sorted(set(l1b_energy_steps.values) - {0, l1b_energy_steps.attrs["FILLVAL"]}),
+        dtype=dtype,
+    )
     coords["esa_energy_step"] = xr.DataArray(
         esa_energy_steps,
         name="esa_energy_step",
@@ -571,11 +575,26 @@ def find_second_de_packet_data(l1b_dataset: xr.Dataset) -> xr.Dataset:
     # We should get two CCSDS packets per 8-spin ESA step.
     # Get the indices of the packet before each ESA change.
     esa_step = epoch_dataset["esa_step"].values
+    esa_energy_step = epoch_dataset["esa_energy_step"].values
+    # A change in esa_step should indicate the location of the second packet in
+    # each pair of DE packets at an esa_energy_step. In practice, during some
+    # calibration activities, it was observed that the esa_energy_step can change
+    # when the esa_step did not. So, we look for either to change and use the
+    # indices of those changes to identify the second packet in each pair. We
+    # also need to add the last packet index and assume an energy step change
+    # occurs after the last packet.
     second_esa_packet_idx = np.append(
-        np.flatnonzero(np.diff(esa_step) != 0), len(esa_step) - 1
+        np.flatnonzero((np.diff(esa_step) != 0) | (np.diff(esa_energy_step) != 0)),
+        len(esa_step) - 1,
+    )
+    # Remove esa energy steps at 0 - these are calibrations
+    keep_mask = esa_energy_step[second_esa_packet_idx] != 0
+    # Remove esa energy steps at FILLVAL - these are unidentified
+    keep_mask &= (
+        esa_energy_step[second_esa_packet_idx]
+        != l1b_dataset["esa_energy_step"].attrs["FILLVAL"]
     )
-    # Remove esa steps at 0 - these are calibrations
-    second_esa_packet_idx = second_esa_packet_idx[esa_step[second_esa_packet_idx] != 0]
+    second_esa_packet_idx = second_esa_packet_idx[keep_mask]
     # Remove indices where we don't have two consecutive packets at the same ESA
     if second_esa_packet_idx[0] == 0:
         logger.warning(
@@ -584,7 +603,8 @@ def find_second_de_packet_data(l1b_dataset: xr.Dataset) -> xr.Dataset:
         )
         second_esa_packet_idx = second_esa_packet_idx[1:]
     missing_esa_pair_mask = (
-        esa_step[second_esa_packet_idx - 1] != esa_step[second_esa_packet_idx]
+        esa_energy_step[second_esa_packet_idx - 1]
+        != esa_energy_step[second_esa_packet_idx]
     )
     if missing_esa_pair_mask.any():
         logger.warning(
@@ -629,9 +649,11 @@ def get_de_clock_ticks_for_esa_step(
     # ESA step group so this match is the end time. The start time is
     # 8-spins earlier.
     spin_start_mets = spin_df.spin_start_met.to_numpy()
-    # CCSDS MET has one second resolution, add one to it to make sure it is
-    # greater than the spin start time it ended on.
-    end_time_ind = np.flatnonzero(ccsds_met + 1 >= spin_start_mets).max()
+    # CCSDS MET has one second resolution, add two to it to make sure it is
+    # greater than the spin start time it ended on. Theotretically, adding
+    # one second should be sufficeint, but in practice, with flight data, adding
+    # two seconds was found to be necessary.
+    end_time_ind = np.flatnonzero(ccsds_met + 2 >= spin_start_mets).max()
 
     # If the minimum absolute difference is greater than 1/2 the spin-phase
     # we have a problem.

imap_processing/tests/hi/test_hi_l1c.py

@@ -62,7 +62,10 @@ def test_generate_pset_dataset(
 def test_empty_pset_dataset():
     """Test coverage for empty_pset_dataset function"""
     n_energy_steps = 8
-    l1b_esa_energy_steps = np.arange(n_energy_steps + 1).repeat(2)
+    l1b_esa_energy_steps = xr.DataArray(
+        data=np.concat((np.arange(n_energy_steps + 1).repeat(2), np.array([255, 255]))),
+        attrs={"FILLVAL": 255},
+    )
     n_calibration_prods = 5
     sensor_str = HIAPID.H90_SCI_DE.sensor
     dataset = hi_l1c.empty_pset_dataset(
@@ -133,7 +136,7 @@ def test_pset_counts(hi_l1_test_data_path, hi_test_cal_prod_config_path):
     )
     empty_pset = hi_l1c.empty_pset_dataset(
         100,
-        l1b_dataset.esa_energy_step.data,
+        l1b_dataset.esa_energy_step,
         cal_config_df.cal_prod_config.number_of_products,
         HIAPID.H90_SCI_DE.sensor,
     )
@@ -154,7 +157,7 @@ def test_pset_counts_empty_l1b(hi_l1_test_data_path, hi_test_cal_prod_config_pat
     )
     empty_pset = hi_l1c.empty_pset_dataset(
         100,
-        l1b_dataset.esa_energy_step.data,
+        l1b_dataset.esa_energy_step,
         cal_config_df.cal_prod_config.number_of_products,
         HIAPID.H90_SCI_DE.sensor,
     )
@@ -262,8 +265,12 @@ def test_pset_exposure(
     mock_spin_data,
 ):
     """Test coverage for pset_exposure function"""
+    l1b_energy_steps = xr.DataArray(
+        np.arange(2) + 1,
+        attrs={"FILLVAL": 255},
+    )
     empty_pset = hi_l1c.empty_pset_dataset(
-        100, np.arange(2) + 1, 2, HIAPID.H90_SCI_DE.sensor
+        100, l1b_energy_steps, 2, HIAPID.H90_SCI_DE.sensor
     )
     # Set the mock of find_second_de_packet_data to return a xr.Dataset
     # with some dummy data. ESA 1 will get binned data once, ESA 2 will get
@@ -316,10 +323,18 @@ def test_pset_exposure(
 def test_find_second_de_packet_data():
     """Test coverage for find_second_de_packet_data function"""
     # Create a test l1b_dataset
-    # Expect to remove index 0 and 5 due to missing esa_step pair
-    # Expect to remove index 11 due to 0 being a calibration step
-    # Expect to return indices 2, 4, 7, 9, 13
-    esa_steps = np.array([1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 0, 0, 7, 7])
+    # Indices represent CCSDS packets at various ESA steps
+    # Index:      0  1  2  3  4  5  6  7  8  9 10 11 12 13
+    # esa_step:   1  2  2  2  2  4  5  5  6  6  0  0  7  7
+    # esa_energy: 1  2  2  3  3  4  5  5  6  6  0  0  7  7
+    #
+    # Expected second packet indices from diff logic: [0, 2, 4, 5, 7, 9, 11, 13]
+    # Remove index 0: missing pair (first packet in series)
+    # Remove index 5: esa_energy_step 4 doesn't match previous packet's 3
+    # Remove index 11: esa_energy_step is 0 (calibration)
+    # Expected final indices: [2, 4, 7, 9, 13]
+    esa_steps = np.array([1, 2, 2, 2, 2, 4, 5, 5, 6, 6, 0, 0, 7, 7])
+    esa_energy_steps = np.array([1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 0, 0, 7, 7])
     l1b_dataset = xr.Dataset(
         coords={
             "epoch": xr.DataArray(
@@ -336,6 +351,11 @@ def test_find_second_de_packet_data():
                 esa_steps,
                 dims=["epoch"],
             ),
+            "esa_energy_step": xr.DataArray(
+                esa_energy_steps,
+                dims=["epoch"],
+                attrs={"FILLVAL": 255},
+            ),
             "coincidence_type": xr.DataArray(
                 np.ones(10),
                 dims=["event_met"],