MNT/ENH: Decommutate Ultra packets with derived values (IMAP-Science-Operations-Center#2323)

The Ultra packet definition has engineering unit conversions, so
we can call the packet_file_to_datasets with `use_derived_value=True`
to get the conversion. These are then l1b products to follow along
with the mission convention. We handle that directly in the l1a
processing by calling the routine again with a different flag. The
metadata for l1a can be re-used, except that FILLVAL, MINVAL, and
MAXVAL are all skipped for l1b because those values try to get
converted to the datatype, but the datatype has changed with enumeration
and conversions from uint8 to str or uint8 to float, etc. For ease
of use, we just skip those metadata fields for now.

Author: greglucas

imap_processing/cli.py

@@ -1435,7 +1435,7 @@ def do_processing(
                     f"Unexpected science_files found for ULTRA L1A:"
                     f"{science_files}. Expected only one dependency."
                 )
-            datasets = ultra_l1a.ultra_l1a(science_files[0])
+            datasets = ultra_l1a.ultra_l1a(science_files[0], create_derived_l1b=True)
         elif self.data_level == "l1b":
             science_files = dependencies.get_file_paths(source="ultra", data_type="l1a")
             l1a_dict = {

imap_processing/tests/ultra/unit/test_ultra_l1a.py

@@ -178,8 +178,13 @@ def test_cdf_rates(ccsds_path_theta_0):
 
 def test_cdf_energy_rates(ccsds_path_functional):
     """Tests that CDF file can be created."""
-    test_data = ultra_l1a(ccsds_path_functional, apid_input=ULTRA_ENERGY_RATES.apid[0])
-    test_data[0].attrs["Data_version"] = "999"
+    test_data = ultra_l1a(
+        ccsds_path_functional,
+        apid_input=ULTRA_ENERGY_RATES.apid[0],
+        create_derived_l1b=True,
+    )
+    assert len(test_data) == 2  # l1a and l1b
+
     test_data[0].attrs["Repointing"] = "repoint99999"
     test_data_path = write_cdf(test_data[0], istp=True)
 
@@ -189,6 +194,17 @@ def test_cdf_energy_rates(ccsds_path_functional):
         == "imap_ultra_l1a_45sensor-energy-rates_20240122-repoint99999_v999.cdf"
     )
 
+    # L1b dataset
+    assert "1B" in test_data[1].attrs["Data_type"]
+    test_data[1].attrs["Repointing"] = "repoint99999"
+    test_data_path = write_cdf(test_data[1], istp=True)
+
+    assert test_data_path.exists()
+    assert (
+        test_data_path.name
+        == "imap_ultra_l1b_45sensor-energy-rates_20240122-repoint99999_v999.cdf"
+    )
+
 
 def test_cdf_macrodump(ccsds_path_functional):
     """Tests that CDF file can be created."""

imap_processing/ultra/l1a/ultra_l1a.py

@@ -43,7 +43,7 @@
 
 
 def ultra_l1a(  # noqa: PLR0912
-    packet_file: str, apid_input: int | None = None
+    packet_file: str, apid_input: int | None = None, create_derived_l1b: bool = False
 ) -> list[xr.Dataset]:
     """
     Will process ULTRA L0 data into L1A CDF files at output_filepath.
@@ -54,6 +54,8 @@ def ultra_l1a(  # noqa: PLR0912
         Path to the CCSDS data packet file.
     apid_input : Optional[int]
         Optional apid.
+    create_derived_l1b : bool
+        Whether to create the l1b datasets with derived values.
 
     Returns
     -------
@@ -64,7 +66,17 @@ def ultra_l1a(  # noqa: PLR0912
         f"{imap_module_directory}/ultra/packet_definitions/ULTRA_SCI_COMBINED.xml"
     )
 
+    # Keep a list to track the two versions, l1a and l1b with the derived values.
+    decommutated_packet_datasets = []
     datasets_by_apid = packet_file_to_datasets(packet_file, xtce)
+    decommutated_packet_datasets.append(datasets_by_apid)
+    if create_derived_l1b:
+        # For the housekeeping products, we can create the l1b at the same time
+        # as the l1a since there is no additional processing needed.
+        datasets_by_apid = packet_file_to_datasets(
+            packet_file, xtce, use_derived_value=True
+        )
+        decommutated_packet_datasets.append(datasets_by_apid)
 
     output_datasets = []
 
@@ -109,77 +121,114 @@ def ultra_l1a(  # noqa: PLR0912
     attr_mgr.add_instrument_global_attrs("ultra")
     attr_mgr.add_instrument_variable_attrs("ultra", "l1a")
 
-    for apid in apids:
-        if apid in ULTRA_AUX.apid:
-            decom_ultra_dataset = datasets_by_apid[apid]
-            gattr_key = ULTRA_AUX.logical_source[ULTRA_AUX.apid.index(apid)]
-        elif apid in all_l1a_image_apids:
-            packet_props = all_l1a_image_apids[apid]
-            decom_ultra_dataset = process_ultra_tof(
-                datasets_by_apid[apid], packet_props
-            )
-            gattr_key = packet_props.logical_source[packet_props.apid.index(apid)]
-        elif apid in ULTRA_RATES.apid:
-            decom_ultra_dataset = process_ultra_rates(datasets_by_apid[apid])
-            decom_ultra_dataset = decom_ultra_dataset.drop_vars("fastdata_00")
-            gattr_key = ULTRA_RATES.logical_source[ULTRA_RATES.apid.index(apid)]
-        elif apid in ULTRA_ENERGY_RATES.apid:
-            decom_ultra_dataset = process_ultra_energy_rates(datasets_by_apid[apid])
-            decom_ultra_dataset = decom_ultra_dataset.drop_vars("ratedata")
-            gattr_key = ULTRA_ENERGY_RATES.logical_source[
-                ULTRA_ENERGY_RATES.apid.index(apid)
-            ]
-        elif apid in all_event_apids:
-            decom_ultra_dataset = process_ultra_events(datasets_by_apid[apid], apid)
-            gattr_key = all_event_apids[apid]
+    for i, datasets_by_apid in enumerate(decommutated_packet_datasets):
+        for apid in apids:
+            if apid in ULTRA_AUX.apid:
+                decom_ultra_dataset = datasets_by_apid[apid]
+                gattr_key = ULTRA_AUX.logical_source[ULTRA_AUX.apid.index(apid)]
+            elif apid in all_l1a_image_apids:
+                packet_props = all_l1a_image_apids[apid]
+                decom_ultra_dataset = process_ultra_tof(
+                    datasets_by_apid[apid], packet_props
+                )
+                gattr_key = packet_props.logical_source[packet_props.apid.index(apid)]
+            elif apid in ULTRA_RATES.apid:
+                decom_ultra_dataset = process_ultra_rates(datasets_by_apid[apid])
+                decom_ultra_dataset = decom_ultra_dataset.drop_vars("fastdata_00")
+                gattr_key = ULTRA_RATES.logical_source[ULTRA_RATES.apid.index(apid)]
+            elif apid in ULTRA_ENERGY_RATES.apid:
+                decom_ultra_dataset = process_ultra_energy_rates(datasets_by_apid[apid])
+                decom_ultra_dataset = decom_ultra_dataset.drop_vars("ratedata")
+                gattr_key = ULTRA_ENERGY_RATES.logical_source[
+                    ULTRA_ENERGY_RATES.apid.index(apid)
+                ]
+            elif apid in all_event_apids:
+                # We don't want to process the event l1b datasets since those l1b
+                # products need more information
+                if i == 1:
+                    continue
+                decom_ultra_dataset = process_ultra_events(datasets_by_apid[apid], apid)
+                gattr_key = all_event_apids[apid]
+                # Add coordinate attributes
+                attrs = attr_mgr.get_variable_attributes("event_id")
+                decom_ultra_dataset.coords["event_id"].attrs.update(attrs)
+            elif apid in ULTRA_ENERGY_SPECTRA.apid:
+                decom_ultra_dataset = process_ultra_energy_spectra(
+                    datasets_by_apid[apid]
+                )
+                decom_ultra_dataset = decom_ultra_dataset.drop_vars("compdata")
+                gattr_key = ULTRA_ENERGY_SPECTRA.logical_source[
+                    ULTRA_ENERGY_SPECTRA.apid.index(apid)
+                ]
+            elif apid in ULTRA_MACROS_CHECKSUM.apid:
+                decom_ultra_dataset = process_ultra_macros_checksum(
+                    datasets_by_apid[apid]
+                )
+                gattr_key = ULTRA_MACROS_CHECKSUM.logical_source[
+                    ULTRA_MACROS_CHECKSUM.apid.index(apid)
+                ]
+            elif apid in ULTRA_HK.apid:
+                decom_ultra_dataset = datasets_by_apid[apid]
+                gattr_key = ULTRA_HK.logical_source[ULTRA_HK.apid.index(apid)]
+            elif apid in ULTRA_CMD_TEXT.apid:
+                decom_ultra_dataset = datasets_by_apid[apid]
+                decoded_strings = [
+                    s.decode("ascii").rstrip("\x00")
+                    for s in decom_ultra_dataset["text"].values
+                ]
+                decom_ultra_dataset = decom_ultra_dataset.drop_vars("text")
+                decom_ultra_dataset["text"] = xr.DataArray(
+                    decoded_strings,
+                    dims=["epoch"],
+                    coords={"epoch": decom_ultra_dataset["epoch"]},
+                )
+                gattr_key = ULTRA_CMD_TEXT.logical_source[
+                    ULTRA_CMD_TEXT.apid.index(apid)
+                ]
+            elif apid in ULTRA_CMD_ECHO.apid:
+                decom_ultra_dataset = process_ultra_cmd_echo(datasets_by_apid[apid])
+                gattr_key = ULTRA_CMD_ECHO.logical_source[
+                    ULTRA_CMD_ECHO.apid.index(apid)
+                ]
+            else:
+                logger.error(f"APID {apid} not recognized.")
+                continue
+
+            decom_ultra_dataset.attrs.update(attr_mgr.get_global_attributes(gattr_key))
+
+            if i == 1:
+                # Derived values dataset at l1b
+                # We already have the l1a attributes, just update the l1a -> l1b
+                # in the metadata.
+                decom_ultra_dataset.attrs["Data_type"] = decom_ultra_dataset.attrs[
+                    "Data_type"
+                ].replace("1A", "1B")
+                decom_ultra_dataset.attrs["Logical_source"] = decom_ultra_dataset.attrs[
+                    "Logical_source"
+                ].replace("l1a", "l1b")
+                decom_ultra_dataset.attrs["Logical_source_description"] = (
+                    decom_ultra_dataset.attrs["Logical_source_description"].replace(
+                        "1A", "1B"
+                    )
+                )
+
+            # Add data variable attributes
+            for key in decom_ultra_dataset.data_vars:
+                attrs = attr_mgr.get_variable_attributes(key.lower())
+                decom_ultra_dataset.data_vars[key].attrs.update(attrs)
+                if i == 1:
+                    # For l1b datasets, the FILLVAL and VALIDMIN/MAX may be
+                    # different datatypes, so we can't use them directly from l1a.
+                    # just remove them for now since we don't really have a need for
+                    # for them currently.
+                    for attr_key in ["FILLVAL", "VALIDMIN", "VALIDMAX"]:
+                        if attr_key in decom_ultra_dataset.data_vars[key].attrs:
+                            decom_ultra_dataset.data_vars[key].attrs.pop(attr_key)
+
             # Add coordinate attributes
-            attrs = attr_mgr.get_variable_attributes("event_id")
-            decom_ultra_dataset.coords["event_id"].attrs.update(attrs)
-        elif apid in ULTRA_ENERGY_SPECTRA.apid:
-            decom_ultra_dataset = process_ultra_energy_spectra(datasets_by_apid[apid])
-            decom_ultra_dataset = decom_ultra_dataset.drop_vars("compdata")
-            gattr_key = ULTRA_ENERGY_SPECTRA.logical_source[
-                ULTRA_ENERGY_SPECTRA.apid.index(apid)
-            ]
-        elif apid in ULTRA_MACROS_CHECKSUM.apid:
-            decom_ultra_dataset = process_ultra_macros_checksum(datasets_by_apid[apid])
-            gattr_key = ULTRA_MACROS_CHECKSUM.logical_source[
-                ULTRA_MACROS_CHECKSUM.apid.index(apid)
-            ]
-        elif apid in ULTRA_HK.apid:
-            decom_ultra_dataset = datasets_by_apid[apid]
-            gattr_key = ULTRA_HK.logical_source[ULTRA_HK.apid.index(apid)]
-        elif apid in ULTRA_CMD_TEXT.apid:
-            decom_ultra_dataset = datasets_by_apid[apid]
-            decoded_strings = [
-                s.decode("ascii").rstrip("\x00")
-                for s in decom_ultra_dataset["text"].values
-            ]
-            decom_ultra_dataset = decom_ultra_dataset.drop_vars("text")
-            decom_ultra_dataset["text"] = xr.DataArray(
-                decoded_strings,
-                dims=["epoch"],
-                coords={"epoch": decom_ultra_dataset["epoch"]},
-            )
-            gattr_key = ULTRA_CMD_TEXT.logical_source[ULTRA_CMD_TEXT.apid.index(apid)]
-        elif apid in ULTRA_CMD_ECHO.apid:
-            decom_ultra_dataset = process_ultra_cmd_echo(datasets_by_apid[apid])
-            gattr_key = ULTRA_CMD_ECHO.logical_source[ULTRA_CMD_ECHO.apid.index(apid)]
-        else:
-            logger.error(f"APID {apid} not recognized.")
-            continue
-
-        decom_ultra_dataset.attrs.update(attr_mgr.get_global_attributes(gattr_key))
-
-        # Add data variable attributes
-        for key in decom_ultra_dataset.data_vars:
-            attrs = attr_mgr.get_variable_attributes(key.lower())
-            decom_ultra_dataset.data_vars[key].attrs.update(attrs)
-
-        # Add coordinate attributes
-        attrs = attr_mgr.get_variable_attributes("epoch", check_schema=False)
-        decom_ultra_dataset.coords["epoch"].attrs.update(attrs)
-
-        output_datasets.append(decom_ultra_dataset)
+            attrs = attr_mgr.get_variable_attributes("epoch", check_schema=False)
+            decom_ultra_dataset.coords["epoch"].attrs.update(attrs)
+
+            output_datasets.append(decom_ultra_dataset)
 
     return output_datasets