CoDICE l1a-l2 FSW updates 20260129 (IMAP-Science-Operations-Center#2679)

* codice de fsw stuff

* update sci lut ot contain compression info

* more fsw trickle down changes

* clean up tests

* remove print statement

* remove duplicate code

* copilot suggestiosn

* carry product names through

* add product_names to dataset

* remove product name list

* comment update

* fix tests

* fix ialirt test

Author: lacoak21

imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml

@@ -162,7 +162,6 @@ priority_label:
     FORMAT: A2
     VAR_TYPE: metadata
 
-
 # Common energy labels for species for omni and sectored data
 energy_species_label:
     CATDESC:  Energy Table for {species}
@@ -214,6 +213,20 @@ data_quality:
     VALIDMAX: 1
     VAR_TYPE: data
 
+packet_version:
+    CATDESC: Packet version. Incremented each time the format of the packet changes.
+    DEPEND_0: epoch
+    DISPLAY_TYPE: time_series
+    FIELDNAM: Packet Version
+    FILLVAL: 65535
+    FORMAT: I5
+    LABLAXIS: Packet Version
+    SCALETYP: linear
+    UNITS: " "
+    VALIDMIN: 0
+    VALIDMAX: 65535
+    VAR_TYPE: data
+
 voltage_table:
     CATDESC: ElectroStatic Analyzer Voltage Values
     DEPEND_1: esa_step
@@ -242,30 +255,90 @@ nso_half_spin:
     CATDESC: When No Scan Operation (NSO) was activated
     DEPEND_0: epoch
     DISPLAY_TYPE: time_series
-    FIELDNAM: NSO Mode
+    FIELDNAM: NSO Half Spin
     FILLVAL: 255
     FORMAT: I3
     LABLAXIS: NSO Half Spin
     SCALETYP: linear
     UNITS: half spin number
     VALIDMIN: 0
     VALIDMAX: 255
-    VAR_NOTES: Indicates the point when No Scan Operation (NSO) was activated. In NSO, the ESA voltage is set to the first step in the scan and remains fixed until the next cycle boundary.
+    VAR_NOTES: Indicates the half spin when No Scan Operation (NSO) was activated. In NSO, the ESA voltage is set to the first step in the scan and remains fixed until the next cycle boundary.
+    VAR_TYPE: data
+
+nso_spin_sector:
+    CATDESC: Spin Sector When No Scan Operation (NSO) was activated
+    DEPEND_0: epoch
+    DISPLAY_TYPE: time_series
+    FIELDNAM: NSO Spin Sector
+    FILLVAL: 255
+    FORMAT: I3
+    LABLAXIS: NSO Spin Sector
+    SCALETYP: linear
+    UNITS:  spin sector
+    VALIDMIN: 0
+    VALIDMAX: 255
+    VAR_NOTES: Indicates the spin sector when No Scan Operation (NSO) was activated. In NSO, the ESA voltage is set to the first step in the scan and remains fixed until the next cycle boundary.
+    VAR_TYPE: data
+
+nso_esa_step:
+    CATDESC: Energy Step When No Scan Operation (NSO) was activated
+    DEPEND_0: epoch
+    DISPLAY_TYPE: time_series
+    FIELDNAM: NSO Energy Step
+    FILLVAL: 255
+    FORMAT: I3
+    LABLAXIS: NSO Energy Step
+    SCALETYP: linear
+    UNITS: energy step
+    VALIDMIN: 0
+    VALIDMAX: 255
+    VAR_NOTES: Indicates the energy step when No Scan Operation (NSO) was activated. In NSO, the ESA voltage is set to the first step in the scan and remains fixed until the next cycle boundary.
     VAR_TYPE: data
 
 rgfo_half_spin:
-    CATDESC: When Reduced Gain Factor Operation (RGFO) was activated
+    CATDESC: Half Spin When Reduced Gain Factor Operation (RGFO) was activated
     DEPEND_0: epoch
     DISPLAY_TYPE: time_series
-    FIELDNAM: RGFO Mode
+    FIELDNAM: RGFO Half Spin
     FILLVAL: 255
     FORMAT: I3
     LABLAXIS: RGFO Half Spin
     SCALETYP: linear
     UNITS: half spin number
     VALIDMIN: 0
     VALIDMAX: 255
-    VAR_NOTES: Indicates the point when Reduced Gain Factor Operation (RGFO) was activated. In RGFO, the Entrance ESA voltage is reduced in order to limit the number of ions that reach the detectors.
+    VAR_NOTES: Indicates the half spin when Reduced Gain Factor Operation (RGFO) was activated. In RGFO, the Entrance ESA voltage is reduced in order to limit the number of ions that reach the detectors.
+    VAR_TYPE: data
+
+rgfo_spin_sector:
+    CATDESC: Spin Sector When Reduced Gain Factor Operation (RGFO) was activated
+    DEPEND_0: epoch
+    DISPLAY_TYPE: time_series
+    FIELDNAM: RGFO Spin Sector
+    FILLVAL: 255
+    FORMAT: I3
+    LABLAXIS: RGFO Spin Sector
+    SCALETYP: linear
+    UNITS:  spin sector
+    VALIDMIN: 0
+    VALIDMAX: 255
+    VAR_NOTES: Indicates the spin sector when Reduced Gain Factor Operation (RGFO) was activated. In RGFO, the Entrance ESA voltage is reduced in order to limit the number of ions that reach the detectors.
+    VAR_TYPE: data
+
+rgfo_esa_step:
+    CATDESC: Energy Step When Reduced Gain Factor Operation (RGFO) was activated
+    DEPEND_0: epoch
+    DISPLAY_TYPE: time_series
+    FIELDNAM: RGFO Energy Step
+    FILLVAL: 255
+    FORMAT: I3
+    LABLAXIS: RGFO Energy Step
+    SCALETYP: linear
+    UNITS: energy step
+    VALIDMIN: 0
+    VALIDMAX: 255
+    VAR_NOTES: Indicates the energy step when Reduced Gain Factor Operation (RGFO) was activated. In RGFO, the Entrance ESA voltage is reduced in order to limit the number of ions that reach the detectors.
     VAR_TYPE: data
 
 spin_period:

imap_processing/codice/codice_l1a.py

@@ -1,6 +1,8 @@
 """CoDICE L1A processing functions."""
 
+import datetime
 import logging
+import os
 
 import xarray as xr
 from imap_data_access import ProcessingInputCollection
@@ -52,16 +54,22 @@ def process_l1a(  # noqa: PLR0912
     """
     # Get science data which is L0 packet file
     science_file = dependency.get_file_paths(data_type="l0")[0]
+    # TODO get the exact time the FSW changed on january 29 and relabel the xml file
+    # On January 29, 2026, the CoDICE flight software was updated to a new version.
+    # This update included changes to the packet definitions.
+    start_date = datetime.datetime.strptime(
+        os.path.basename(science_file).split("_")[4], "%Y%m%d"
+    )  # Extract the date from the filename
+    path = imap_module_directory / "codice/packet_definitions/"
+    if start_date >= datetime.datetime(2026, 1, 29):
+        xtce_file = path / "imap_codice_packet-definition_20260129_v001.xml"
+    else:
+        xtce_file = path / "imap_codice_packet-definition_20250101_v001.xml"
 
-    xtce_file = (
-        imap_module_directory / "codice/packet_definitions/codice_packet_definition.xml"
-    )
-    # Decom packet
     datasets_by_apid = packet_file_to_datasets(
         science_file,
         xtce_file,
     )
-
     datasets = []
     for apid in datasets_by_apid:
         if apid not in [CODICEAPID.COD_LO_PHA, CODICEAPID.COD_HI_PHA]:

imap_processing/codice/codice_l1a_de.py

@@ -56,9 +56,16 @@ def extract_initial_items_from_combined_packets(
     spare_1 = np.zeros(n_packets, dtype=np.uint8)
     st_bias_gain_mode = np.zeros(n_packets, dtype=np.uint8)
     sw_bias_gain_mode = np.zeros(n_packets, dtype=np.uint8)
-    priority = np.zeros(n_packets, dtype=np.uint8)
     suspect = np.zeros(n_packets, dtype=np.uint8)
+    priority = np.zeros(n_packets, dtype=np.uint8)
     compressed = np.zeros(n_packets, dtype=np.uint8)
+    rgfo_half_spin = np.zeros(n_packets, dtype=np.uint8)
+    rgfo_esa_step = np.zeros(n_packets, dtype=np.uint8)
+    rgfo_spin_sector = np.zeros(n_packets, dtype=np.uint8)
+    nso_half_spin = np.zeros(n_packets, dtype=np.uint8)
+    nso_spin_sector = np.zeros(n_packets, dtype=np.uint8)
+    nso_esa_step = np.zeros(n_packets, dtype=np.uint8)
+    spare_2 = np.zeros(n_packets, dtype=np.uint16)
     num_events = np.zeros(n_packets, dtype=np.uint32)
     byte_count = np.zeros(n_packets, dtype=np.uint32)
 
@@ -89,20 +96,42 @@ def extract_initial_items_from_combined_packets(
         suspect[pkt_idx] = (mixed_bytes >> 1) & 0x1
         # compressed: 1 bit (LSB)
         compressed[pkt_idx] = mixed_bytes & 0x1
-
-        # Remaining byte-aligned fields
-        num_events[pkt_idx] = int.from_bytes(event_data[12:16], byteorder="big")
-        byte_count[pkt_idx] = int.from_bytes(event_data[16:20], byteorder="big")
-
-        # Remove the first 20 bytes from event_data (header fields from above)
+        # After packet version 1, the fields below are present in event_data
+        if packet_version[pkt_idx] > 1:
+            # All of the fields below are single byte fields
+            rgfo_half_spin[pkt_idx] = event_data[12]
+            rgfo_spin_sector[pkt_idx] = event_data[13]
+            rgfo_esa_step[pkt_idx] = event_data[14]
+            nso_half_spin[pkt_idx] = event_data[15]
+            nso_spin_sector[pkt_idx] = event_data[16]
+            nso_esa_step[pkt_idx] = event_data[17]
+
+            # spare_2 is 16 bits
+            spare_2[pkt_idx] = int.from_bytes(event_data[18:20], byteorder="big")
+            # Remaining byte-aligned fields
+            num_events[pkt_idx] = int.from_bytes(event_data[20:24], byteorder="big")
+            byte_count[pkt_idx] = int.from_bytes(event_data[24:28], byteorder="big")
+            # Header is 28 bytes total for version > 1
+            len_header = 28
+        else:
+            # Remaining byte-aligned fields
+            num_events[pkt_idx] = int.from_bytes(event_data[12:16], byteorder="big")
+            byte_count[pkt_idx] = int.from_bytes(event_data[16:20], byteorder="big")
+            # Header is 20 bytes total for version 1
+            len_header = 20
+
+        # Remove the first len_header bytes from event_data (header fields from above)
         # Then trim to the number of bytes indicated by byte_count
-        if byte_count[pkt_idx] > len(event_data) - 20:
+        if byte_count[pkt_idx] > len(event_data) - len_header:
             raise ValueError(
                 f"Byte count {byte_count[pkt_idx]} exceeds available "
-                f"data length {len(event_data) - 20} for packet index {pkt_idx}."
+                f"data length {len(event_data) - len_header} for packet index"
+                f" {pkt_idx}."
             )
-        packets.event_data.data[pkt_idx] = event_data[20 : 20 + byte_count[pkt_idx]]
 
+        packets.event_data.data[pkt_idx] = event_data[
+            len_header : byte_count[pkt_idx] + len_header
+        ]
         if compressed[pkt_idx]:
             packets.event_data.data[pkt_idx] = decompress(
                 packets.event_data.data[pkt_idx],
@@ -120,6 +149,13 @@ def extract_initial_items_from_combined_packets(
     packets["priority"] = xr.DataArray(priority, dims=["epoch"])
     packets["suspect"] = xr.DataArray(suspect, dims=["epoch"])
     packets["compressed"] = xr.DataArray(compressed, dims=["epoch"])
+    packets["rgfo_half_spin"] = xr.DataArray(rgfo_half_spin, dims=["epoch"])
+    packets["rgfo_spin_sector"] = xr.DataArray(rgfo_spin_sector, dims=["epoch"])
+    packets["rgfo_esa_step"] = xr.DataArray(rgfo_esa_step, dims=["epoch"])
+    packets["nso_half_spin"] = xr.DataArray(nso_half_spin, dims=["epoch"])
+    packets["nso_spin_sector"] = xr.DataArray(nso_spin_sector, dims=["epoch"])
+    packets["nso_esa_step"] = xr.DataArray(nso_esa_step, dims=["epoch"])
+    packets["spare_2"] = xr.DataArray(spare_2, dims=["epoch"])
     packets["num_events"] = xr.DataArray(num_events, dims=["epoch"])
     packets["byte_count"] = xr.DataArray(byte_count, dims=["epoch"])
 
@@ -203,6 +239,7 @@ def _create_dataset_coords(
         collapse_table=0,
         three_d_collapsed=0,
         view_id=0,
+        compression=CoDICECompression.LOSSLESS.value,  # DE data is always lossless
     )
     epochs, epochs_delta = get_codice_epoch_time(
         packets["acq_start_seconds"].isel(epoch=epoch_slice),
@@ -316,7 +353,6 @@ def _unpack_and_store_events(
         n_events = int(num_events_arr[pkt_idx])
         if n_events == 0:
             continue
-
         # Extract and byte-reverse events for LSB unpacking
         pkt_bytes = np.asarray(event_data_arr[pkt_idx], dtype=np.uint8)
         pkt_bytes = pkt_bytes.reshape(n_events, 8)[:, ::-1]
@@ -450,7 +486,16 @@ def process_de_data(
 
     # Add per-epoch metadata from first packet of each epoch
     epoch_slice = slice(None, None, num_priorities)
-    for var in ["sw_bias_gain_mode", "st_bias_gain_mode"]:
+    for var in [
+        "sw_bias_gain_mode",
+        "st_bias_gain_mode",
+        "rgfo_esa_step",
+        "rgfo_half_spin",
+        "rgfo_spin_sector",
+        "nso_esa_step",
+        "nso_half_spin",
+        "nso_spin_sector",
+    ]:
         de_data[var] = xr.DataArray(
             packets[var].isel(epoch=epoch_slice).values,
             dims=["epoch"],
@@ -482,7 +527,6 @@ def process_de_data(
             dims=["epoch", "priority"],
             attrs=cdf_attrs.get_variable_attributes("de_2d_attrs"),
         )
-
     # Reshape packet arrays for validation and assignment
     priorities_2d = packets.priority.values.reshape(num_epochs, num_priorities)
     num_events_2d = packets.num_events.values.reshape(num_epochs, num_priorities)
@@ -534,6 +578,7 @@ def l1a_direct_event(unpacked_dataset: xr.Dataset, apid: int) -> xr.Dataset:
     packets = combine_segmented_packets(
         unpacked_dataset, binary_field_name="event_data"
     )
+
     packets = extract_initial_items_from_combined_packets(packets)
 
     # Gather the CDF attributes

imap_processing/codice/codice_l1a_hi_counters_aggregated.py

@@ -10,6 +10,7 @@
 from imap_processing.codice import constants
 from imap_processing.codice.decompress import decompress
 from imap_processing.codice.utils import (
+    CoDICECompression,
     ViewTabInfo,
     get_codice_epoch_time,
     get_counters_aggregated_pattern,
@@ -61,6 +62,7 @@ def l1a_hi_counters_aggregated(
         sensor=view_tab_info["sensor"],
         three_d_collapsed=view_tab_info["3d_collapse"],
         collapse_table=view_tab_info["collapse_table"],
+        compression=view_tab_info["compression"],
     )
 
     if view_tab_obj.sensor != 1:
@@ -86,7 +88,7 @@ def l1a_hi_counters_aggregated(
     binary_data_list = unpacked_dataset["data"].values
     byte_count_list = unpacked_dataset["byte_count"].values
 
-    compression_algorithm = constants.HI_COMPRESSION_ID_LOOKUP[view_tab_obj.view_id]
+    compression_algorithm = CoDICECompression(view_tab_obj.compression)
 
     # The decompressed data in the shape of (epoch, n). Then reshape later.
     decompressed_data = [

imap_processing/codice/codice_l1a_hi_counters_singles.py

@@ -10,6 +10,7 @@
 from imap_processing.codice import constants
 from imap_processing.codice.decompress import decompress
 from imap_processing.codice.utils import (
+    CoDICECompression,
     ViewTabInfo,
     get_codice_epoch_time,
     get_collapse_pattern_shape,
@@ -59,6 +60,7 @@ def l1a_hi_counters_singles(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.
         sensor=view_tab_info["sensor"],
         three_d_collapsed=view_tab_info["3d_collapse"],
         collapse_table=view_tab_info["collapse_table"],
+        compression=view_tab_info["compression"],
     )
 
     if view_tab_obj.sensor != 1:
@@ -78,7 +80,7 @@ def l1a_hi_counters_singles(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.
     # spin sector size is 1.
     inst_az = collapse_shape[1]
 
-    compression_algorithm = constants.HI_COMPRESSION_ID_LOOKUP[view_tab_obj.view_id]
+    compression_algorithm = CoDICECompression(view_tab_obj.compression)
     # Decompress data using byte count information from decommed data
     binary_data_list = unpacked_dataset["data"].values
     byte_count_list = unpacked_dataset["byte_count"].values

imap_processing/codice/codice_l1a_hi_omni.py

@@ -10,6 +10,7 @@
 from imap_processing.codice import constants
 from imap_processing.codice.decompress import decompress
 from imap_processing.codice.utils import (
+    CoDICECompression,
     ViewTabInfo,
     apply_replacements_to_attrs,
     get_codice_epoch_time,
@@ -61,6 +62,7 @@ def l1a_hi_omni(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
         sensor=view_tab_info["sensor"],
         three_d_collapsed=view_tab_info["3d_collapse"],
         collapse_table=view_tab_info["collapse_table"],
+        compression=view_tab_info["compression"],
     )
 
     if view_tab_obj.sensor != 1:
@@ -71,7 +73,7 @@ def l1a_hi_omni(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
     species_names = species_data.keys()
     logical_source_id = "imap_codice_l1a_hi-omni"
 
-    compression_algorithm = constants.HI_COMPRESSION_ID_LOOKUP[view_tab_obj.view_id]
+    compression_algorithm = CoDICECompression(view_tab_obj.compression)
     # Decompress data using byte count information from decommed data
     binary_data_list = unpacked_dataset["data"].values
     byte_count_list = unpacked_dataset["byte_count"].values