ULTRA l1b filter out events that are during a repointing (IMAP-Science-Operations-Center#2324)

* filter out events that were not during the current pointing

Author: lacoak21

imap_processing/tests/ultra/unit/test_de.py

@@ -6,7 +6,10 @@
 
 from imap_processing import imap_module_directory
 from imap_processing.cdf.utils import load_cdf
+from imap_processing.spice.repoint import get_repoint_data
+from imap_processing.spice.time import met_to_ttj2000ns, ttj2000ns_to_et
 from imap_processing.ultra.constants import UltraConstants
+from imap_processing.ultra.l1b.de import calculate_events_in_pointing
 
 TEST_PATH = imap_module_directory / "tests" / "ultra" / "data" / "l1"
 
@@ -119,3 +122,24 @@ def test_calculate_de(df_filt):
         len(l1b_de_dataset["epoch"]),
         3,
     )
+
+
+def test_calculate_events_in_pointing(use_fake_repoint_data_for_time):
+    """Tests calculate_events_in_pointing function."""
+    use_fake_repoint_data_for_time(np.arange(511000000, 511000000 + 86400 * 5, 86400))
+    repoint_id = 1
+    repoint_data = get_repoint_data()
+    pointing_start = repoint_data[repoint_data["repoint_id"] == repoint_id][
+        "repoint_end_met"
+    ].values[0]
+    # Create array of event times that are all within a pointing.
+    event_times = np.arange(pointing_start, pointing_start + 10)
+    # Edit the first event time to be during a repoint.
+    event_times[0] = pointing_start - 1
+    valid_events = np.ones_like(event_times, dtype=bool)
+    in_poiting = calculate_events_in_pointing(
+        repoint_id, ttj2000ns_to_et(met_to_ttj2000ns(event_times)), valid_events
+    )
+    # The first event should be False (not during a pointing), and the rest True.
+    assert np.all(not in_poiting[0])
+    assert np.all(in_poiting[1:])

imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py

@@ -7,7 +7,7 @@
 from imap_processing import imap_module_directory
 from imap_processing.quality_flags import ImapDEOutliersUltraFlags
 from imap_processing.spice.spin import get_spin_data
-from imap_processing.spice.time import sct_to_et
+from imap_processing.spice.time import met_to_ttj2000ns, ttj2000ns_to_et
 from imap_processing.ultra.constants import UltraConstants
 from imap_processing.ultra.l1b.lookup_utils import get_angular_profiles
 from imap_processing.ultra.l1b.ultra_l1b_extended import (
@@ -559,8 +559,12 @@ def test_get_eventtimes(test_fixture, use_fake_spin_data_for_time):
         + expected_max_df["spin_start_subsec_sclk"] / 1e6
     )
 
-    assert sct_to_et(spin_start_min.values[0]) == spin_starts.min()
-    assert sct_to_et(spin_start_max.values[0]) == spin_starts.max()
+    assert (
+        ttj2000ns_to_et(met_to_ttj2000ns(spin_start_min.values[0])) == spin_starts.min()
+    )
+    assert (
+        ttj2000ns_to_et(met_to_ttj2000ns(spin_start_max.values[0])) == spin_starts.max()
+    )
 
     event_times_min = spin_start_min.values[0] + spin_period_sec_min * (
         de_dataset["phase_angle"][0] / 720
@@ -569,8 +573,8 @@ def test_get_eventtimes(test_fixture, use_fake_spin_data_for_time):
         de_dataset["phase_angle"][-1] / 720
     )
 
-    assert sct_to_et(event_times_min) == event_times.min()
-    assert sct_to_et(event_times_max) == event_times.max()
+    assert ttj2000ns_to_et(met_to_ttj2000ns(event_times_min)) == event_times.min()
+    assert ttj2000ns_to_et(met_to_ttj2000ns(event_times_max)) == event_times.max()
 
 
 @pytest.mark.external_test_data

imap_processing/ultra/l1b/de.py

@@ -9,6 +9,8 @@
     ImapDEScatteringUltraFlags,
 )
 from imap_processing.spice.geometry import SpiceFrame
+from imap_processing.spice.repoint import get_repoint_data
+from imap_processing.spice.time import et_to_met
 from imap_processing.ultra.l1b.lookup_utils import get_geometric_factor
 from imap_processing.ultra.l1b.ultra_l1b_annotated import (
     get_annotated_particle_velocity,
@@ -74,6 +76,10 @@ def calculate_de(
     spin_number = get_spin_number(
         de_dataset["shcoarse"].values, de_dataset["spin"].values
     )
+    repoint_id = de_dataset.attrs.get("Repointing", None)
+    if repoint_id is not None:
+        repoint_id = int(repoint_id.replace("repoint", ""))
+
     de_dict["spin"] = spin_number
 
     # Add already populated fields.
@@ -312,6 +318,14 @@ def calculate_de(
 
     # Account for counts=0 (event times have FILL value)
     valid_events = event_times != FILLVAL_FLOAT32
+    # TODO - find a better solution than filtering out data from repointings?
+    if repoint_id is not None:
+        in_pointing = calculate_events_in_pointing(
+            repoint_id, event_times, valid_events
+        )
+        # Update valid_events to only include times within a pointing
+        valid_events &= in_pointing
+
     if np.any(valid_events):
         (
             sc_velocity[valid_events],
@@ -369,5 +383,49 @@ def calculate_de(
     de_dict["quality_scattering"] = scattering_quality_flags
 
     dataset = create_dataset(de_dict, name, "l1b")
+    if repoint_id is not None:
+        # filter out the dataset to only include events in pointing
+        dataset = dataset.isel(epoch=in_pointing)
 
     return dataset
+
+
+def calculate_events_in_pointing(
+    repoint_id: int, event_times: np.ndarray, valid_events: np.ndarray
+) -> np.ndarray:
+    """
+    Calculate boolean array of events within a pointing.
+
+    Parameters
+    ----------
+    repoint_id : int
+        The repointing ID.
+    event_times : np.ndarray
+        Array of event times in ET.
+    valid_events : np.ndarray
+        Boolean array indicating valid events.
+
+    Returns
+    -------
+    in_pointing : np.ndarray
+        Boolean array indicating whether each event is within the pointing period
+        combined with the valid_events mask.
+    """
+    # TODO add this as a helper function in repoint.py
+    repoint_data = get_repoint_data()
+    # To find the pointing start and stop, get the end of the current repointing
+    # and the start of the next repointing
+    repoint_row = repoint_data[repoint_data["repoint_id"] == repoint_id]
+    next_repoint_row = repoint_data[repoint_data["repoint_id"] == repoint_id + 1]
+    pointing_start_met = repoint_row["repoint_end_met"].values[0]
+    pointing_end_met = next_repoint_row["repoint_start_met"].values[0]
+
+    # Create a boolean array for events within the pointing
+    in_pointing = np.zeros(len(event_times), dtype=bool)
+
+    # Check which events are within the pointing
+    in_pointing[valid_events] = (
+        et_to_met(event_times[valid_events]) >= pointing_start_met
+    ) & (et_to_met(event_times[valid_events]) <= pointing_end_met)
+
+    return in_pointing

imap_processing/ultra/l1b/ultra_l1b_annotated.py

@@ -52,7 +52,6 @@ def get_annotated_particle_velocity(
         from_frame=instrument_frame,
         to_frame=spacecraft_frame,
     )
-
     # Particle velocity in the pointing (DPS) frame wrt spacecraft.
     particle_velocity_dps_spacecraft = frame_transform(
         et=time,

imap_processing/ultra/l1b/ultra_l1b_extended.py

@@ -14,7 +14,7 @@
 
 from imap_processing.quality_flags import ImapDEOutliersUltraFlags
 from imap_processing.spice.spin import get_spin_data
-from imap_processing.spice.time import sct_to_et
+from imap_processing.spice.time import met_to_ttj2000ns, ttj2000ns_to_et
 from imap_processing.ultra.constants import UltraConstants
 from imap_processing.ultra.l1b.lookup_utils import (
     get_angular_profiles,
@@ -996,17 +996,21 @@ def get_eventtimes(
     t_spin_period_sec * phase_angle/720
     """
     spin_df = get_spin_data()
+
     index = np.searchsorted(spin_df["spin_number"].values, spin)
     spin_starts = (
         spin_df["spin_start_sec_sclk"].values[index]
         + spin_df["spin_start_subsec_sclk"].values[index] / 1e6
     )
 
     spin_period_sec = spin_df["spin_period_sec"].values[index]
-
     event_times = spin_starts + spin_period_sec * (phase_angle / 720)
 
-    return sct_to_et(event_times), sct_to_et(spin_starts), spin_period_sec
+    return (
+        ttj2000ns_to_et(met_to_ttj2000ns(event_times)),
+        ttj2000ns_to_et(met_to_ttj2000ns(spin_starts)),
+        spin_period_sec,
+    )
 
 
 def interpolate_fwhm(