Harrison #56 - PMCL - GLOWS: Use Lo l1b housekeeping CDF to determine daily pivot angle for L3e

Author: pleasant-menlo

imap_l3_processing/glows/glows_processor.py

@@ -38,7 +38,7 @@
 from imap_l3_processing.glows.l3e.glows_l3e_initializer import GlowsL3EInitializer, GlowsL3EInitializerOutput
 from imap_l3_processing.glows.l3e.glows_l3e_lo_model import GlowsL3ELoData
 from imap_l3_processing.glows.l3e.glows_l3e_ultra_model import GlowsL3EUltraData
-from imap_l3_processing.glows.l3e.glows_l3e_utils import determine_call_args_for_l3e_executable
+from imap_l3_processing.glows.l3e.glows_l3e_utils import determine_call_args_for_l3e_executable, get_lo_pivot_angles
 from imap_l3_processing.models import InputMetadata
 from imap_l3_processing.processor import Processor
 from imap_l3_processing.utils import save_data
@@ -260,7 +260,7 @@ def process_l3e_lo(
         repointing: int,
         repointing_start: datetime,
         epoch_delta: timedelta,
-        elongation_value: int,
+        elongation_value: float,
         version: int
 ) -> list[Path]:
     repointing_midpoint = repointing_start + epoch_delta
@@ -437,20 +437,17 @@ def process_l3e_hi(parent_file_names: list[str], repointing: int, repointing_sta
 def process_l3e(initializer_data: GlowsL3EInitializerOutput):
     products_list = []
 
+    lo_pivot_angles = get_lo_pivot_angles(initializer_data.repointings.repointing_numbers)
     for repointing in initializer_data.repointings.repointing_numbers:
         with SwallowExceptionAndLog(f"Exception encountered when processing L3e for repointing {repointing}"):
             start_repointing, end_repointing = get_pointing_date_range(repointing)
             epoch_delta: timedelta = (end_repointing - start_repointing) / 2
 
             with SwallowExceptionAndLog(f"Exception encountered when processing L3e lo for repointing {repointing}"):
                 lo_parent_file_names = initializer_data.dependencies.get_lo_parents()
-                repointing_doy = start_repointing.timetuple().tm_yday
-                lo_elongation = initializer_data.dependencies.elongation.get(f"{start_repointing.year}{repointing_doy:03}")
-                if lo_elongation is not None:
-                    lo_version = initializer_data.repointings.lo_repointings[repointing]
-                    products_list.extend(process_l3e_lo(lo_parent_file_names, repointing, start_repointing, epoch_delta, lo_elongation, lo_version))
-                else:
-                    logger.warning(f"Skipping L3e Lo processing for {repointing=} because there is no elongation data")
+                lo_elongation = lo_pivot_angles[repointing]
+                lo_version = initializer_data.repointings.lo_repointings[repointing]
+                products_list.extend(process_l3e_lo(lo_parent_file_names, repointing, start_repointing, epoch_delta, lo_elongation, lo_version))
 
             with SwallowExceptionAndLog(f"Exception encountered when processing L3e hi-90 for repointing {repointing}"):
                 hi_parent_file_names = initializer_data.dependencies.get_hi_parents()

imap_l3_processing/glows/l3e/glows_l3e_dependencies.py

@@ -34,8 +34,6 @@ class GlowsL3EDependencies:
     ionization_files: Path
     pipeline_settings: dict
     pipeline_settings_file: Path
-    elongation: dict
-    elongation_file: Path
     repointing_file: Path
     spice_kernels: list[str] = field(default_factory=list)
 
@@ -62,16 +60,8 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection):
 
         energy_grid_lo_dependency = dependencies.get_file_paths(source='glows', descriptor='energy-grid-lo')
         tess_xyz_dependency = dependencies.get_file_paths(source='glows', descriptor='tess-xyz-8')
-        elongation_dependency = dependencies.get_file_paths(source='lo', descriptor='elongation-data')
         energy_grid_lo_path = imap_data_access.download(energy_grid_lo_dependency[0])
         tess_xyz_path = imap_data_access.download(tess_xyz_dependency[0])
-        elongation_path = imap_data_access.download(elongation_dependency[0])
-        with open(elongation_path) as f:
-            elongation_data = {}
-            lines = [line.rstrip('\n') for line in f.readlines()]
-            for line in lines:
-                repointing, elongation = line.split('\t')
-                elongation_data[repointing] = int(elongation)
 
         energy_grid_hi_dependency = dependencies.get_file_paths(source='glows', descriptor='energy-grid-hi')
         energy_grid_hi_path = imap_data_access.download(energy_grid_hi_dependency[0])
@@ -102,8 +92,6 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection):
             ionization_files_path,
             pipeline_settings,
             pipeline_settings_path,
-            elongation_data,
-            elongation_path,
             repoint_file_path
         )
 
@@ -163,7 +151,6 @@ def get_lo_parents(self):
             self.ionization_files.name,
             self.pipeline_settings_file.name,
             self.repointing_file.name,
-            self.elongation_file.name,
             *self.spice_kernels
         ]
 

imap_l3_processing/glows/l3e/glows_l3e_initializer.py

@@ -35,7 +35,6 @@ def get_repointings_to_process(l3d_output: GlowsL3DProcessorOutput, previous_l3d
         pipeline_settings_l3bcde = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='pipeline-settings-l3bcde'))
         energy_grid_lo = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='energy-grid-lo'))
         tess_xyz_8 = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='tess-xyz-8'))
-        elongation_data = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='lo', descriptor='elongation-data'))
         energy_grid_hi = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='energy-grid-hi'))
         energy_grid_ultra = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='energy-grid-ultra'))
         tess_ang_16 = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='tess-ang-16'))
@@ -47,7 +46,6 @@ def get_repointings_to_process(l3d_output: GlowsL3DProcessorOutput, previous_l3d
             AncillaryInput(str(pipeline_settings_l3bcde["file_path"])),
             AncillaryInput(str(energy_grid_lo["file_path"])),
             AncillaryInput(str(tess_xyz_8["file_path"])),
-            AncillaryInput(str(elongation_data["file_path"])),
             AncillaryInput(str(energy_grid_hi["file_path"])),
             AncillaryInput(str(energy_grid_ultra["file_path"])),
             AncillaryInput(str(tess_ang_16["file_path"])),

imap_l3_processing/glows/l3e/glows_l3e_utils.py

@@ -1,5 +1,6 @@
+from __future__ import annotations
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timedelta
 from pathlib import Path
 
 import imap_data_access
@@ -125,3 +126,44 @@ def find_first_updated_cr(new_l3d: Path, old_l3d: str) -> Optional[int]:
         return int(old_l3d_cdf['cr_grid'][-1]) + 1
 
     return None
+
+@dataclass
+class LoL1bNHK:
+    epoch: np.ndarray
+    angle: np.ndarray
+
+    @classmethod
+    def read_from_cdf(cls, path: Path) -> LoL1bNHK:
+        with CDF(str(path)) as cdf:
+            epoch = cdf['epoch'][...]
+            angle = cdf['pcc_coarse_pot_pri'][...]
+        return cls(epoch, angle)
+
+    def get_pivot_angle(self) -> float:
+        if len(self.epoch) == 0:
+            return 90
+        t0 = self.epoch[0]
+        start = t0 + timedelta(hours=3)
+        end = t0 + timedelta(hours=15)
+        start_index, end_index = np.searchsorted(self.epoch, [start, end])
+        angles = self.angle[start_index:end_index]
+        if len(angles) == 0:
+            return 90
+        return np.round(np.median(angles))
+
+def get_lo_pivot_angles(repointings: list[int]) -> dict[int, float]:
+    l1b_results = imap_data_access.query(
+        instrument="lo",
+        data_level="l1b",
+        descriptor="nhk",
+        version="latest",
+    )
+    paths_by_repointing = {f["repointing"]:f["file_path"] for f in l1b_results}
+    result = {}
+    for repointing in repointings:
+        if path := paths_by_repointing.get(repointing):
+            downloaded_path = imap_data_access.download(path)
+            result[repointing] = LoL1bNHK.read_from_cdf(downloaded_path).get_pivot_angle()
+        else:
+            result[repointing] = 90
+    return result

run_local.py

@@ -648,7 +648,6 @@ def run_glows_l3e_with_less_mocks(mock_spiceypy):
         AncillaryInput("imap_glows_speed-3d_19640117_v002.dat"),
         AncillaryInput("imap_glows_sw-eqtr-electrons_19710416_v002.dat"),
         AncillaryInput("imap_glows_tess-xyz-8_20100101_v002.dat"),
-        AncillaryInput("imap_lo_elongation-data_20100101_v001.dat"),
         ScienceInput(l3d_file),
         RepointInput("imap_2001_052_001.repoint.csv"),
     )

tests/glows/l3e/test_glows_l3e_dependencies.py

@@ -26,14 +26,13 @@ def test_fetch_dependencies(self, mock_download):
         mock_sw_eqtr_electrons = Path('sw_eqtr_electrons_sdc_path')
         ionization_files = Path('ionization_files_path')
         mock_pipeline_settings = Path('l3bcde_pipeline_settings.json')
-        mock_lo_elongation_file = Path('lo_elongation_data.dat')
         mock_tess_ang_16 = Path('tess_ang_16_path')
         mock_repoint_file = Path('repoint.csv')
 
         mock_processing_input_collection.get_file_paths.side_effect = [
             [mock_lya_series], [mock_solar_uv_anisotropy], [mock_speed_3d_sw], [mock_density_3d_sw],
             [mock_phion_hydrogen], [mock_sw_eqtr_electrons], [ionization_files], [mock_pipeline_settings],
-            [mock_tess_xyz_8], [mock_energy_grid_lo], [mock_lo_elongation_file],
+            [mock_tess_xyz_8], [mock_energy_grid_lo],
             [mock_energy_grid_hi],
             [mock_energy_grid_ultra], [mock_tess_ang_16],
             [mock_repoint_file]
@@ -50,7 +49,6 @@ def test_fetch_dependencies(self, mock_download):
         mock_ionization_files_path = Mock()
         mock_energy_grid_lo_path = Mock()
         mock_tess_xyz_8_path = Mock()
-        mock_lo_elongation_path = get_test_data_path('glows/imap_lo_elongation-data_20100101_v001.dat')
         mock_energy_grid_hi_path = Mock()
         mock_energy_grid_ultra_path = Mock()
         mock_tess_ang_16_path = Mock()
@@ -59,15 +57,15 @@ def test_fetch_dependencies(self, mock_download):
         mock_download.side_effect = [
             mock_lya_series_path, mock_solar_uv_anisotropy_path, mock_speed_3d_sw_path,
             mock_density_3d_sw_path, mock_phion_hydrogen_path, mock_sw_eqtr_electrons_path, mock_ionization_files_path,
-            fake_pipeline_settings_path, mock_energy_grid_lo_path, mock_tess_xyz_8_path, mock_lo_elongation_path,
+            fake_pipeline_settings_path, mock_energy_grid_lo_path, mock_tess_xyz_8_path,
             mock_energy_grid_hi_path,
             mock_energy_grid_ultra_path, mock_tess_ang_16_path,
             mock_downloaded_repoint_file,
         ]
 
         actual_dependencies = GlowsL3EDependencies.fetch_dependencies(mock_processing_input_collection)
 
-        self.assertEqual(15, mock_processing_input_collection.get_file_paths.call_count)
+        self.assertEqual(14, mock_processing_input_collection.get_file_paths.call_count)
 
         mock_processing_input_collection.get_file_paths.assert_has_calls([
             call(source="glows", descriptor="lya"),
@@ -80,7 +78,6 @@ def test_fetch_dependencies(self, mock_download):
             call(source="glows", descriptor="pipeline-settings-l3bcde"),
             call(source="glows", descriptor="energy-grid-lo"),
             call(source="glows", descriptor="tess-xyz-8"),
-            call(source="lo", descriptor="elongation-data"),
             call(source="glows", descriptor="energy-grid-hi"),
             call(source="glows", descriptor="energy-grid-ultra"),
             call(source="glows", descriptor="tess-ang-16"),
@@ -98,7 +95,6 @@ def test_fetch_dependencies(self, mock_download):
             call(mock_pipeline_settings),
             call(mock_tess_xyz_8),
             call(mock_energy_grid_lo),
-            call(mock_lo_elongation_file),
             call(mock_energy_grid_hi),
             call(mock_energy_grid_ultra),
             call(mock_tess_ang_16),
@@ -135,12 +131,6 @@ def test_fetch_dependencies(self, mock_download):
         self.assertEqual(mock_tess_xyz_8_path, actual_dependencies.tess_xyz_8)
         self.assertEqual(mock_tess_ang_16_path, actual_dependencies.tess_ang16)
         self.assertEqual(fake_pipeline_settings_path, actual_dependencies.pipeline_settings_file)
-        self.assertEqual(365, len(actual_dependencies.elongation))
-        first_dict_value = actual_dependencies.elongation['2010001']
-        last_dict_value = actual_dependencies.elongation['2010365']
-        self.assertEqual(mock_lo_elongation_path, actual_dependencies.elongation_file)
-        self.assertEqual(105, first_dict_value)
-        self.assertEqual(105, last_dict_value)
 
         self.assertEqual(mock_downloaded_repoint_file, actual_dependencies.repointing_file)
 
@@ -213,7 +203,6 @@ def test_get_lo_parents(self):
             "ionization_files.dat",
             "pipeline_settings.json",
             "repointing_file.csv",
-            "elongation.csv"
         ]
 
         self.assertEqual(expected_parent_file_names, dependencies.get_lo_parents())
@@ -269,8 +258,6 @@ def test_rename_dependencies(self, mock_shutil):
                 }
             },
             Path("path/to/some/pipeline_settings_file.csv"),
-            {},
-            Path("path/to/some/elongation_file.csv"),
             Path("repoint.csv")
         )
 
@@ -358,7 +345,5 @@ def _create_l3e_dependencies(self) -> GlowsL3EDependencies:
             ionization_files=Path("some/folder/ionization_files.dat"),
             pipeline_settings_file=Path("some/folder/pipeline_settings.json"),
             pipeline_settings={},
-            elongation_file=Path("some/folder/elongation.csv"),
-            elongation={},
             repointing_file=Path("some/folder/repointing_file.csv"),
         )

tests/glows/l3e/test_glows_l3e_initializer.py

@@ -26,7 +26,6 @@ def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_upl
             'imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf',
             'imap_glows_energy-grid-lo_20200101_v000.cdf',
             'imap_glows_tess-xyz-8_20200101_v000.cdf',
-            'imap_glows_elongation-data_20200101_v000.cdf',
             'imap_glows_energy-grid-hi_20200101_v000.cdf',
             'imap_glows_energy-grid-ultra_20200101_v000.cdf',
             'imap_glows_tess-ang-16_20200101_v000.cdf',
@@ -37,7 +36,6 @@ def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_upl
             create_mock_query_results(['imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf'])[0],
             create_mock_query_results(['imap_glows_energy-grid-lo_20200101_v000.cdf'])[0],
             create_mock_query_results(['imap_glows_tess-xyz-8_20200101_v000.cdf'])[0],
-            create_mock_query_results(['imap_glows_elongation-data_20200101_v000.cdf'])[0],
             create_mock_query_results(['imap_glows_energy-grid-hi_20200101_v000.cdf'])[0],
             create_mock_query_results(['imap_glows_energy-grid-ultra_20200101_v000.cdf'])[0],
             create_mock_query_results(['imap_glows_tess-ang-16_20200101_v000.cdf'])[0],
@@ -93,7 +91,6 @@ def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_upl
             call(table="ancillary", instrument='glows', descriptor='pipeline-settings-l3bcde'),
             call(table="ancillary", instrument='glows', descriptor='energy-grid-lo'),
             call(table="ancillary", instrument='glows', descriptor='tess-xyz-8'),
-            call(table="ancillary", instrument='lo', descriptor='elongation-data'),
             call(table="ancillary", instrument='glows', descriptor='energy-grid-hi'),
             call(table="ancillary", instrument='glows', descriptor='energy-grid-ultra'),
             call(table="ancillary", instrument='glows', descriptor='tess-ang-16'),
@@ -118,7 +115,6 @@ def test_get_repointings_to_process(self, mock_query, mock_get_most_recently_upl
             'imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf',
             'imap_glows_energy-grid-lo_20200101_v000.cdf',
             'imap_glows_tess-xyz-8_20200101_v000.cdf',
-            'imap_glows_elongation-data_20200101_v000.cdf',
             'imap_glows_energy-grid-hi_20200101_v000.cdf',
             'imap_glows_energy-grid-ultra_20200101_v000.cdf',
             'imap_glows_tess-ang-16_20200101_v000.cdf',
@@ -170,7 +166,6 @@ def test_get_repointings_to_process_handles_no_previous_l3d(self, mock_find_firs
             create_mock_query_results(['imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf'])[0],
             create_mock_query_results(['imap_glows_energy-grid-lo_20200101_v000.cdf'])[0],
             create_mock_query_results(['imap_glows_tess-xyz-8_20200101_v000.cdf'])[0],
-            create_mock_query_results(['imap_glows_elongation-data_20200101_v000.cdf'])[0],
             create_mock_query_results(['imap_glows_energy-grid-hi_20200101_v000.cdf'])[0],
             create_mock_query_results(['imap_glows_energy-grid-ultra_20200101_v000.cdf'])[0],
             create_mock_query_results(['imap_glows_tess-ang-16_20200101_v000.cdf'])[0],