Skip to content

Commit eff9e94

Browse files
author
Menlo Innovations - CAVA Project
committed
Harrison - GSTU/EYEA - GLOWS: Pull in latest instrument team ancillary files and add L2 -> L3e integration test
1 parent 24bedc4 commit eff9e94

8 files changed

Lines changed: 169 additions & 37 deletions

‎imap_l3_processing/glows/l3a/utils.py‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import numpy as np
66
from spacepy.pycdf import CDF
77

8-
from imap_l3_processing.constants import TTJ2000_EPOCH
98
from imap_l3_processing.glows.l3a.models import GlowsL2Data, GlowsL2LightCurve, GlowsLatLon, GlowsL3LightCurve, XYZ, \
109
GlowsL2Header
1110
from imap_l3_processing.models import InputMetadata
@@ -14,10 +13,9 @@
1413
def read_l2_glows_data(cdf: CDF) -> GlowsL2Data:
1514
assert 1 == cdf['photon_flux'].shape[0], "Level 2 file should have only one histogram"
1615

17-
flags = cdf['histogram_flag_array'][0]
1816
light_curve = GlowsL2LightCurve(photon_flux=cdf['photon_flux'][0],
1917
spin_angle=cdf['spin_angle'][0],
20-
histogram_flag_array=flags,
18+
histogram_flag_array=cdf['histogram_flag_array'][0],
2119
exposure_times=cdf['exposure_times'][0],
2220
flux_uncertainties=cdf['flux_uncertainties'][0],
2321
raw_histogram=cdf['raw_histograms'][0],
@@ -40,12 +38,9 @@ def read_l2_glows_data(cdf: CDF) -> GlowsL2Data:
4038
y=cdf['spacecraft_velocity_std_dev'][0, 1],
4139
z=cdf['spacecraft_velocity_std_dev'][0, 2], )
4240

43-
start_time = (TTJ2000_EPOCH + timedelta(seconds=cdf['start_time'][0] / 1e9))
44-
end_time = (TTJ2000_EPOCH + timedelta(seconds=cdf['end_time'][0] / 1e9))
45-
4641
return GlowsL2Data(identifier=cdf['identifier'][0],
47-
start_time=str(start_time),
48-
end_time=str(end_time),
42+
start_time=cdf['start_time'][0],
43+
end_time=cdf['end_time'][0],
4944
daily_lightcurve=light_curve,
5045
number_of_bins=cdf['number_of_bins'][...],
5146
spin_axis_orientation_average=spin_axis_average,

‎tests/glows/l3a/test_utils.py‎

Lines changed: 16 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -15,44 +15,41 @@
1515

1616
class TestUtils(unittest.TestCase):
1717
def test_reading_l2_glows_data_into_models(self):
18-
cdf = CDF(str(get_test_data_path('glows/imap_glows_l2_hist_20130908-repoint01000_v001.cdf')))
18+
cdf = CDF(str(get_test_data_path('glows/imap_glows_l2_hist_20251224-repoint00088_v001.cdf')))
1919

2020
glows_l2_data: GlowsL2Data = read_l2_glows_data(cdf)
2121

22-
expected_start_time = datetime(2013, 9, 8, 8, 52, 14)
23-
self.assertEqual(str(expected_start_time), glows_l2_data["start_time"])
24-
25-
expected_end_time = datetime(2013, 9, 9, 4, 58, 14)
26-
self.assertEqual(str(expected_end_time), glows_l2_data["end_time"])
22+
self.assertEqual("2025-12-24T11:36:21.588935256", glows_l2_data["start_time"])
23+
self.assertEqual("2025-12-25T09:39:47.969074488", glows_l2_data["end_time"])
2724

2825
self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["photon_flux"].shape)
29-
self.assertEqual(681.6, glows_l2_data["daily_lightcurve"]["photon_flux"][0])
30-
self.assertEqual(678.4, glows_l2_data["daily_lightcurve"]["photon_flux"][-1])
26+
self.assertEqual(887.6062785692615, glows_l2_data["daily_lightcurve"]["photon_flux"][0])
27+
self.assertEqual(872.260576384695, glows_l2_data["daily_lightcurve"]["photon_flux"][-1])
3128

3229
self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["flux_uncertainties"].shape)
33-
self.assertEqual(5.828, glows_l2_data["daily_lightcurve"]["flux_uncertainties"][0])
30+
self.assertEqual(6.338751356172933, glows_l2_data["daily_lightcurve"]["flux_uncertainties"][0])
3431

3532
self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["exposure_times"].shape)
36-
self.assertEqual(2.007e+01, glows_l2_data["daily_lightcurve"]["exposure_times"][0])
33+
self.assertEqual(22.090875733333327, glows_l2_data["daily_lightcurve"]["exposure_times"][0])
3734

3835
self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["spin_angle"].shape)
39-
self.assertEqual(0.072, glows_l2_data["daily_lightcurve"]["spin_angle"][0])
36+
self.assertEqual(0.08840215463658296, glows_l2_data["daily_lightcurve"]["spin_angle"][0])
4037

4138
self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["histogram_flag_array"].shape)
42-
self.assertEqual(np.float64, glows_l2_data["daily_lightcurve"]["histogram_flag_array"].dtype)
43-
self.assertEqual(np.uint8(4), glows_l2_data["daily_lightcurve"]["histogram_flag_array"][84])
39+
self.assertEqual(np.int64, glows_l2_data["daily_lightcurve"]["histogram_flag_array"].dtype)
40+
self.assertEqual(np.uint8(0), glows_l2_data["daily_lightcurve"]["histogram_flag_array"][84])
4441

4542
self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["ecliptic_lat"].shape)
4643
self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["ecliptic_lon"].shape)
47-
self.assertEqual(75.000, glows_l2_data["daily_lightcurve"]["ecliptic_lat"][0])
48-
self.assertEqual(161.823, glows_l2_data["daily_lightcurve"]["ecliptic_lon"][0])
44+
self.assertEqual(75.04420784333895, glows_l2_data["daily_lightcurve"]["ecliptic_lat"][0])
45+
self.assertEqual(268.7416390634532, glows_l2_data["daily_lightcurve"]["ecliptic_lon"][0])
4946

5047
self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["raw_histogram"].shape)
51-
self.assertEqual(13677, glows_l2_data["daily_lightcurve"]["raw_histogram"][0])
48+
self.assertEqual(19608, glows_l2_data["daily_lightcurve"]["raw_histogram"][0])
5249

5350
self.assertEqual({
54-
"lon": 162.092,
55-
"lat": 0.000
51+
"lon": -2.9634262485046428,
52+
"lat": 0.06861815065029926
5653
}, glows_l2_data["spin_axis_orientation_average"])
5754

5855
self.assertEqual(cdf['identifier'][0], glows_l2_data['identifier'])
@@ -68,7 +65,7 @@ def test_reading_l2_glows_data_into_models(self):
6865
self.assertEqual(cdf['pulse_length_std_dev'][0], glows_l2_data['pulse_length_std_dev'])
6966
self.assertEqual(cdf['position_angle_offset_average'][0], glows_l2_data['position_angle_offset_average'])
7067
self.assertEqual(cdf['position_angle_offset_std_dev'][0], glows_l2_data['position_angle_offset_std_dev'])
71-
self.assertEqual("imap_glows_l2_hist_20130908-repoint01000_v001.cdf", glows_l2_data['l2_file_name'])
68+
self.assertEqual("imap_glows_l2_hist_20251224-repoint00088_v001.cdf", glows_l2_data['l2_file_name'])
7269

7370
self.assert_equal_xyz(cdf, "spacecraft_location_average", glows_l2_data)
7471
self.assert_equal_xyz(cdf, "spacecraft_location_std_dev", glows_l2_data)

‎tests/integration/test_data/glows/imap_glows_map-of-extra-helio-bckgrd_20251112_v001.dat‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:3b8a2ca14cfb45eda5d717c0830f214cedb4ebcbe2f1cd374b4435945a139590
3+
size 232289

‎tests/integration/test_data/glows/imap_glows_pipeline-settings_20251112_v001.json‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
{
2+
"description": "Settings for ground-processing pipeline for IMAP/GLOWS instrument",
3+
"version": "1.0",
4+
"date_of_creation_yyyymmdd": "20250703",
5+
"filter_based_on_daily_statistical_error": {
6+
"n_sigma_threshold_lower": 3.0,
7+
"n_sigma_threshold_upper": 3.0
8+
},
9+
"filter_based_on_comparison_of_spin_periods": {
10+
"relative_difference_threshold": 7.0e-4
11+
},
12+
"filter_based_on_temperature_std_dev": {
13+
"std_dev_threshold__celsius_deg": 2.03
14+
},
15+
"filter_based_on_hv_voltage_std_dev": {
16+
"std_dev_threshold__volt": 50.0
17+
},
18+
"filter_based_on_spin_period_std_dev": {
19+
"std_dev_threshold__sec": 0.033333
20+
},
21+
"filter_based_on_pulse_length_std_dev": {
22+
"std_dev_threshold__usec": 1.0
23+
},
24+
"filter_based_on_maps": {
25+
"angular_radius_for_excl_regions__deg": 0.5
26+
},
27+
"active_bad_time_flags": {
28+
"is_pps_missing": true,
29+
"is_time_status_missing": true,
30+
"is_phase_missing": true,
31+
"is_spin_period_missing": true,
32+
"is_overexposed": false,
33+
"is_direct_event_non_monotonic": true,
34+
"is_night": true,
35+
"is_hv_test_in_progress": false,
36+
"is_test_pulse_in_progress": true,
37+
"is_memory_error_detected": true,
38+
"is_generated_on_ground": true,
39+
"is_beyond_daily_statistical_error": false,
40+
"is_temperature_std_dev_beyond_threshold": true,
41+
"is_hv_voltage_std_dev_beyond_threshold": true,
42+
"is_spin_period_std_dev_beyond_threshold": true,
43+
"is_pulse_length_std_dev_beyond_threshold": true,
44+
"is_spin_period_difference_beyond_threshold": true
45+
},
46+
"active_bad_angle_flags": {
47+
"is_close_to_uv_source": true,
48+
"is_inside_excluded_region": true,
49+
"is_excluded_by_instr_team": true,
50+
"is_suspected_transient": true
51+
},
52+
"sunrise_offset": 0,
53+
"sunset_offset": 0,
54+
"l3a_nominal_number_of_bins": 90
55+
}

‎tests/integration/test_data/glows/imap_glows_time-dep-bckgrd_20251112_v001.dat‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:6f123a73486a209cd102e66f64816214d2f8b1ba04bf12fea81e36251477829a
3+
size 44290

‎tests/integration/test_glows_processor_integration.py‎

Lines changed: 81 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -468,9 +468,87 @@ def determine_spice_offset(date: datetime):
468468
processor = GlowsProcessor(processing_input, input_metadata)
469469
processor.process()
470470

471+
@run_periodically(timedelta(days=14))
472+
def test_glows_l3abcde_with_prod_l2(self):
473+
ancillary_file_paths = [
474+
GLOWS_TEST_DATA / "imap_glows_calibration-data_20000101_v003.dat",
475+
GLOWS_TEST_DATA / "imap_glows_map-of-extra-helio-bckgrd_20251112_v001.dat",
476+
GLOWS_TEST_DATA / "imap_glows_pipeline-settings_20251112_v001.json",
477+
GLOWS_TEST_DATA / "imap_glows_time-dep-bckgrd_20251112_v001.dat"
478+
]
479+
480+
prod_data_folder = get_run_local_data_path("glows_prod_data")
481+
482+
l2_paths = list((prod_data_folder / "imap/glows/l2").rglob("*.cdf"))
483+
input_files = l2_paths + ancillary_file_paths
484+
485+
l3a_integration_data_dir = get_run_local_data_path("test_glows_l3a_with_prod_l2")
486+
with (mock_imap_data_access(l3a_integration_data_dir, input_files)):
487+
for i, l2_path in enumerate(l2_paths):
488+
l2_science_file_path = ScienceFilePath(l2_path)
489+
l2_science_input = ScienceInput(l2_path.name)
490+
input_files.extend(ancillary_file_paths)
491+
492+
start_date, end_date = l2_science_input.get_time_range()
493+
494+
input_metadata = InputMetadata(
495+
instrument="glows",
496+
data_level="l3a",
497+
start_date=start_date,
498+
end_date=end_date,
499+
version="v001",
500+
descriptor="hist",
501+
repointing=l2_science_file_path.repointing,
502+
)
503+
504+
ancillary_inputs = [AncillaryInput(ancillary.name) for ancillary in ancillary_file_paths]
505+
processing_inputs = [l2_science_input, *ancillary_inputs]
506+
processor = GlowsProcessor(ProcessingInputCollection(*processing_inputs), input_metadata)
507+
508+
try:
509+
_ = processor.process()
510+
except Exception as e:
511+
print(f"Processing L3a day {start_date} failed! Reason: {e}")
512+
513+
l3bcde_ancillary_inputs = [
514+
GLOWS_TEST_DATA / "imap_glows_uv-anisotropy-1CR_20100101_v001.json",
515+
GLOWS_TEST_DATA / "imap_glows_WawHelioIonMP_20100101_v001.json",
516+
GLOWS_TEST_DATA / "imap_glows_bad-days-list_20100101_v001.dat",
517+
GLOWS_TEST_DATA / "imap_glows_pipeline-settings-l3bcde_20100101_v003.json",
518+
GLOWS_TEST_DATA / "imap_glows_plasma-speed-2010a_20100101_v001.dat",
519+
GLOWS_TEST_DATA / "imap_glows_proton-density-2010a_20100101_v001.dat",
520+
GLOWS_TEST_DATA / "imap_glows_uv-anisotropy-2010a_20100101_v001.dat",
521+
GLOWS_TEST_DATA / "imap_glows_photoion-2010a_20100101_v001.dat",
522+
GLOWS_TEST_DATA / "imap_glows_lya-2010a_20100101_v001.dat",
523+
GLOWS_TEST_DATA / "imap_glows_electron-density-2010a_20100101_v001.dat",
524+
GLOWS_TEST_DATA / "imap_glows_ionization-files_20100101_v001.dat",
525+
GLOWS_TEST_DATA / "imap_glows_energy-grid-lo_20100101_v001.dat",
526+
GLOWS_TEST_DATA / "imap_glows_tess-xyz-8_20100101_v001.dat",
527+
GLOWS_TEST_DATA / "imap_lo_elongation-data_20100101_v001.dat",
528+
GLOWS_TEST_DATA / "imap_glows_energy-grid-hi_20100101_v001.dat",
529+
GLOWS_TEST_DATA / "imap_glows_energy-grid-ultra_20100101_v001.dat",
530+
GLOWS_TEST_DATA / "imap_glows_tess-ang-16_20100101_v001.dat",
531+
GLOWS_TEST_DATA / "imap_2026_090_01.repoint"
532+
]
533+
l3a_inputs = list((l3a_integration_data_dir / "imap/glows/l3a").rglob("*.cdf"))
534+
prod_spice_inputs = list((prod_data_folder / "imap/spice/repoint").rglob("*"))
535+
536+
logging.basicConfig(force=True, level=logging.INFO,
537+
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
538+
539+
input_paths = l3bcde_ancillary_inputs + l3a_inputs + prod_spice_inputs
540+
with mock_imap_data_access(get_run_local_data_path("test_glows_l3bcde_with_prod_l2"), input_paths):
541+
processing_input = ProcessingInputCollection(RepointInput("imap_2026_090_01.repoint"))
542+
input_metadata = InputMetadata(instrument="glows", data_level="l3b", descriptor="ion-rate-profile",
543+
version="v001", start_date=datetime(2000, 1, 1),
544+
end_date=datetime(2000, 1, 1))
545+
546+
processor = GlowsProcessor(processing_input, input_metadata)
547+
processor.process()
548+
471549
@staticmethod
472550
def _fill_official_l2_cdf_with_json_values(output_folder: Path) -> Path:
473-
official_l2_path = get_test_data_path("glows/imap_glows_l2_hist_20130908-repoint01000_v001.cdf")
551+
official_l2_path = get_test_data_path("glows/imap_glows_l2_hist_20251224-repoint00088_v001.cdf")
474552
json_file_path = get_test_instrument_team_data_path("glows/imap_glows_l2_20130908085214_orbX_modX_p_v00.json")
475553

476554
new_file_path = output_folder / official_l2_path.name
@@ -486,8 +564,8 @@ def _fill_official_l2_cdf_with_json_values(output_folder: Path) -> Path:
486564
epoch = start_of_epoch_window + epoch_window / 2
487565

488566
cdf["epoch"][0] = epoch
489-
cdf['start_time'][0] = (start_of_epoch_window - TTJ2000_EPOCH).total_seconds() * 1e9
490-
cdf['end_time'][0] = (end_of_epoch_window - TTJ2000_EPOCH).total_seconds() * 1e9
567+
cdf['start_time'][0] = instrument_data["start_time"]
568+
cdf['end_time'][0] = instrument_data["end_time"]
491569

492570
lightcurve_vars = [
493571
"spin_angle",

‎tests/periodically_run_tests.json‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
{
2-
"test_l3bcde_first_time_processing": "2026-03-19T13:31:57.152750",
3-
"test_spectral_fit_against_validation_data": "2026-03-23T10:37:49.828951",
4-
"test_calculate_pickup_ions_with_minimize": "2026-03-23T10:40:37.460017",
5-
"test_ultra_all_sp_maps": "2026-03-23T10:39:56.176185",
2+
"test_l3bcde_first_time_processing": "2026-04-06T11:46:43.319293",
3+
"test_spectral_fit_against_validation_data": "2026-04-06T11:46:11.333215",
4+
"test_calculate_pickup_ions_with_minimize": "2026-04-06T11:47:38.436058",
5+
"test_ultra_all_sp_maps": "2026-04-06T11:47:00.156330",
66
"test_ultra_combined_maps": null,
7-
"test_ultra_combined_nsp_maps": "2026-03-20T14:32:30.413629",
8-
"test_ultra_combined_sp_maps": "2026-03-20T14:33:10.459309",
9-
"test_l3bcde_reprocessing": "2026-03-12T17:39:34.585134"
7+
"test_ultra_combined_nsp_maps": "2026-04-06T11:47:10.063085",
8+
"test_ultra_combined_sp_maps": "2026-04-06T11:47:22.601935",
9+
"test_l3bcde_reprocessing": "2026-04-06T11:46:43.618159",
10+
"test_glows_l3abcde_with_prod_l2": "2026-04-06T11:51:05.383052"
1011
}

‎tests/test_data/glows/imap_glows_l2_hist_20251224-repoint00088_v001.cdf‎

192 KB
Binary file not shown.

0 commit comments

Comments
 (0)