Skip to content

Commit 789e728

Browse files
author
Menlo Innovations - CAVA Project
committed
Merge remote-tracking branch 'origin/main'
2 parents f22cada + 5349680 commit 789e728

9 files changed

Lines changed: 128 additions & 47 deletions

File tree

‎imap_processing/cdf/cdf_utils.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,20 @@ def write_cdf(file_path: str, data: DataProduct, attribute_manager: ImapAttribut
2222

2323
for data_product in data.to_data_product_variables():
2424
var_name = data_product.name
25-
cdf.new(var_name, data=data_product.value,
25+
variable_attributes = attribute_manager.get_variable_attributes(var_name)
26+
data_array = np.asarray(data_product.value)
27+
if 'FILLVAL' in variable_attributes and np.issubdtype(data_array.dtype, np.floating):
28+
data_array = np.where(np.isnan(data_array), variable_attributes["FILLVAL"], data_array)
29+
cdf.new(var_name, data_array,
2630
recVary=data_product.record_varying,
2731
type=data_product.cdf_data_type)
28-
for k, v in attribute_manager.get_variable_attributes(var_name).items():
32+
for k, v in variable_attributes.items():
2933
if k == 'DEPEND_0' and v == '':
3034
continue
31-
cdf[var_name].attrs[k] = v
35+
if k == 'FILLVAL' and data_product.cdf_data_type is not None:
36+
cdf[var_name].attrs.new(k, v, data_product.cdf_data_type)
37+
else:
38+
cdf[var_name].attrs[k] = v
3239

3340

3441
def read_variable(var: pycdf.Var) -> np.ndarray:

‎imap_processing/swe/l3/models.py‎

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
from typing import TypedDict
33

44
import numpy as np
5+
from spacepy import pycdf
56

67
from imap_processing.models import DataProduct, DataProductVariable
78

@@ -57,23 +58,30 @@ class SweL3Data(DataProduct):
5758

5859
def to_data_product_variables(self) -> list[DataProductVariable]:
5960
return [
60-
DataProductVariable(EPOCH_CDF_VAR_NAME, value=self.epoch),
61+
DataProductVariable(EPOCH_CDF_VAR_NAME, value=self.epoch, cdf_data_type=pycdf.const.CDF_TIME_TT2000),
6162
DataProductVariable(EPOCH_DELTA_CDF_VAR_NAME,
62-
value=[delta.total_seconds() * 1e9 for delta in self.epoch_delta]),
63-
DataProductVariable(ENERGY_CDF_VAR_NAME, value=self.energy, record_varying=False),
64-
DataProductVariable(ENERGY_DELTA_PLUS_CDF_VAR_NAME, value=self.energy_delta_plus, record_varying=False),
65-
DataProductVariable(ENERGY_DELTA_MINUS_CDF_VAR_NAME, value=self.energy_delta_minus, record_varying=False),
66-
DataProductVariable(PITCH_ANGLE_CDF_VAR_NAME, value=self.pitch_angle, record_varying=False),
67-
DataProductVariable(PITCH_ANGLE_DELTA_CDF_VAR_NAME, value=self.pitch_angle_delta, record_varying=False),
68-
DataProductVariable(FLUX_BY_PITCH_ANGLE_CDF_VAR_NAME, value=self.flux_by_pitch_angle),
63+
value=[delta.total_seconds() * 1e9 for delta in self.epoch_delta],
64+
cdf_data_type=pycdf.const.CDF_INT8),
65+
DataProductVariable(ENERGY_CDF_VAR_NAME, value=self.energy, cdf_data_type=pycdf.const.CDF_REAL4,
66+
record_varying=False),
67+
DataProductVariable(ENERGY_DELTA_PLUS_CDF_VAR_NAME, value=self.energy_delta_plus,
68+
cdf_data_type=pycdf.const.CDF_REAL4, record_varying=False),
69+
DataProductVariable(ENERGY_DELTA_MINUS_CDF_VAR_NAME, value=self.energy_delta_minus,
70+
cdf_data_type=pycdf.const.CDF_REAL4, record_varying=False),
71+
DataProductVariable(PITCH_ANGLE_CDF_VAR_NAME, value=self.pitch_angle, cdf_data_type=pycdf.const.CDF_REAL4,
72+
record_varying=False),
73+
DataProductVariable(PITCH_ANGLE_DELTA_CDF_VAR_NAME, value=self.pitch_angle_delta,
74+
cdf_data_type=pycdf.const.CDF_REAL4, record_varying=False),
75+
DataProductVariable(FLUX_BY_PITCH_ANGLE_CDF_VAR_NAME, value=self.flux_by_pitch_angle,
76+
cdf_data_type=pycdf.const.CDF_REAL4),
6977
DataProductVariable(PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME,
70-
value=self.phase_space_density_by_pitch_angle),
78+
value=self.phase_space_density_by_pitch_angle, cdf_data_type=pycdf.const.CDF_REAL4),
7179
DataProductVariable(ENERGY_SPECTRUM_CDF_VAR_NAME,
72-
value=self.energy_spectrum),
80+
value=self.energy_spectrum, cdf_data_type=pycdf.const.CDF_REAL4),
7381
DataProductVariable(ENERGY_SPECTRUM_INBOUND_CDF_VAR_NAME,
74-
value=self.energy_spectrum_inbound),
82+
value=self.energy_spectrum_inbound, cdf_data_type=pycdf.const.CDF_REAL4),
7583
DataProductVariable(ENERGY_SPECTRUM_OUTBOUND_CDF_VAR_NAME,
76-
value=self.energy_spectrum_outbound),
84+
value=self.energy_spectrum_outbound, cdf_data_type=pycdf.const.CDF_REAL4),
7785
]
7886

7987

‎run_local.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -289,9 +289,9 @@ def create_hit_direct_event_cdf():
289289

290290
if "swe_pitch_angels" in sys.argv:
291291
dependencies = SweL3Dependencies.from_file_paths(
292-
get_test_data_path("swe/imap_swe_l2_sci-with-ace-data_19990609_v002.cdf"),
293-
get_test_data_path("swe/imap_mag_l1d_mago-normal_19990609_v001.cdf"),
294-
get_test_data_path("swe/imap_swapi_l3a_proton-sw_19990609_v001.cdf"),
292+
get_test_data_path("swe/imap_swe_l2_sci-with-ace-data_19990906_v002.cdf"),
293+
get_test_data_path("swe/imap_mag_l1d_mago-normal_19990906_v001.cdf"),
294+
get_test_data_path("swe/imap_swapi_l3a_proton-sw_19990906_v001.cdf"),
295295
get_test_data_path("swe/example_swe_config.json"),
296296
)
297297
print(create_swe_pitch_angle_cdf(dependencies))

‎scripts/swe/create_fake_l2_cdf.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,9 @@ def replace_variable(cdf: CDF, variable_name: str, new_values: np.ndarray):
9494
l1_hdf_path = path.parent.parent.parent / "instrument_team_data/swe/ACE_LV1_1999-159.swepam.hdf"
9595
l2_hdf_path = path.parent.parent.parent / "instrument_team_data/swe/swepam-nswe-1999-159.v1-02.hdf"
9696

97-
l2_swe_cdf_file_path = path.parent.parent.parent / "tests" / "test_data" / "swe" / "imap_swe_l2_sci-with-ace-data_19990609_v002.cdf"
98-
mag_file_path = path.parent.parent.parent / "tests" / "test_data" / "swe" / "imap_mag_l1d_mago-normal_19990609_v001.cdf"
99-
swapi_file_path = path.parent.parent.parent / "tests" / "test_data" / "swe" / "imap_swapi_l3a_proton-sw_19990609_v001.cdf"
97+
l2_swe_cdf_file_path = path.parent.parent.parent / "tests" / "test_data" / "swe" / "imap_swe_l2_sci-with-ace-data_19990906_v002.cdf"
98+
mag_file_path = path.parent.parent.parent / "tests" / "test_data" / "swe" / "imap_mag_l1d_mago-normal_19990906_v001.cdf"
99+
swapi_file_path = path.parent.parent.parent / "tests" / "test_data" / "swe" / "imap_swapi_l3a_proton-sw_19990906_v001.cdf"
100100

101101
mag_file_path.unlink(missing_ok=True)
102102
swapi_file_path.unlink(missing_ok=True)

‎tests/cdf/test_cdf_utils.py‎

Lines changed: 65 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,14 @@ class TestCdfUtils(TempFileTestCase):
1414
def test_write_cdf(self):
1515
path = str(self.temp_directory / "write_cdf.cdf")
1616
data = TestDataProduct()
17-
regular_var, time_var, non_rec_varying_var = data.to_data_product_variables()
17+
var_without_explicit_type, int_var, float_var, time_var, non_rec_varying_var = data.to_data_product_variables()
1818
attribute_manager = Mock(spec=ImapAttributeManager)
1919
attribute_manager.get_global_attributes.return_value = {"global1": "global_val1", "global2": "global_val2"}
2020
attribute_manager.get_variable_attributes.side_effect = [
21-
{"variable_attr1": "var_val1", "variable_attr2": "var_val2"},
22-
{"variable_attr3": "var_val3", "variable_attr4": "var_val4"},
21+
{"variable_attr1": "var_val1"},
22+
{"variable_attr1": "var_val1", "FILLVAL": -9223372036854775808},
23+
{"variable_attr1": "var_val1", "FILLVAL": -1e31},
24+
{"variable_attr3": "var_val3", "FILLVAL": -9223372036854775808},
2325
{"variable_attr5": "var_val5", "variable_attr6": "var_val6"},
2426
]
2527

@@ -36,15 +38,30 @@ def test_write_cdf(self):
3638
self.assertEqual('global_val1', actual_cdf.attrs['global1'][...][0])
3739
self.assertEqual('global_val2', actual_cdf.attrs['global2'][...][0])
3840

39-
np.testing.assert_array_equal(regular_var.value, actual_cdf[regular_var.name][...])
40-
self.assertEqual('var_val1', actual_cdf[regular_var.name].attrs['variable_attr1'])
41-
self.assertEqual('var_val2', actual_cdf[regular_var.name].attrs['variable_attr2'])
42-
self.assertEqual(pycdf.const.CDF_INT8.value, actual_cdf[regular_var.name].type())
43-
self.assertTrue(actual_cdf[regular_var.name].rv())
44-
np.testing.assert_array_equal(time_var.value, actual_cdf.raw_var(time_var.name))
41+
np.testing.assert_array_equal(var_without_explicit_type.value,
42+
actual_cdf[var_without_explicit_type.name][...])
43+
self.assertEqual('var_val1', actual_cdf[var_without_explicit_type.name].attrs['variable_attr1'])
44+
self.assertEqual(pycdf.const.CDF_INT8.value, actual_cdf[var_without_explicit_type.name].type())
45+
self.assertTrue(actual_cdf[var_without_explicit_type.name].rv())
46+
47+
np.testing.assert_array_equal(int_var.value, actual_cdf.raw_var(int_var.name))
48+
self.assertEqual('var_val1', actual_cdf[int_var.name].attrs['variable_attr1'])
49+
self.assertEqual(-9223372036854775808, actual_cdf[int_var.name].attrs['FILLVAL'])
50+
self.assertEqual(pycdf.const.CDF_INT8.value, actual_cdf[int_var.name].attrs.type("FILLVAL"))
51+
self.assertEqual(pycdf.const.CDF_INT8.value, actual_cdf[int_var.name].type())
52+
self.assertTrue(actual_cdf[int_var.name].rv())
53+
54+
np.testing.assert_array_equal(float_var.value, actual_cdf.raw_var(float_var.name))
55+
self.assertEqual('var_val1', actual_cdf[float_var.name].attrs['variable_attr1'])
56+
self.assertEqual(-1e31, actual_cdf[float_var.name].attrs['FILLVAL'])
57+
self.assertEqual(pycdf.const.CDF_REAL4.value, actual_cdf[float_var.name].attrs.type("FILLVAL"))
58+
self.assertEqual(pycdf.const.CDF_REAL4.value, actual_cdf[float_var.name].type())
59+
self.assertTrue(actual_cdf[float_var.name].rv())
4560

61+
np.testing.assert_array_equal(time_var.value, actual_cdf.raw_var(time_var.name))
4662
self.assertEqual('var_val3', actual_cdf[time_var.name].attrs['variable_attr3'])
47-
self.assertEqual('var_val4', actual_cdf[time_var.name].attrs['variable_attr4'])
63+
self.assertEqual(datetime(9999, 12, 31, 23, 59, 59, 999999), actual_cdf[time_var.name].attrs['FILLVAL'])
64+
self.assertEqual(pycdf.const.CDF_TIME_TT2000.value, actual_cdf[time_var.name].attrs.type("FILLVAL"))
4865
self.assertEqual(pycdf.const.CDF_TIME_TT2000.value, actual_cdf[time_var.name].type())
4966
self.assertTrue(actual_cdf[time_var.name].rv())
5067

@@ -74,13 +91,45 @@ def test_write_cdf_trims_numbers_in_logical_source_when_fetching_global_metadata
7491
self.assertEqual('global_val1', str(actual_cdf.attrs['global1']))
7592
self.assertEqual('global_val2', str(actual_cdf.attrs['global2']))
7693

94+
def test_write_cdf_replaces_nan_with_fill_value(self):
95+
epoch_data = [datetime(2025, 3, 7, 17, 0)]
96+
97+
class DataProductWithNan(DataProduct):
98+
def __init__(self):
99+
self.input_metadata = Mock()
100+
101+
def to_data_product_variables(self) -> list[DataProductVariable]:
102+
return [
103+
DataProductVariable("epoch", epoch_data, pycdf.const.CDF_TIME_TT2000),
104+
DataProductVariable("float_var", np.array([3, 5, np.nan, 9, np.nan, np.nan]),
105+
pycdf.const.CDF_REAL4),
106+
]
107+
108+
path = str(self.temp_directory / "write_cdf.cdf")
109+
data = DataProductWithNan()
110+
attribute_manager = Mock(spec=ImapAttributeManager)
111+
attribute_manager.get_global_attributes.return_value = {}
112+
attribute_manager.get_variable_attributes.side_effect = [
113+
{"VAR_NAME": "epoch", "FILLVAL": datetime.fromisoformat("9999-12-31T23:59:59.999999999")},
114+
{"VAR_NAME": "float_var", "FILLVAL": -1e31},
115+
]
116+
117+
write_cdf(path, data, attribute_manager)
118+
with pycdf.CDF(path) as actual_cdf:
119+
self.assertFalse(np.any(np.isnan(actual_cdf["float_var"][...])))
120+
np.testing.assert_array_equal(np.array([3, 5, -1e31, 9, -1e31, -1e31], dtype=np.float32),
121+
actual_cdf["float_var"][...], strict=True)
122+
np.testing.assert_array_equal(actual_cdf["epoch"][...], np.array(epoch_data), strict=True)
123+
77124
def test_does_not_write_depend_0_variable_attribute_if_it_is_empty(self):
78125
path = str(self.temp_directory / "write_cdf.cdf")
79126
data = TestDataProduct()
80-
regular_var, time_var, non_rec_varying_var = data.to_data_product_variables()
127+
_, _, regular_var, time_var, non_rec_varying_var = data.to_data_product_variables()
81128
attribute_manager = Mock(spec=ImapAttributeManager)
82129
attribute_manager.get_global_attributes.return_value = {"global1": "global_val1", "global2": "global_val2"}
83130
attribute_manager.get_variable_attributes.side_effect = [
131+
{"ignored_attr": "var_val3", "variable_attr4": "var_val4", "DEPEND_0": ""},
132+
{"ignored_attr": "var_val3", "variable_attr4": "var_val4", "DEPEND_0": ""},
84133
{"variable_attr1": "var_val1", "variable_attr2": "var_val2", "DEPEND_0": "epoch"},
85134
{"variable_attr3": "var_val3", "variable_attr4": "var_val4", "DEPEND_0": ""},
86135
{"variable_attr5": "var_val5", "variable_attr6": "var_val6", "DEPEND_0": ""},
@@ -114,7 +163,9 @@ def __init__(self):
114163

115164
def to_data_product_variables(self) -> list[DataProductVariable]:
116165
return [
117-
DataProductVariable("var1", np.arange(0, 10)),
118-
DataProductVariable("var2", np.arange(10, 20), pycdf.const.CDF_TIME_TT2000),
119-
DataProductVariable("var3", 100, record_varying=False)
166+
DataProductVariable("var_without_explicit_type", np.arange(0, 10)),
167+
DataProductVariable("int_var", np.arange(0, 10), pycdf.const.CDF_INT8),
168+
DataProductVariable("float_var", np.arange(0, 10), pycdf.const.CDF_REAL4),
169+
DataProductVariable("time_var", np.arange(10, 20), pycdf.const.CDF_TIME_TT2000),
170+
DataProductVariable("non_record_varying", 100, record_varying=False)
120171
]

‎tests/swe/l3/science/test_models.py‎

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
from datetime import timedelta
33
from unittest.mock import Mock
44

5+
from spacepy import pycdf
6+
from spacepy.pycdf.const import CDF_TIME_TT2000, CDF_INT8
7+
58
from imap_processing.swe.l3.models import SweL3Data, EPOCH_CDF_VAR_NAME, EPOCH_DELTA_CDF_VAR_NAME, ENERGY_CDF_VAR_NAME, \
69
ENERGY_DELTA_PLUS_CDF_VAR_NAME, ENERGY_DELTA_MINUS_CDF_VAR_NAME, PITCH_ANGLE_CDF_VAR_NAME, \
710
PITCH_ANGLE_DELTA_CDF_VAR_NAME, FLUX_BY_PITCH_ANGLE_CDF_VAR_NAME, PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME, \
@@ -43,18 +46,30 @@ def test_data_to_product_variables(self):
4346

4447
variables = iter(variables)
4548
# @formatter:off
46-
self.assert_variable_attributes(next(variables), epoch, EPOCH_CDF_VAR_NAME)
47-
self.assert_variable_attributes(next(variables), [86400*1e9, 40*1e9], EPOCH_DELTA_CDF_VAR_NAME)
48-
self.assert_variable_attributes(next(variables), energy, ENERGY_CDF_VAR_NAME, expected_record_varying=False)
49-
self.assert_variable_attributes(next(variables), energy_delta_plus, ENERGY_DELTA_PLUS_CDF_VAR_NAME, expected_record_varying=False)
50-
self.assert_variable_attributes(next(variables), energy_delta_minus, ENERGY_DELTA_MINUS_CDF_VAR_NAME, expected_record_varying=False)
51-
self.assert_variable_attributes(next(variables), pitch_angle, PITCH_ANGLE_CDF_VAR_NAME, expected_record_varying=False)
52-
self.assert_variable_attributes(next(variables), pitch_angle_delta, PITCH_ANGLE_DELTA_CDF_VAR_NAME, expected_record_varying=False)
53-
self.assert_variable_attributes(next(variables), flux_by_pitch_angle, FLUX_BY_PITCH_ANGLE_CDF_VAR_NAME)
54-
self.assert_variable_attributes(next(variables), psd_by_pitch_angle, PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME)
55-
self.assert_variable_attributes(next(variables), energy_spectrum, ENERGY_SPECTRUM_CDF_VAR_NAME)
56-
self.assert_variable_attributes(next(variables), energy_spectrum_inbound, ENERGY_SPECTRUM_INBOUND_CDF_VAR_NAME)
57-
self.assert_variable_attributes(next(variables), energy_spectrum_outbound, ENERGY_SPECTRUM_OUTBOUND_CDF_VAR_NAME)
49+
self.assert_variable_attributes(
50+
next(variables), epoch, EPOCH_CDF_VAR_NAME, pycdf.const.CDF_TIME_TT2000)
51+
self.assert_variable_attributes(
52+
next(variables), [86400 * 1e9, 40 * 1e9], EPOCH_DELTA_CDF_VAR_NAME, pycdf.const.CDF_INT8)
53+
self.assert_variable_attributes(
54+
next(variables), energy, ENERGY_CDF_VAR_NAME, pycdf.const.CDF_REAL4, expected_record_varying=False)
55+
self.assert_variable_attributes(
56+
next(variables), energy_delta_plus, ENERGY_DELTA_PLUS_CDF_VAR_NAME, pycdf.const.CDF_REAL4, expected_record_varying=False)
57+
self.assert_variable_attributes(
58+
next(variables), energy_delta_minus, ENERGY_DELTA_MINUS_CDF_VAR_NAME, pycdf.const.CDF_REAL4, expected_record_varying=False)
59+
self.assert_variable_attributes(
60+
next(variables), pitch_angle, PITCH_ANGLE_CDF_VAR_NAME, pycdf.const.CDF_REAL4, expected_record_varying=False)
61+
self.assert_variable_attributes(
62+
next(variables), pitch_angle_delta, PITCH_ANGLE_DELTA_CDF_VAR_NAME, pycdf.const.CDF_REAL4, expected_record_varying=False)
63+
self.assert_variable_attributes(
64+
next(variables), flux_by_pitch_angle, FLUX_BY_PITCH_ANGLE_CDF_VAR_NAME, pycdf.const.CDF_REAL4)
65+
self.assert_variable_attributes(
66+
next(variables), psd_by_pitch_angle, PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME, pycdf.const.CDF_REAL4)
67+
self.assert_variable_attributes(
68+
next(variables), energy_spectrum, ENERGY_SPECTRUM_CDF_VAR_NAME, pycdf.const.CDF_REAL4)
69+
self.assert_variable_attributes(
70+
next(variables), energy_spectrum_inbound, ENERGY_SPECTRUM_INBOUND_CDF_VAR_NAME, pycdf.const.CDF_REAL4)
71+
self.assert_variable_attributes(
72+
next(variables), energy_spectrum_outbound, ENERGY_SPECTRUM_OUTBOUND_CDF_VAR_NAME, pycdf.const.CDF_REAL4)
5873

5974

6075
if __name__ == '__main__':

tests/test_data/swe/imap_mag_l1d_mago-normal_19990609_v001.cdf renamed to tests/test_data/swe/imap_mag_l1d_mago-normal_19990906_v001.cdf

File renamed without changes.

tests/test_data/swe/imap_swapi_l3a_proton-sw_19990609_v001.cdf renamed to tests/test_data/swe/imap_swapi_l3a_proton-sw_19990906_v001.cdf

File renamed without changes.

tests/test_data/swe/imap_swe_l2_sci-with-ace-data_19990609_v002.cdf renamed to tests/test_data/swe/imap_swe_l2_sci-with-ace-data_19990906_v002.cdf

File renamed without changes.

0 commit comments

Comments
 (0)