Skip to content

Commit 5d80cf0

Browse files
author
Menlo Innovations - CAVA Project
committed
Harrison 2381 - PMCL/TLAY - FILLVAL data type matches variable datatype if FILLVAL is specified
1 parent b726929 commit 5d80cf0

2 files changed

Lines changed: 39 additions & 15 deletions

File tree

‎imap_processing/cdf/cdf_utils.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,10 @@ def write_cdf(file_path: str, data: DataProduct, attribute_manager: ImapAttribut
2828
for k, v in attribute_manager.get_variable_attributes(var_name).items():
2929
if k == 'DEPEND_0' and v == '':
3030
continue
31-
cdf[var_name].attrs[k] = v
31+
if k == 'FILLVAL' and data_product.cdf_data_type is not None:
32+
cdf[var_name].attrs.new(k, v, data_product.cdf_data_type)
33+
else:
34+
cdf[var_name].attrs[k] = v
3235

3336

3437
def read_variable(var: pycdf.Var) -> np.ndarray:

‎tests/cdf/test_cdf_utils.py‎

Lines changed: 35 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,14 @@ class TestCdfUtils(TempFileTestCase):
1414
def test_write_cdf(self):
1515
path = str(self.temp_directory / "write_cdf.cdf")
1616
data = TestDataProduct()
17-
regular_var, time_var, non_rec_varying_var = data.to_data_product_variables()
17+
var_without_explicit_type, int_var, float_var, time_var, non_rec_varying_var = data.to_data_product_variables()
1818
attribute_manager = Mock(spec=ImapAttributeManager)
1919
attribute_manager.get_global_attributes.return_value = {"global1": "global_val1", "global2": "global_val2"}
2020
attribute_manager.get_variable_attributes.side_effect = [
21-
{"variable_attr1": "var_val1", "variable_attr2": "var_val2"},
22-
{"variable_attr3": "var_val3", "variable_attr4": "var_val4"},
21+
{"variable_attr1": "var_val1"},
22+
{"variable_attr1": "var_val1", "FILLVAL": -9223372036854775808},
23+
{"variable_attr1": "var_val1", "FILLVAL": -1e31},
24+
{"variable_attr3": "var_val3", "FILLVAL": -9223372036854775808},
2325
{"variable_attr5": "var_val5", "variable_attr6": "var_val6"},
2426
]
2527

@@ -36,15 +38,30 @@ def test_write_cdf(self):
3638
self.assertEqual('global_val1', actual_cdf.attrs['global1'][...][0])
3739
self.assertEqual('global_val2', actual_cdf.attrs['global2'][...][0])
3840

39-
np.testing.assert_array_equal(regular_var.value, actual_cdf[regular_var.name][...])
40-
self.assertEqual('var_val1', actual_cdf[regular_var.name].attrs['variable_attr1'])
41-
self.assertEqual('var_val2', actual_cdf[regular_var.name].attrs['variable_attr2'])
42-
self.assertEqual(pycdf.const.CDF_INT8.value, actual_cdf[regular_var.name].type())
43-
self.assertTrue(actual_cdf[regular_var.name].rv())
44-
np.testing.assert_array_equal(time_var.value, actual_cdf.raw_var(time_var.name))
41+
np.testing.assert_array_equal(var_without_explicit_type.value,
42+
actual_cdf[var_without_explicit_type.name][...])
43+
self.assertEqual('var_val1', actual_cdf[var_without_explicit_type.name].attrs['variable_attr1'])
44+
self.assertEqual(pycdf.const.CDF_INT8.value, actual_cdf[var_without_explicit_type.name].type())
45+
self.assertTrue(actual_cdf[var_without_explicit_type.name].rv())
46+
47+
np.testing.assert_array_equal(int_var.value, actual_cdf.raw_var(int_var.name))
48+
self.assertEqual('var_val1', actual_cdf[int_var.name].attrs['variable_attr1'])
49+
self.assertEqual(-9223372036854775808, actual_cdf[int_var.name].attrs['FILLVAL'])
50+
self.assertEqual(pycdf.const.CDF_INT8.value, actual_cdf[int_var.name].attrs.type("FILLVAL"))
51+
self.assertEqual(pycdf.const.CDF_INT8.value, actual_cdf[int_var.name].type())
52+
self.assertTrue(actual_cdf[int_var.name].rv())
53+
54+
np.testing.assert_array_equal(float_var.value, actual_cdf.raw_var(float_var.name))
55+
self.assertEqual('var_val1', actual_cdf[float_var.name].attrs['variable_attr1'])
56+
self.assertEqual(-1e31, actual_cdf[float_var.name].attrs['FILLVAL'])
57+
self.assertEqual(pycdf.const.CDF_REAL4.value, actual_cdf[float_var.name].attrs.type("FILLVAL"))
58+
self.assertEqual(pycdf.const.CDF_REAL4.value, actual_cdf[float_var.name].type())
59+
self.assertTrue(actual_cdf[float_var.name].rv())
4560

61+
np.testing.assert_array_equal(time_var.value, actual_cdf.raw_var(time_var.name))
4662
self.assertEqual('var_val3', actual_cdf[time_var.name].attrs['variable_attr3'])
47-
self.assertEqual('var_val4', actual_cdf[time_var.name].attrs['variable_attr4'])
63+
self.assertEqual(datetime(9999, 12, 31, 23, 59, 59, 999999), actual_cdf[time_var.name].attrs['FILLVAL'])
64+
self.assertEqual(pycdf.const.CDF_TIME_TT2000.value, actual_cdf[time_var.name].attrs.type("FILLVAL"))
4865
self.assertEqual(pycdf.const.CDF_TIME_TT2000.value, actual_cdf[time_var.name].type())
4966
self.assertTrue(actual_cdf[time_var.name].rv())
5067

@@ -77,10 +94,12 @@ def test_write_cdf_trims_numbers_in_logical_source_when_fetching_global_metadata
7794
def test_does_not_write_depend_0_variable_attribute_if_it_is_empty(self):
7895
path = str(self.temp_directory / "write_cdf.cdf")
7996
data = TestDataProduct()
80-
regular_var, time_var, non_rec_varying_var = data.to_data_product_variables()
97+
_, _, regular_var, time_var, non_rec_varying_var = data.to_data_product_variables()
8198
attribute_manager = Mock(spec=ImapAttributeManager)
8299
attribute_manager.get_global_attributes.return_value = {"global1": "global_val1", "global2": "global_val2"}
83100
attribute_manager.get_variable_attributes.side_effect = [
101+
{"ignored_attr": "var_val3", "variable_attr4": "var_val4", "DEPEND_0": ""},
102+
{"ignored_attr": "var_val3", "variable_attr4": "var_val4", "DEPEND_0": ""},
84103
{"variable_attr1": "var_val1", "variable_attr2": "var_val2", "DEPEND_0": "epoch"},
85104
{"variable_attr3": "var_val3", "variable_attr4": "var_val4", "DEPEND_0": ""},
86105
{"variable_attr5": "var_val5", "variable_attr6": "var_val6", "DEPEND_0": ""},
@@ -114,7 +133,9 @@ def __init__(self):
114133

115134
def to_data_product_variables(self) -> list[DataProductVariable]:
116135
return [
117-
DataProductVariable("var1", np.arange(0, 10)),
118-
DataProductVariable("var2", np.arange(10, 20), pycdf.const.CDF_TIME_TT2000),
119-
DataProductVariable("var3", 100, record_varying=False)
136+
DataProductVariable("var_without_explicit_type", np.arange(0, 10)),
137+
DataProductVariable("int_var", np.arange(0, 10), pycdf.const.CDF_INT8),
138+
DataProductVariable("float_var", np.arange(0, 10), pycdf.const.CDF_REAL4),
139+
DataProductVariable("time_var", np.arange(10, 20), pycdf.const.CDF_TIME_TT2000),
140+
DataProductVariable("non_record_varying", 100, record_varying=False)
120141
]

0 commit comments

Comments
 (0)