Harrison 2381 - PMCL/TLAY - Replace NaNs with fill value when writing CDF

Only do so if the input is a float and the fill value is specified.

Author: Menlo Innovations - CAVA Project

imap_processing/cdf/cdf_utils.py

@@ -22,10 +22,14 @@ def write_cdf(file_path: str, data: DataProduct, attribute_manager: ImapAttribut
 
         for data_product in data.to_data_product_variables():
             var_name = data_product.name
-            cdf.new(var_name, data=data_product.value,
+            variable_attributes = attribute_manager.get_variable_attributes(var_name)
+            data_array = np.asarray(data_product.value)
+            if 'FILLVAL' in variable_attributes and np.issubdtype(data_array.dtype, np.floating):
+                data_array = np.where(np.isnan(data_array), variable_attributes["FILLVAL"], data_array)
+            cdf.new(var_name, data_array,
                     recVary=data_product.record_varying,
                     type=data_product.cdf_data_type)
-            for k, v in attribute_manager.get_variable_attributes(var_name).items():
+            for k, v in variable_attributes.items():
                 if k == 'DEPEND_0' and v == '':
                     continue
                 if k == 'FILLVAL' and data_product.cdf_data_type is not None:

tests/cdf/test_cdf_utils.py

@@ -91,6 +91,36 @@ def test_write_cdf_trims_numbers_in_logical_source_when_fetching_global_metadata
             self.assertEqual('global_val1', str(actual_cdf.attrs['global1']))
             self.assertEqual('global_val2', str(actual_cdf.attrs['global2']))
 
+    def test_write_cdf_replaces_nan_with_fill_value(self):
+        epoch_data = [datetime(2025, 3, 7, 17, 0)]
+
+        class DataProductWithNan(DataProduct):
+            def __init__(self):
+                self.input_metadata = Mock()
+
+            def to_data_product_variables(self) -> list[DataProductVariable]:
+                return [
+                    DataProductVariable("epoch", epoch_data, pycdf.const.CDF_TIME_TT2000),
+                    DataProductVariable("float_var", np.array([3, 5, np.nan, 9, np.nan, np.nan]),
+                                        pycdf.const.CDF_REAL4),
+                ]
+
+        path = str(self.temp_directory / "write_cdf.cdf")
+        data = DataProductWithNan()
+        attribute_manager = Mock(spec=ImapAttributeManager)
+        attribute_manager.get_global_attributes.return_value = {}
+        attribute_manager.get_variable_attributes.side_effect = [
+            {"VAR_NAME": "epoch", "FILLVAL": datetime.fromisoformat("9999-12-31T23:59:59.999999999")},
+            {"VAR_NAME": "float_var", "FILLVAL": -1e31},
+        ]
+
+        write_cdf(path, data, attribute_manager)
+        with pycdf.CDF(path) as actual_cdf:
+            self.assertFalse(np.any(np.isnan(actual_cdf["float_var"][...])))
+            np.testing.assert_array_equal(np.array([3, 5, -1e31, 9, -1e31, -1e31], dtype=np.float32),
+                                          actual_cdf["float_var"][...], strict=True)
+            np.testing.assert_array_equal(actual_cdf["epoch"][...], np.array(epoch_data), strict=True)
+
     def test_does_not_write_depend_0_variable_attribute_if_it_is_empty(self):
         path = str(self.temp_directory / "write_cdf.cdf")
         data = TestDataProduct()