ensure missing values in aggregated variables are filled instead of set to NaN

+ update tests for aggregated_timeseries to check for these issues

Author: mhidas

aodntools/timeseries_products/aggregated_timeseries.py

@@ -39,7 +39,7 @@ def get_variable_values(nc, variable):
     Get values of the variable and its QC flags.
     If variable is not present, nan returned, its QC flags set to 9
     If variable present but not its QC flags, QC set to 0
-    :param nc: dataset
+    :param nc: xarray dataset
     :param variable: name of the variable to get
     :return: variable values and variable qc flags
     """
@@ -48,6 +48,8 @@ def get_variable_values(nc, variable):
 
     if variable in file_variables:
         variable_values = nc[variable].values
+        if any(np.isnan(variable_values)):
+            variable_values = np.ma.masked_array(variable_values, mask=np.isnan(variable_values))
         if variable+'_quality_control' in file_variables:
             variableQC_values = nc[variable+'_quality_control'].values
         else:
@@ -317,12 +319,12 @@ def main_aggregator(files_to_agg, var_to_agg, site_code, input_dir='', output_di
                     'site_code':                site_code,
                     'time_coverage_start':      time_start,
                     'time_coverage_end':        time_end,
-                    'geospatial_vertical_min':  np.min(ds['DEPTH']),
-                    'geospatial_vertical_max':  np.max(ds['DEPTH']),
-                    'geospatial_lat_min':       np.min(ds['LATITUDE']),
-                    'geospatial_lat_max':       np.max(ds['LATITUDE']),
-                    'geospatial_lon_min':       np.min(ds['LONGITUDE']),
-                    'geospatial_lon_max':       np.max(ds['LONGITUDE']),
+                    'geospatial_vertical_min':  np.min(ds['DEPTH'][:]),
+                    'geospatial_vertical_max':  np.max(ds['DEPTH'][:]),
+                    'geospatial_lat_min':       np.min(ds['LATITUDE'][:]),
+                    'geospatial_lat_max':       np.max(ds['LATITUDE'][:]),
+                    'geospatial_lon_min':       np.min(ds['LONGITUDE'][:]),
+                    'geospatial_lon_max':       np.max(ds['LONGITUDE'][:]),
                     'date_created':             datetime.utcnow().strftime(timeformat),
                     'history':                  datetime.utcnow().strftime(timeformat) + ': Aggregated file created.',
                     'keywords':                 ', '.join([var_to_agg, 'AGGREGATED']),

…ed-timeseries_END-20190523_C-20200622.nc …ed-timeseries_END-20190523_C-20220607.nctest_aodntools/timeseries_products/IMOS_ANMN-NRS_TZ_20181213_NRSROT_FV01_TEMP-aggregated-timeseries_END-20190523_C-20200622.nc renamed to test_aodntools/timeseries_products/IMOS_ANMN-NRS_TZ_20181213_NRSROT_FV01_TEMP-aggregated-timeseries_END-20190523_C-20220607.nc test_aodntools/timeseries_products/IMOS_ANMN-NRS_TZ_20181213_NRSROT_FV01_TEMP-aggregated-timeseries_END-20190523_C-20200622.nc renamed to test_aodntools/timeseries_products/IMOS_ANMN-NRS_TZ_20181213_NRSROT_FV01_TEMP-aggregated-timeseries_END-20190523_C-20220607.nc

@@ -3,6 +3,7 @@
 import os
 import unittest
 
+import numpy as np
 from netCDF4 import Dataset, chartostring
 
 from aodntools import __version__
@@ -19,7 +20,7 @@
     BAD_FILE
 ]
 EXPECTED_OUTPUT_FILE = os.path.join(
-    TEST_ROOT, 'IMOS_ANMN-NRS_TZ_20181213_NRSROT_FV01_TEMP-aggregated-timeseries_END-20190523_C-20200622.nc'
+    TEST_ROOT, 'IMOS_ANMN-NRS_TZ_20181213_NRSROT_FV01_TEMP-aggregated-timeseries_END-20190523_C-20220607.nc'
 )
 
 
@@ -69,13 +70,29 @@ def test_main_aggregator(self):
         self.assertIn(__version__, dataset.lineage)
         self.assertIn(BAD_FILE, dataset.rejected_files)
 
-        # check aggregated variable values
+        compare_attrs = ('Conventions', 'feature_type',  'author', 'author_email', 'file_version',
+                         'geospatial_lat_max', 'geospatial_lat_min', 'geospatial_lon_max', 'geospatial_lon_min',
+                         'geospatial_vertical_max', 'geospatial_vertical_min', 'naming_authority', 'project',
+                         'time_coverage_start', 'time_coverage_end'
+                         )
         expected = Dataset(EXPECTED_OUTPUT_FILE)
-        compare_vars = ('TIME', 'TEMP', 'TEMP_quality_control', 'NOMINAL_DEPTH', 'instrument_index')
-        non_match_vars = [var for var in compare_vars
-                          if not all(dataset[var][:] == expected[var][:])
-                          ]
-        self.assertEqual(non_match_vars, [])
+        for attr in compare_attrs:
+            self.assertEqual(dataset.getncattr(attr), expected.getncattr(attr))
+
+        # check that there are no NaN values in any variable (they should be fill values instead)
+        nan_vars = [name
+                    for name, var in dataset.variables.items()
+                    if var.dtype in (np.dtype('float32'), np.dtype('float64')) and any(np.isnan(var[:]))
+                    ]
+        self.assertEqual([], nan_vars)
+
+        # check aggregated variable values
+        non_match_vars = []
+        for var in set(expected.variables.keys()) - string_vars:
+            # compare the raw data arrays (not the masked_array)
+            if not all(dataset[var][:].data == expected[var][:].data):
+                non_match_vars.append(var)
+        self.assertEqual([], non_match_vars)
 
     def test_source_file_attributes(self):
         output_file, bad_files = main_aggregator(INPUT_FILES, 'PSAL', 'NRSROT', input_dir=TEST_ROOT,