fix NaN variable values in the remaining products (fixes #118)

Author: mhidas

aodntools/timeseries_products/velocity_aggregated_timeseries.py

@@ -144,7 +144,7 @@ def velocity_aggregated(files_to_agg, site_code, input_dir='', output_dir='./',
                 WCUR[start:end] = flat_variable(nc, 'WCUR')
                 WCURqc[start:end] = flat_variable(nc, 'WCUR_quality_control')
             else:
-                WCUR[start:end] = np.full(n_obs, np.nan)
+                WCUR[start:end] = np.ma.masked
                 WCURqc[start:end] = np.full(n_obs, 9)
 
             ##calculate depth and add CELL_INDEX

aodntools/timeseries_products/velocity_hourly_timeseries.py

@@ -32,16 +32,15 @@ def cell_velocity_resample(df, binning_function):
     :return: binned U, v, W CUR according to the binning function
     """
     df_binned = df.apply(binning_function)
-    UCUR = np.array(df_binned['UCUR'])
-    VCUR = np.array(df_binned['VCUR'])
-    if 'WCUR' in df_binned:
-        WCUR = np.array(df_binned['WCUR'])
-    else:
-        WCUR = np.full(len(df), np.nan)
-    DEPTH = np.array(df_binned['DEPTH'])
-
-    return UCUR, VCUR, WCUR, DEPTH
-
+    binned_vars = []
+    for var in ('UCUR', 'VCUR', 'WCUR', 'DEPTH'):
+        if var in df_binned:
+            x = np.ma.masked_array(df_binned[var], mask=np.isnan(df_binned[var]))
+        else:
+            x = np.ma.masked
+        binned_vars.append(x)
+
+    return tuple(binned_vars)
 
 def append_resampled_values(nc_cell, ds, slice_start, binning_functions):
     """

test_aodntools/timeseries_products/IMOS_ANMN-NRS_AETVZ_20191016T080000Z_NRSROT-ADCP_FV01_NRSROT-ADCP-1910-Sentinel-or-Monitor-Workhorse-ADCP-44_END-20191018T100000Z_C-20200430T000000Z.nc

@@ -141,6 +141,8 @@ def test_some_files_without_good_data(self):
         for path, errors in bad_files.items():
             self.assertEqual(NO_INWATER_DATA_FILE, path)
             self.assertIn('no in-water data', errors)
+        with Dataset(output_file) as dataset:
+            self.check_nan_values(dataset)
 
     def test_bad_timestamps(self):
         output_file, bad_files = hourly_aggregator(files_to_aggregate=SYD100_FILES,

test_aodntools/timeseries_products/IMOS_ANMN-NRS_VZ_20180816_NRSROT_FV01_velocity-aggregated-timeseries_END-20191018_C-20200623.nc

@@ -31,7 +31,7 @@
 
 class TestVelocityHourlyTimeseries(BaseTestCase):
     EXPECTED_OUTPUT_FILE = os.path.join(
-        TEST_ROOT, 'IMOS_ANMN-NRS_VZ_20180816_NRSROT_FV02_velocity-hourly-timeseries_END-20191018_C-20220502.nc'
+        TEST_ROOT, 'IMOS_ANMN-NRS_VZ_20180816_NRSROT_FV02_velocity-hourly-timeseries_END-20191018_C-20220608.nc'
     )
 
     def test_velocity_hourly(self):